Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions docker-compose/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ services:
source: ./s3-credentials.yaml
target: /bin/config/override.yaml
ports:
- "9671:9671" # expose REST api for chorctl and web UI
- "9670:9670" # expose grpc management API
- "127.0.0.1:9671:9671" # expose REST api for chorctl and web UI
- "127.0.0.1:9670:9670" # expose grpc management API
# Chorus S3 proxy
# Can be used capture data change and backup it with worker to another S3.
# Can be used to migrate to another S3 by bucket. Before migration it will route all requests to old S3.
Expand All @@ -60,7 +60,7 @@ services:
source: ./s3-credentials.yaml
target: /bin/config/override.yaml
ports:
- "9669:9669" # expose proxy s3 api
- "127.0.0.1:9669:9669" # expose proxy s3 api
# Chorus web UI
# Access UI on http://localhost:8080
web-ui:
Expand All @@ -70,7 +70,7 @@ services:
context: ../ui
dockerfile: Dockerfile
ports:
- "8080:80"
- "127.0.0.1:8080:80"
environment:
# URL to worker REST API
- "BACKEND_URL=http://worker:9671"
Expand All @@ -84,7 +84,7 @@ services:
expose:
- 9000
ports:
- "9001:9000" # expose main s3 to host
- "127.0.0.1:9001:9000" # expose main s3 to host
fake-s3-follower:
profiles:
- "fake"
Expand All @@ -94,5 +94,5 @@ services:
expose:
- 9000
ports:
- "9002:9000" # expose follower s3 to host
- "127.0.0.1:9002:9000" # expose follower s3 to host

94 changes: 94 additions & 0 deletions pkg/api/chorus_handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,23 +20,28 @@ import (
"context"
"fmt"

"github.com/minio/minio-go/v7"
"google.golang.org/protobuf/types/known/emptypb"
"google.golang.org/protobuf/types/known/timestamppb"

"github.com/clyso/chorus/pkg/dom"
"github.com/clyso/chorus/pkg/objstore"
"github.com/clyso/chorus/pkg/rpc"
"github.com/clyso/chorus/pkg/s3"
"github.com/clyso/chorus/pkg/s3client"
"github.com/clyso/chorus/pkg/swift"
pb "github.com/clyso/chorus/proto/gen/go/chorus"
)

func ChorusHandlers(
credsSvc objstore.CredsService,
clientRegistry objstore.Clients,
proxyClient rpc.Proxy,
appInfo *dom.AppInfo,
) pb.ChorusServer {
return &chorusHandlers{
credsSvc: credsSvc,
clients: clientRegistry,
proxyClient: proxyClient,
appInfo: appInfo,
}
Expand All @@ -48,6 +53,7 @@ type chorusHandlers struct {
proxyClient rpc.Proxy
appInfo *dom.AppInfo
credsSvc objstore.CredsService
clients objstore.Clients
}

func (h *chorusHandlers) GetAppVersion(_ context.Context, _ *emptypb.Empty) (*pb.GetAppVersionResponse, error) {
Expand All @@ -63,6 +69,94 @@ func (h *chorusHandlers) GetStorages(_ context.Context, _ *emptypb.Empty) (*pb.G
return &pb.GetStoragesResponse{Storages: res}, nil
}

func (h *chorusHandlers) ListBrowserBuckets(ctx context.Context, req *pb.ListBrowserBucketsRequest) (*pb.ListBrowserBucketsResponse, error) {
client, err := h.browserClient(ctx, req.Storage, req.User)
if err != nil {
return nil, err
}
buckets, err := client.S3().ListBuckets(ctx)
if err != nil {
return nil, fmt.Errorf("unable to list buckets: %w", err)
}

result := make([]string, 0, len(buckets))
for _, bucket := range buckets {
result = append(result, bucket.Name)
}
return &pb.ListBrowserBucketsResponse{Buckets: result}, nil
}

func (h *chorusHandlers) ListBrowserObjects(ctx context.Context, req *pb.ListBrowserObjectsRequest) (*pb.ListBrowserObjectsResponse, error) {
if req.Bucket == "" {
return nil, fmt.Errorf("%w: bucket is required", dom.ErrInvalidArg)
}
client, err := h.browserClient(ctx, req.Storage, req.User)
if err != nil {
return nil, err
}

limit := req.Limit
if limit == 0 {
limit = 50
} else if limit > 100 {
limit = 100
}

listed := make([]minio.ObjectInfo, 0, limit+1)
objects := client.S3().ListObjects(ctx, req.Bucket, minio.ListObjectsOptions{
Prefix: req.Prefix,
StartAfter: req.After,
Recursive: true,
MaxKeys: int(limit + 1),
})
for object := range objects {
if object.Err != nil {
return nil, fmt.Errorf("unable to list objects: %w", object.Err)
}
listed = append(listed, object)
}

hasMore := len(listed) > int(limit)
if hasMore {
listed = listed[:limit]
}

result := &pb.ListBrowserObjectsResponse{
Objects: make([]*pb.BrowserObject, 0, len(listed)),
HasMore: hasMore,
}
for _, object := range listed {
info, err := client.S3().StatObject(ctx, req.Bucket, object.Key, minio.StatObjectOptions{})
if err != nil {
return nil, fmt.Errorf("unable to read metadata for object %q: %w", object.Key, err)
}
result.Objects = append(result.Objects, &pb.BrowserObject{
Key: object.Key,
Size: uint64(object.Size),
LastModified: timestamppb.New(object.LastModified),
SourceLastModified: info.Metadata.Get("x-amz-meta-chorus-source-last-modified"),
})
}
if hasMore && len(listed) > 0 {
result.NextAfter = listed[len(listed)-1].Key
}
return result, nil
}

func (h *chorusHandlers) browserClient(ctx context.Context, storage, user string) (s3client.Client, error) {
if storage == "" || user == "" {
return nil, fmt.Errorf("%w: storage and user are required", dom.ErrInvalidArg)
}
storageType, ok := h.credsSvc.Storages()[storage]
if !ok {
return nil, fmt.Errorf("%w: storage %q not found", dom.ErrNotFound, storage)
}
if storageType != dom.S3 {
return nil, fmt.Errorf("%w: object browser only supports S3 storage", dom.ErrInvalidArg)
}
return h.clients.AsS3(ctx, storage, user)
}

func (h *chorusHandlers) GetProxyCredentials(ctx context.Context, _ *emptypb.Empty) (*pb.GetProxyCredentialsResponse, error) {
return h.proxyClient.GetCredentials(ctx)
}
Expand Down
35 changes: 35 additions & 0 deletions proto/chorus/chorus.proto
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@ service Chorus {
rpc GetAppVersion(google.protobuf.Empty) returns (GetAppVersionResponse);
// Lists configured storages with users
rpc GetStorages(google.protobuf.Empty) returns (GetStoragesResponse);
// Lists buckets for an object browser.
rpc ListBrowserBuckets(ListBrowserBucketsRequest)
returns (ListBrowserBucketsResponse);
// Lists a page of objects and their original migration timestamps.
rpc ListBrowserObjects(ListBrowserObjectsRequest)
returns (ListBrowserObjectsResponse);
// Returns connection details for proxy s3 endpoint
rpc GetProxyCredentials(google.protobuf.Empty)
returns (GetProxyCredentialsResponse);
Expand All @@ -29,6 +35,35 @@ message GetAppVersionResponse {

message GetStoragesResponse { repeated Storage storages = 1; }

message ListBrowserBucketsRequest {
string storage = 1;
string user = 2;
}

message ListBrowserBucketsResponse { repeated string buckets = 1; }

message ListBrowserObjectsRequest {
string storage = 1;
string user = 2;
string bucket = 3;
string prefix = 4;
string after = 5;
uint32 limit = 6;
}

message BrowserObject {
string key = 1;
uint64 size = 2;
google.protobuf.Timestamp last_modified = 3;
string source_last_modified = 4;
}

message ListBrowserObjectsResponse {
repeated BrowserObject objects = 1;
string next_after = 2;
bool has_more = 3;
}

message Storage {
enum Type {
S3 = 0;
Expand Down
Loading
Loading