diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 6a8ee387..bc4e6b46 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -34,8 +34,8 @@ services: source: ./s3-credentials.yaml target: /bin/config/override.yaml ports: - - "9671:9671" # expose REST api for chorctl and web UI - - "9670:9670" # expose grpc management API + - "127.0.0.1:9671:9671" # expose REST api for chorctl and web UI + - "127.0.0.1:9670:9670" # expose grpc management API # Chorus S3 proxy # Can be used capture data change and backup it with worker to another S3. # Can be used to migrate to another S3 by bucket. Before migration it will route all requests to old S3. @@ -60,7 +60,7 @@ services: source: ./s3-credentials.yaml target: /bin/config/override.yaml ports: - - "9669:9669" # expose proxy s3 api + - "127.0.0.1:9669:9669" # expose proxy s3 api # Chorus web UI # Access UI on http://localhost:8080 web-ui: @@ -70,7 +70,7 @@ services: context: ../ui dockerfile: Dockerfile ports: - - "8080:80" + - "127.0.0.1:8080:80" environment: # URL to worker REST API - "BACKEND_URL=http://worker:9671" @@ -84,7 +84,7 @@ services: expose: - 9000 ports: - - "9001:9000" # expose main s3 to host + - "127.0.0.1:9001:9000" # expose main s3 to host fake-s3-follower: profiles: - "fake" @@ -94,5 +94,5 @@ services: expose: - 9000 ports: - - "9002:9000" # expose follower s3 to host + - "127.0.0.1:9002:9000" # expose follower s3 to host diff --git a/pkg/api/chorus_handlers.go b/pkg/api/chorus_handlers.go index d09b48f0..246cb9ab 100644 --- a/pkg/api/chorus_handlers.go +++ b/pkg/api/chorus_handlers.go @@ -20,23 +20,28 @@ import ( "context" "fmt" + "github.com/minio/minio-go/v7" "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/clyso/chorus/pkg/dom" "github.com/clyso/chorus/pkg/objstore" "github.com/clyso/chorus/pkg/rpc" "github.com/clyso/chorus/pkg/s3" + "github.com/clyso/chorus/pkg/s3client" "github.com/clyso/chorus/pkg/swift" pb "github.com/clyso/chorus/proto/gen/go/chorus" ) func ChorusHandlers( credsSvc objstore.CredsService, + clientRegistry objstore.Clients, proxyClient rpc.Proxy, appInfo *dom.AppInfo, ) pb.ChorusServer { return &chorusHandlers{ credsSvc: credsSvc, + clients: clientRegistry, proxyClient: proxyClient, appInfo: appInfo, } @@ -48,6 +53,7 @@ type chorusHandlers struct { proxyClient rpc.Proxy appInfo *dom.AppInfo credsSvc objstore.CredsService + clients objstore.Clients } func (h *chorusHandlers) GetAppVersion(_ context.Context, _ *emptypb.Empty) (*pb.GetAppVersionResponse, error) { @@ -63,6 +69,94 @@ func (h *chorusHandlers) GetStorages(_ context.Context, _ *emptypb.Empty) (*pb.G return &pb.GetStoragesResponse{Storages: res}, nil } +func (h *chorusHandlers) ListBrowserBuckets(ctx context.Context, req *pb.ListBrowserBucketsRequest) (*pb.ListBrowserBucketsResponse, error) { + client, err := h.browserClient(ctx, req.Storage, req.User) + if err != nil { + return nil, err + } + buckets, err := client.S3().ListBuckets(ctx) + if err != nil { + return nil, fmt.Errorf("unable to list buckets: %w", err) + } + + result := make([]string, 0, len(buckets)) + for _, bucket := range buckets { + result = append(result, bucket.Name) + } + return &pb.ListBrowserBucketsResponse{Buckets: result}, nil +} + +func (h *chorusHandlers) ListBrowserObjects(ctx context.Context, req *pb.ListBrowserObjectsRequest) (*pb.ListBrowserObjectsResponse, error) { + if req.Bucket == "" { + return nil, fmt.Errorf("%w: bucket is required", dom.ErrInvalidArg) + } + client, err := h.browserClient(ctx, req.Storage, req.User) + if err != nil { + return nil, err + } + + limit := req.Limit + if limit == 0 { + limit = 50 + } else if limit > 100 { + limit = 100 + } + + listed := make([]minio.ObjectInfo, 0, limit+1) + objects := client.S3().ListObjects(ctx, req.Bucket, minio.ListObjectsOptions{ + Prefix: req.Prefix, + StartAfter: req.After, + Recursive: true, + MaxKeys: int(limit + 1), + }) + for object := range objects { + if object.Err != nil { + return nil, fmt.Errorf("unable to list objects: %w", object.Err) + } + listed = append(listed, object) + } + + hasMore := len(listed) > int(limit) + if hasMore { + listed = listed[:limit] + } + + result := &pb.ListBrowserObjectsResponse{ + Objects: make([]*pb.BrowserObject, 0, len(listed)), + HasMore: hasMore, + } + for _, object := range listed { + info, err := client.S3().StatObject(ctx, req.Bucket, object.Key, minio.StatObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("unable to read metadata for object %q: %w", object.Key, err) + } + result.Objects = append(result.Objects, &pb.BrowserObject{ + Key: object.Key, + Size: uint64(object.Size), + LastModified: timestamppb.New(object.LastModified), + SourceLastModified: info.Metadata.Get("x-amz-meta-chorus-source-last-modified"), + }) + } + if hasMore && len(listed) > 0 { + result.NextAfter = listed[len(listed)-1].Key + } + return result, nil +} + +func (h *chorusHandlers) browserClient(ctx context.Context, storage, user string) (s3client.Client, error) { + if storage == "" || user == "" { + return nil, fmt.Errorf("%w: storage and user are required", dom.ErrInvalidArg) + } + storageType, ok := h.credsSvc.Storages()[storage] + if !ok { + return nil, fmt.Errorf("%w: storage %q not found", dom.ErrNotFound, storage) + } + if storageType != dom.S3 { + return nil, fmt.Errorf("%w: object browser only supports S3 storage", dom.ErrInvalidArg) + } + return h.clients.AsS3(ctx, storage, user) +} + func (h *chorusHandlers) GetProxyCredentials(ctx context.Context, _ *emptypb.Empty) (*pb.GetProxyCredentialsResponse, error) { return h.proxyClient.GetCredentials(ctx) } diff --git a/proto/chorus/chorus.proto b/proto/chorus/chorus.proto index 6f7028bb..1c610a32 100644 --- a/proto/chorus/chorus.proto +++ b/proto/chorus/chorus.proto @@ -13,6 +13,12 @@ service Chorus { rpc GetAppVersion(google.protobuf.Empty) returns (GetAppVersionResponse); // Lists configured storages with users rpc GetStorages(google.protobuf.Empty) returns (GetStoragesResponse); + // Lists buckets for an object browser. + rpc ListBrowserBuckets(ListBrowserBucketsRequest) + returns (ListBrowserBucketsResponse); + // Lists a page of objects and their original migration timestamps. + rpc ListBrowserObjects(ListBrowserObjectsRequest) + returns (ListBrowserObjectsResponse); // Returns connection details for proxy s3 endpoint rpc GetProxyCredentials(google.protobuf.Empty) returns (GetProxyCredentialsResponse); @@ -29,6 +35,35 @@ message GetAppVersionResponse { message GetStoragesResponse { repeated Storage storages = 1; } +message ListBrowserBucketsRequest { + string storage = 1; + string user = 2; +} + +message ListBrowserBucketsResponse { repeated string buckets = 1; } + +message ListBrowserObjectsRequest { + string storage = 1; + string user = 2; + string bucket = 3; + string prefix = 4; + string after = 5; + uint32 limit = 6; +} + +message BrowserObject { + string key = 1; + uint64 size = 2; + google.protobuf.Timestamp last_modified = 3; + string source_last_modified = 4; +} + +message ListBrowserObjectsResponse { + repeated BrowserObject objects = 1; + string next_after = 2; + bool has_more = 3; +} + message Storage { enum Type { S3 = 0; diff --git a/proto/gen/go/chorus/chorus.pb.go b/proto/gen/go/chorus/chorus.pb.go index 733b4ad4..a5b364d3 100644 --- a/proto/gen/go/chorus/chorus.pb.go +++ b/proto/gen/go/chorus/chorus.pb.go @@ -11,7 +11,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" _ "google.golang.org/protobuf/types/known/durationpb" emptypb "google.golang.org/protobuf/types/known/emptypb" - _ "google.golang.org/protobuf/types/known/timestamppb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" unsafe "unsafe" @@ -67,7 +67,7 @@ func (x Storage_Type) Number() protoreflect.EnumNumber { // Deprecated: Use Storage_Type.Descriptor instead. func (Storage_Type) EnumDescriptor() ([]byte, []int) { - return file_chorus_chorus_proto_rawDescGZIP(), []int{2, 0} + return file_chorus_chorus_proto_rawDescGZIP(), []int{7, 0} } type GetAppVersionResponse struct { @@ -174,6 +174,314 @@ func (x *GetStoragesResponse) GetStorages() []*Storage { return nil } +type ListBrowserBucketsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Storage string `protobuf:"bytes,1,opt,name=storage,proto3" json:"storage,omitempty"` + User string `protobuf:"bytes,2,opt,name=user,proto3" json:"user,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBrowserBucketsRequest) Reset() { + *x = ListBrowserBucketsRequest{} + mi := &file_chorus_chorus_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBrowserBucketsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBrowserBucketsRequest) ProtoMessage() {} + +func (x *ListBrowserBucketsRequest) ProtoReflect() protoreflect.Message { + mi := &file_chorus_chorus_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBrowserBucketsRequest.ProtoReflect.Descriptor instead. +func (*ListBrowserBucketsRequest) Descriptor() ([]byte, []int) { + return file_chorus_chorus_proto_rawDescGZIP(), []int{2} +} + +func (x *ListBrowserBucketsRequest) GetStorage() string { + if x != nil { + return x.Storage + } + return "" +} + +func (x *ListBrowserBucketsRequest) GetUser() string { + if x != nil { + return x.User + } + return "" +} + +type ListBrowserBucketsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Buckets []string `protobuf:"bytes,1,rep,name=buckets,proto3" json:"buckets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBrowserBucketsResponse) Reset() { + *x = ListBrowserBucketsResponse{} + mi := &file_chorus_chorus_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBrowserBucketsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBrowserBucketsResponse) ProtoMessage() {} + +func (x *ListBrowserBucketsResponse) ProtoReflect() protoreflect.Message { + mi := &file_chorus_chorus_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBrowserBucketsResponse.ProtoReflect.Descriptor instead. +func (*ListBrowserBucketsResponse) Descriptor() ([]byte, []int) { + return file_chorus_chorus_proto_rawDescGZIP(), []int{3} +} + +func (x *ListBrowserBucketsResponse) GetBuckets() []string { + if x != nil { + return x.Buckets + } + return nil +} + +type ListBrowserObjectsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Storage string `protobuf:"bytes,1,opt,name=storage,proto3" json:"storage,omitempty"` + User string `protobuf:"bytes,2,opt,name=user,proto3" json:"user,omitempty"` + Bucket string `protobuf:"bytes,3,opt,name=bucket,proto3" json:"bucket,omitempty"` + Prefix string `protobuf:"bytes,4,opt,name=prefix,proto3" json:"prefix,omitempty"` + After string `protobuf:"bytes,5,opt,name=after,proto3" json:"after,omitempty"` + Limit uint32 `protobuf:"varint,6,opt,name=limit,proto3" json:"limit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBrowserObjectsRequest) Reset() { + *x = ListBrowserObjectsRequest{} + mi := &file_chorus_chorus_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBrowserObjectsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBrowserObjectsRequest) ProtoMessage() {} + +func (x *ListBrowserObjectsRequest) ProtoReflect() protoreflect.Message { + mi := &file_chorus_chorus_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBrowserObjectsRequest.ProtoReflect.Descriptor instead. +func (*ListBrowserObjectsRequest) Descriptor() ([]byte, []int) { + return file_chorus_chorus_proto_rawDescGZIP(), []int{4} +} + +func (x *ListBrowserObjectsRequest) GetStorage() string { + if x != nil { + return x.Storage + } + return "" +} + +func (x *ListBrowserObjectsRequest) GetUser() string { + if x != nil { + return x.User + } + return "" +} + +func (x *ListBrowserObjectsRequest) GetBucket() string { + if x != nil { + return x.Bucket + } + return "" +} + +func (x *ListBrowserObjectsRequest) GetPrefix() string { + if x != nil { + return x.Prefix + } + return "" +} + +func (x *ListBrowserObjectsRequest) GetAfter() string { + if x != nil { + return x.After + } + return "" +} + +func (x *ListBrowserObjectsRequest) GetLimit() uint32 { + if x != nil { + return x.Limit + } + return 0 +} + +type BrowserObject struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Size uint64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + LastModified *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=last_modified,json=lastModified,proto3" json:"last_modified,omitempty"` + SourceLastModified string `protobuf:"bytes,4,opt,name=source_last_modified,json=sourceLastModified,proto3" json:"source_last_modified,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BrowserObject) Reset() { + *x = BrowserObject{} + mi := &file_chorus_chorus_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BrowserObject) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BrowserObject) ProtoMessage() {} + +func (x *BrowserObject) ProtoReflect() protoreflect.Message { + mi := &file_chorus_chorus_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BrowserObject.ProtoReflect.Descriptor instead. +func (*BrowserObject) Descriptor() ([]byte, []int) { + return file_chorus_chorus_proto_rawDescGZIP(), []int{5} +} + +func (x *BrowserObject) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *BrowserObject) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *BrowserObject) GetLastModified() *timestamppb.Timestamp { + if x != nil { + return x.LastModified + } + return nil +} + +func (x *BrowserObject) GetSourceLastModified() string { + if x != nil { + return x.SourceLastModified + } + return "" +} + +type ListBrowserObjectsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Objects []*BrowserObject `protobuf:"bytes,1,rep,name=objects,proto3" json:"objects,omitempty"` + NextAfter string `protobuf:"bytes,2,opt,name=next_after,json=nextAfter,proto3" json:"next_after,omitempty"` + HasMore bool `protobuf:"varint,3,opt,name=has_more,json=hasMore,proto3" json:"has_more,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBrowserObjectsResponse) Reset() { + *x = ListBrowserObjectsResponse{} + mi := &file_chorus_chorus_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBrowserObjectsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBrowserObjectsResponse) ProtoMessage() {} + +func (x *ListBrowserObjectsResponse) ProtoReflect() protoreflect.Message { + mi := &file_chorus_chorus_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBrowserObjectsResponse.ProtoReflect.Descriptor instead. +func (*ListBrowserObjectsResponse) Descriptor() ([]byte, []int) { + return file_chorus_chorus_proto_rawDescGZIP(), []int{6} +} + +func (x *ListBrowserObjectsResponse) GetObjects() []*BrowserObject { + if x != nil { + return x.Objects + } + return nil +} + +func (x *ListBrowserObjectsResponse) GetNextAfter() string { + if x != nil { + return x.NextAfter + } + return "" +} + +func (x *ListBrowserObjectsResponse) GetHasMore() bool { + if x != nil { + return x.HasMore + } + return false +} + type Storage struct { state protoimpl.MessageState `protogen:"open.v1"` // some human-readable alias for storage config @@ -190,7 +498,7 @@ type Storage struct { func (x *Storage) Reset() { *x = Storage{} - mi := &file_chorus_chorus_proto_msgTypes[2] + mi := &file_chorus_chorus_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -202,7 +510,7 @@ func (x *Storage) String() string { func (*Storage) ProtoMessage() {} func (x *Storage) ProtoReflect() protoreflect.Message { - mi := &file_chorus_chorus_proto_msgTypes[2] + mi := &file_chorus_chorus_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -215,7 +523,7 @@ func (x *Storage) ProtoReflect() protoreflect.Message { // Deprecated: Use Storage.ProtoReflect.Descriptor instead. func (*Storage) Descriptor() ([]byte, []int) { - return file_chorus_chorus_proto_rawDescGZIP(), []int{2} + return file_chorus_chorus_proto_rawDescGZIP(), []int{7} } func (x *Storage) GetName() string { @@ -264,7 +572,7 @@ type Credential struct { func (x *Credential) Reset() { *x = Credential{} - mi := &file_chorus_chorus_proto_msgTypes[3] + mi := &file_chorus_chorus_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -276,7 +584,7 @@ func (x *Credential) String() string { func (*Credential) ProtoMessage() {} func (x *Credential) ProtoReflect() protoreflect.Message { - mi := &file_chorus_chorus_proto_msgTypes[3] + mi := &file_chorus_chorus_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -289,7 +597,7 @@ func (x *Credential) ProtoReflect() protoreflect.Message { // Deprecated: Use Credential.ProtoReflect.Descriptor instead. func (*Credential) Descriptor() ([]byte, []int) { - return file_chorus_chorus_proto_rawDescGZIP(), []int{3} + return file_chorus_chorus_proto_rawDescGZIP(), []int{8} } func (x *Credential) GetAlias() string { @@ -323,7 +631,7 @@ type GetProxyCredentialsResponse struct { func (x *GetProxyCredentialsResponse) Reset() { *x = GetProxyCredentialsResponse{} - mi := &file_chorus_chorus_proto_msgTypes[4] + mi := &file_chorus_chorus_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -335,7 +643,7 @@ func (x *GetProxyCredentialsResponse) String() string { func (*GetProxyCredentialsResponse) ProtoMessage() {} func (x *GetProxyCredentialsResponse) ProtoReflect() protoreflect.Message { - mi := &file_chorus_chorus_proto_msgTypes[4] + mi := &file_chorus_chorus_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -348,7 +656,7 @@ func (x *GetProxyCredentialsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetProxyCredentialsResponse.ProtoReflect.Descriptor instead. func (*GetProxyCredentialsResponse) Descriptor() ([]byte, []int) { - return file_chorus_chorus_proto_rawDescGZIP(), []int{4} + return file_chorus_chorus_proto_rawDescGZIP(), []int{9} } func (x *GetProxyCredentialsResponse) GetAddress() string { @@ -377,7 +685,7 @@ type SetUserCredentialsRequest struct { func (x *SetUserCredentialsRequest) Reset() { *x = SetUserCredentialsRequest{} - mi := &file_chorus_chorus_proto_msgTypes[5] + mi := &file_chorus_chorus_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -389,7 +697,7 @@ func (x *SetUserCredentialsRequest) String() string { func (*SetUserCredentialsRequest) ProtoMessage() {} func (x *SetUserCredentialsRequest) ProtoReflect() protoreflect.Message { - mi := &file_chorus_chorus_proto_msgTypes[5] + mi := &file_chorus_chorus_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -402,7 +710,7 @@ func (x *SetUserCredentialsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SetUserCredentialsRequest.ProtoReflect.Descriptor instead. func (*SetUserCredentialsRequest) Descriptor() ([]byte, []int) { - return file_chorus_chorus_proto_rawDescGZIP(), []int{5} + return file_chorus_chorus_proto_rawDescGZIP(), []int{10} } func (x *SetUserCredentialsRequest) GetStorage() string { @@ -443,7 +751,7 @@ type S3Credential struct { func (x *S3Credential) Reset() { *x = S3Credential{} - mi := &file_chorus_chorus_proto_msgTypes[6] + mi := &file_chorus_chorus_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -455,7 +763,7 @@ func (x *S3Credential) String() string { func (*S3Credential) ProtoMessage() {} func (x *S3Credential) ProtoReflect() protoreflect.Message { - mi := &file_chorus_chorus_proto_msgTypes[6] + mi := &file_chorus_chorus_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -468,7 +776,7 @@ func (x *S3Credential) ProtoReflect() protoreflect.Message { // Deprecated: Use S3Credential.ProtoReflect.Descriptor instead. func (*S3Credential) Descriptor() ([]byte, []int) { - return file_chorus_chorus_proto_rawDescGZIP(), []int{6} + return file_chorus_chorus_proto_rawDescGZIP(), []int{11} } func (x *S3Credential) GetAccessKey() string { @@ -497,7 +805,7 @@ type SwiftCredential struct { func (x *SwiftCredential) Reset() { *x = SwiftCredential{} - mi := &file_chorus_chorus_proto_msgTypes[7] + mi := &file_chorus_chorus_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -509,7 +817,7 @@ func (x *SwiftCredential) String() string { func (*SwiftCredential) ProtoMessage() {} func (x *SwiftCredential) ProtoReflect() protoreflect.Message { - mi := &file_chorus_chorus_proto_msgTypes[7] + mi := &file_chorus_chorus_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -522,7 +830,7 @@ func (x *SwiftCredential) ProtoReflect() protoreflect.Message { // Deprecated: Use SwiftCredential.ProtoReflect.Descriptor instead. func (*SwiftCredential) Descriptor() ([]byte, []int) { - return file_chorus_chorus_proto_rawDescGZIP(), []int{7} + return file_chorus_chorus_proto_rawDescGZIP(), []int{12} } func (x *SwiftCredential) GetUsername() string { @@ -563,7 +871,29 @@ const file_chorus_chorus_proto_rawDesc = "" + "\x06commit\x18\x02 \x01(\tR\x06commit\x12\x12\n" + "\x04date\x18\x03 \x01(\tR\x04date\"B\n" + "\x13GetStoragesResponse\x12+\n" + - "\bstorages\x18\x01 \x03(\v2\x0f.chorus.StorageR\bstorages\"\xd3\x01\n" + + "\bstorages\x18\x01 \x03(\v2\x0f.chorus.StorageR\bstorages\"I\n" + + "\x19ListBrowserBucketsRequest\x12\x18\n" + + "\astorage\x18\x01 \x01(\tR\astorage\x12\x12\n" + + "\x04user\x18\x02 \x01(\tR\x04user\"6\n" + + "\x1aListBrowserBucketsResponse\x12\x18\n" + + "\abuckets\x18\x01 \x03(\tR\abuckets\"\xa5\x01\n" + + "\x19ListBrowserObjectsRequest\x12\x18\n" + + "\astorage\x18\x01 \x01(\tR\astorage\x12\x12\n" + + "\x04user\x18\x02 \x01(\tR\x04user\x12\x16\n" + + "\x06bucket\x18\x03 \x01(\tR\x06bucket\x12\x16\n" + + "\x06prefix\x18\x04 \x01(\tR\x06prefix\x12\x14\n" + + "\x05after\x18\x05 \x01(\tR\x05after\x12\x14\n" + + "\x05limit\x18\x06 \x01(\rR\x05limit\"\xa8\x01\n" + + "\rBrowserObject\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x12\n" + + "\x04size\x18\x02 \x01(\x04R\x04size\x12?\n" + + "\rlast_modified\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\flastModified\x120\n" + + "\x14source_last_modified\x18\x04 \x01(\tR\x12sourceLastModified\"\x87\x01\n" + + "\x1aListBrowserObjectsResponse\x12/\n" + + "\aobjects\x18\x01 \x03(\v2\x15.chorus.BrowserObjectR\aobjects\x12\x1d\n" + + "\n" + + "next_after\x18\x02 \x01(\tR\tnextAfter\x12\x19\n" + + "\bhas_more\x18\x03 \x01(\bR\ahasMore\"\xd3\x01\n" + "\aStorage\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n" + "\ais_main\x18\x02 \x01(\bR\x06isMain\x12\x18\n" + @@ -603,10 +933,12 @@ const file_chorus_chorus_proto_rawDesc = "" + "\vdomain_name\x18\x03 \x01(\tR\n" + "domainName\x12\x1f\n" + "\vtenant_name\x18\x04 \x01(\tR\n" + - "tenantName2\xb9\x02\n" + + "tenantName2\xf3\x03\n" + "\x06Chorus\x12F\n" + "\rGetAppVersion\x12\x16.google.protobuf.Empty\x1a\x1d.chorus.GetAppVersionResponse\x12B\n" + - "\vGetStorages\x12\x16.google.protobuf.Empty\x1a\x1b.chorus.GetStoragesResponse\x12R\n" + + "\vGetStorages\x12\x16.google.protobuf.Empty\x1a\x1b.chorus.GetStoragesResponse\x12[\n" + + "\x12ListBrowserBuckets\x12!.chorus.ListBrowserBucketsRequest\x1a\".chorus.ListBrowserBucketsResponse\x12[\n" + + "\x12ListBrowserObjects\x12!.chorus.ListBrowserObjectsRequest\x1a\".chorus.ListBrowserObjectsResponse\x12R\n" + "\x13GetProxyCredentials\x12\x16.google.protobuf.Empty\x1a#.chorus.GetProxyCredentialsResponse\x12O\n" + "\x12SetUserCredentials\x12!.chorus.SetUserCredentialsRequest\x1a\x16.google.protobuf.EmptyB)Z'github.com/clyso/chorus/proto/chorus;pbb\x06proto3" @@ -623,39 +955,51 @@ func file_chorus_chorus_proto_rawDescGZIP() []byte { } var file_chorus_chorus_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_chorus_chorus_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_chorus_chorus_proto_msgTypes = make([]protoimpl.MessageInfo, 13) var file_chorus_chorus_proto_goTypes = []any{ (Storage_Type)(0), // 0: chorus.Storage.Type (*GetAppVersionResponse)(nil), // 1: chorus.GetAppVersionResponse (*GetStoragesResponse)(nil), // 2: chorus.GetStoragesResponse - (*Storage)(nil), // 3: chorus.Storage - (*Credential)(nil), // 4: chorus.Credential - (*GetProxyCredentialsResponse)(nil), // 5: chorus.GetProxyCredentialsResponse - (*SetUserCredentialsRequest)(nil), // 6: chorus.SetUserCredentialsRequest - (*S3Credential)(nil), // 7: chorus.S3Credential - (*SwiftCredential)(nil), // 8: chorus.SwiftCredential - (*emptypb.Empty)(nil), // 9: google.protobuf.Empty + (*ListBrowserBucketsRequest)(nil), // 3: chorus.ListBrowserBucketsRequest + (*ListBrowserBucketsResponse)(nil), // 4: chorus.ListBrowserBucketsResponse + (*ListBrowserObjectsRequest)(nil), // 5: chorus.ListBrowserObjectsRequest + (*BrowserObject)(nil), // 6: chorus.BrowserObject + (*ListBrowserObjectsResponse)(nil), // 7: chorus.ListBrowserObjectsResponse + (*Storage)(nil), // 8: chorus.Storage + (*Credential)(nil), // 9: chorus.Credential + (*GetProxyCredentialsResponse)(nil), // 10: chorus.GetProxyCredentialsResponse + (*SetUserCredentialsRequest)(nil), // 11: chorus.SetUserCredentialsRequest + (*S3Credential)(nil), // 12: chorus.S3Credential + (*SwiftCredential)(nil), // 13: chorus.SwiftCredential + (*timestamppb.Timestamp)(nil), // 14: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 15: google.protobuf.Empty } var file_chorus_chorus_proto_depIdxs = []int32{ - 3, // 0: chorus.GetStoragesResponse.storages:type_name -> chorus.Storage - 0, // 1: chorus.Storage.provider:type_name -> chorus.Storage.Type - 4, // 2: chorus.Storage.credentials:type_name -> chorus.Credential - 4, // 3: chorus.GetProxyCredentialsResponse.credentials:type_name -> chorus.Credential - 7, // 4: chorus.SetUserCredentialsRequest.s3_cred:type_name -> chorus.S3Credential - 8, // 5: chorus.SetUserCredentialsRequest.swift_cred:type_name -> chorus.SwiftCredential - 9, // 6: chorus.Chorus.GetAppVersion:input_type -> google.protobuf.Empty - 9, // 7: chorus.Chorus.GetStorages:input_type -> google.protobuf.Empty - 9, // 8: chorus.Chorus.GetProxyCredentials:input_type -> google.protobuf.Empty - 6, // 9: chorus.Chorus.SetUserCredentials:input_type -> chorus.SetUserCredentialsRequest - 1, // 10: chorus.Chorus.GetAppVersion:output_type -> chorus.GetAppVersionResponse - 2, // 11: chorus.Chorus.GetStorages:output_type -> chorus.GetStoragesResponse - 5, // 12: chorus.Chorus.GetProxyCredentials:output_type -> chorus.GetProxyCredentialsResponse - 9, // 13: chorus.Chorus.SetUserCredentials:output_type -> google.protobuf.Empty - 10, // [10:14] is the sub-list for method output_type - 6, // [6:10] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 8, // 0: chorus.GetStoragesResponse.storages:type_name -> chorus.Storage + 14, // 1: chorus.BrowserObject.last_modified:type_name -> google.protobuf.Timestamp + 6, // 2: chorus.ListBrowserObjectsResponse.objects:type_name -> chorus.BrowserObject + 0, // 3: chorus.Storage.provider:type_name -> chorus.Storage.Type + 9, // 4: chorus.Storage.credentials:type_name -> chorus.Credential + 9, // 5: chorus.GetProxyCredentialsResponse.credentials:type_name -> chorus.Credential + 12, // 6: chorus.SetUserCredentialsRequest.s3_cred:type_name -> chorus.S3Credential + 13, // 7: chorus.SetUserCredentialsRequest.swift_cred:type_name -> chorus.SwiftCredential + 15, // 8: chorus.Chorus.GetAppVersion:input_type -> google.protobuf.Empty + 15, // 9: chorus.Chorus.GetStorages:input_type -> google.protobuf.Empty + 3, // 10: chorus.Chorus.ListBrowserBuckets:input_type -> chorus.ListBrowserBucketsRequest + 5, // 11: chorus.Chorus.ListBrowserObjects:input_type -> chorus.ListBrowserObjectsRequest + 15, // 12: chorus.Chorus.GetProxyCredentials:input_type -> google.protobuf.Empty + 11, // 13: chorus.Chorus.SetUserCredentials:input_type -> chorus.SetUserCredentialsRequest + 1, // 14: chorus.Chorus.GetAppVersion:output_type -> chorus.GetAppVersionResponse + 2, // 15: chorus.Chorus.GetStorages:output_type -> chorus.GetStoragesResponse + 4, // 16: chorus.Chorus.ListBrowserBuckets:output_type -> chorus.ListBrowserBucketsResponse + 7, // 17: chorus.Chorus.ListBrowserObjects:output_type -> chorus.ListBrowserObjectsResponse + 10, // 18: chorus.Chorus.GetProxyCredentials:output_type -> chorus.GetProxyCredentialsResponse + 15, // 19: chorus.Chorus.SetUserCredentials:output_type -> google.protobuf.Empty + 14, // [14:20] is the sub-list for method output_type + 8, // [8:14] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name } func init() { file_chorus_chorus_proto_init() } @@ -663,14 +1007,14 @@ func file_chorus_chorus_proto_init() { if File_chorus_chorus_proto != nil { return } - file_chorus_chorus_proto_msgTypes[5].OneofWrappers = []any{} + file_chorus_chorus_proto_msgTypes[10].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_chorus_chorus_proto_rawDesc), len(file_chorus_chorus_proto_rawDesc)), NumEnums: 1, - NumMessages: 8, + NumMessages: 13, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/gen/go/chorus/chorus.pb.gw.go b/proto/gen/go/chorus/chorus.pb.gw.go index 90f215db..21f0b36c 100644 --- a/proto/gen/go/chorus/chorus.pb.gw.go +++ b/proto/gen/go/chorus/chorus.pb.gw.go @@ -78,6 +78,76 @@ func local_request_Chorus_GetStorages_0(ctx context.Context, marshaler runtime.M return msg, metadata, err } +var filter_Chorus_ListBrowserBuckets_0 = &utilities.DoubleArray{Encoding: map[string]int{}, Base: []int(nil), Check: []int(nil)} + +func request_Chorus_ListBrowserBuckets_0(ctx context.Context, marshaler runtime.Marshaler, client ChorusClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var ( + protoReq ListBrowserBucketsRequest + metadata runtime.ServerMetadata + ) + if err := req.ParseForm(); err != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + if err := runtime.PopulateQueryParameters(&protoReq, req.Form, filter_Chorus_ListBrowserBuckets_0); err != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + } + msg, err := client.ListBrowserBuckets(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err +} + +func local_request_Chorus_ListBrowserBuckets_0(ctx context.Context, marshaler runtime.Marshaler, server ChorusServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var ( + protoReq ListBrowserBucketsRequest + metadata runtime.ServerMetadata + ) + if err := req.ParseForm(); err != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + if err := runtime.PopulateQueryParameters(&protoReq, req.Form, filter_Chorus_ListBrowserBuckets_0); err != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + msg, err := server.ListBrowserBuckets(ctx, &protoReq) + return msg, metadata, err +} + +var filter_Chorus_ListBrowserObjects_0 = &utilities.DoubleArray{Encoding: map[string]int{}, Base: []int(nil), Check: []int(nil)} + +func request_Chorus_ListBrowserObjects_0(ctx context.Context, marshaler runtime.Marshaler, client ChorusClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var ( + protoReq ListBrowserObjectsRequest + metadata runtime.ServerMetadata + ) + if err := req.ParseForm(); err != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + if err := runtime.PopulateQueryParameters(&protoReq, req.Form, filter_Chorus_ListBrowserObjects_0); err != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + } + msg, err := client.ListBrowserObjects(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err +} + +func local_request_Chorus_ListBrowserObjects_0(ctx context.Context, marshaler runtime.Marshaler, server ChorusServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var ( + protoReq ListBrowserObjectsRequest + metadata runtime.ServerMetadata + ) + if err := req.ParseForm(); err != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + if err := runtime.PopulateQueryParameters(&protoReq, req.Form, filter_Chorus_ListBrowserObjects_0); err != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + msg, err := server.ListBrowserObjects(ctx, &protoReq) + return msg, metadata, err +} + func request_Chorus_GetProxyCredentials_0(ctx context.Context, marshaler runtime.Marshaler, client ChorusClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var ( protoReq emptypb.Empty @@ -172,6 +242,46 @@ func RegisterChorusHandlerServer(ctx context.Context, mux *runtime.ServeMux, ser } forward_Chorus_GetStorages_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) + mux.Handle(http.MethodGet, pattern_Chorus_ListBrowserBuckets_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + var stream runtime.ServerTransportStream + ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/chorus.Chorus/ListBrowserBuckets", runtime.WithHTTPPathPattern("/browser/buckets")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := local_request_Chorus_ListBrowserBuckets_0(annotatedContext, inboundMarshaler, server, req, pathParams) + md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) + annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) + if err != nil { + runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) + return + } + forward_Chorus_ListBrowserBuckets_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + }) + mux.Handle(http.MethodGet, pattern_Chorus_ListBrowserObjects_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + var stream runtime.ServerTransportStream + ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/chorus.Chorus/ListBrowserObjects", runtime.WithHTTPPathPattern("/browser/objects")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := local_request_Chorus_ListBrowserObjects_0(annotatedContext, inboundMarshaler, server, req, pathParams) + md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) + annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) + if err != nil { + runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) + return + } + forward_Chorus_ListBrowserObjects_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + }) mux.Handle(http.MethodGet, pattern_Chorus_GetProxyCredentials_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() @@ -286,6 +396,40 @@ func RegisterChorusHandlerClient(ctx context.Context, mux *runtime.ServeMux, cli } forward_Chorus_GetStorages_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) + mux.Handle(http.MethodGet, pattern_Chorus_ListBrowserBuckets_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/chorus.Chorus/ListBrowserBuckets", runtime.WithHTTPPathPattern("/browser/buckets")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_Chorus_ListBrowserBuckets_0(annotatedContext, inboundMarshaler, client, req, pathParams) + annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) + if err != nil { + runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) + return + } + forward_Chorus_ListBrowserBuckets_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + }) + mux.Handle(http.MethodGet, pattern_Chorus_ListBrowserObjects_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/chorus.Chorus/ListBrowserObjects", runtime.WithHTTPPathPattern("/browser/objects")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_Chorus_ListBrowserObjects_0(annotatedContext, inboundMarshaler, client, req, pathParams) + annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) + if err != nil { + runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) + return + } + forward_Chorus_ListBrowserObjects_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + }) mux.Handle(http.MethodGet, pattern_Chorus_GetProxyCredentials_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { ctx, cancel := context.WithCancel(req.Context()) defer cancel() @@ -326,6 +470,8 @@ func RegisterChorusHandlerClient(ctx context.Context, mux *runtime.ServeMux, cli var ( pattern_Chorus_GetAppVersion_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0}, []string{"version"}, "")) pattern_Chorus_GetStorages_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0}, []string{"storage"}, "")) + pattern_Chorus_ListBrowserBuckets_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1}, []string{"browser", "buckets"}, "")) + pattern_Chorus_ListBrowserObjects_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1}, []string{"browser", "objects"}, "")) pattern_Chorus_GetProxyCredentials_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0}, []string{"proxy"}, "")) pattern_Chorus_SetUserCredentials_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0}, []string{"credentials"}, "")) ) @@ -333,6 +479,8 @@ var ( var ( forward_Chorus_GetAppVersion_0 = runtime.ForwardResponseMessage forward_Chorus_GetStorages_0 = runtime.ForwardResponseMessage + forward_Chorus_ListBrowserBuckets_0 = runtime.ForwardResponseMessage + forward_Chorus_ListBrowserObjects_0 = runtime.ForwardResponseMessage forward_Chorus_GetProxyCredentials_0 = runtime.ForwardResponseMessage forward_Chorus_SetUserCredentials_0 = runtime.ForwardResponseMessage ) diff --git a/proto/gen/go/chorus/chorus_grpc.pb.go b/proto/gen/go/chorus/chorus_grpc.pb.go index 07546a5d..7c89bf2f 100644 --- a/proto/gen/go/chorus/chorus_grpc.pb.go +++ b/proto/gen/go/chorus/chorus_grpc.pb.go @@ -22,6 +22,8 @@ const _ = grpc.SupportPackageIsVersion9 const ( Chorus_GetAppVersion_FullMethodName = "/chorus.Chorus/GetAppVersion" Chorus_GetStorages_FullMethodName = "/chorus.Chorus/GetStorages" + Chorus_ListBrowserBuckets_FullMethodName = "/chorus.Chorus/ListBrowserBuckets" + Chorus_ListBrowserObjects_FullMethodName = "/chorus.Chorus/ListBrowserObjects" Chorus_GetProxyCredentials_FullMethodName = "/chorus.Chorus/GetProxyCredentials" Chorus_SetUserCredentials_FullMethodName = "/chorus.Chorus/SetUserCredentials" ) @@ -34,6 +36,10 @@ type ChorusClient interface { GetAppVersion(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*GetAppVersionResponse, error) // Lists configured storages with users GetStorages(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*GetStoragesResponse, error) + // Lists buckets for an object browser. + ListBrowserBuckets(ctx context.Context, in *ListBrowserBucketsRequest, opts ...grpc.CallOption) (*ListBrowserBucketsResponse, error) + // Lists a page of objects and their original migration timestamps. + ListBrowserObjects(ctx context.Context, in *ListBrowserObjectsRequest, opts ...grpc.CallOption) (*ListBrowserObjectsResponse, error) // Returns connection details for proxy s3 endpoint GetProxyCredentials(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*GetProxyCredentialsResponse, error) // Sets user credentials for a given storage. @@ -68,6 +74,26 @@ func (c *chorusClient) GetStorages(ctx context.Context, in *emptypb.Empty, opts return out, nil } +func (c *chorusClient) ListBrowserBuckets(ctx context.Context, in *ListBrowserBucketsRequest, opts ...grpc.CallOption) (*ListBrowserBucketsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListBrowserBucketsResponse) + err := c.cc.Invoke(ctx, Chorus_ListBrowserBuckets_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *chorusClient) ListBrowserObjects(ctx context.Context, in *ListBrowserObjectsRequest, opts ...grpc.CallOption) (*ListBrowserObjectsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListBrowserObjectsResponse) + err := c.cc.Invoke(ctx, Chorus_ListBrowserObjects_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *chorusClient) GetProxyCredentials(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*GetProxyCredentialsResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetProxyCredentialsResponse) @@ -96,6 +122,10 @@ type ChorusServer interface { GetAppVersion(context.Context, *emptypb.Empty) (*GetAppVersionResponse, error) // Lists configured storages with users GetStorages(context.Context, *emptypb.Empty) (*GetStoragesResponse, error) + // Lists buckets for an object browser. + ListBrowserBuckets(context.Context, *ListBrowserBucketsRequest) (*ListBrowserBucketsResponse, error) + // Lists a page of objects and their original migration timestamps. + ListBrowserObjects(context.Context, *ListBrowserObjectsRequest) (*ListBrowserObjectsResponse, error) // Returns connection details for proxy s3 endpoint GetProxyCredentials(context.Context, *emptypb.Empty) (*GetProxyCredentialsResponse, error) // Sets user credentials for a given storage. @@ -115,6 +145,12 @@ func (UnimplementedChorusServer) GetAppVersion(context.Context, *emptypb.Empty) func (UnimplementedChorusServer) GetStorages(context.Context, *emptypb.Empty) (*GetStoragesResponse, error) { return nil, status.Error(codes.Unimplemented, "method GetStorages not implemented") } +func (UnimplementedChorusServer) ListBrowserBuckets(context.Context, *ListBrowserBucketsRequest) (*ListBrowserBucketsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListBrowserBuckets not implemented") +} +func (UnimplementedChorusServer) ListBrowserObjects(context.Context, *ListBrowserObjectsRequest) (*ListBrowserObjectsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListBrowserObjects not implemented") +} func (UnimplementedChorusServer) GetProxyCredentials(context.Context, *emptypb.Empty) (*GetProxyCredentialsResponse, error) { return nil, status.Error(codes.Unimplemented, "method GetProxyCredentials not implemented") } @@ -177,6 +213,42 @@ func _Chorus_GetStorages_Handler(srv interface{}, ctx context.Context, dec func( return interceptor(ctx, in, info, handler) } +func _Chorus_ListBrowserBuckets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListBrowserBucketsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ChorusServer).ListBrowserBuckets(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Chorus_ListBrowserBuckets_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ChorusServer).ListBrowserBuckets(ctx, req.(*ListBrowserBucketsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Chorus_ListBrowserObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListBrowserObjectsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ChorusServer).ListBrowserObjects(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Chorus_ListBrowserObjects_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ChorusServer).ListBrowserObjects(ctx, req.(*ListBrowserObjectsRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Chorus_GetProxyCredentials_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(emptypb.Empty) if err := dec(in); err != nil { @@ -228,6 +300,14 @@ var Chorus_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetStorages", Handler: _Chorus_GetStorages_Handler, }, + { + MethodName: "ListBrowserBuckets", + Handler: _Chorus_ListBrowserBuckets_Handler, + }, + { + MethodName: "ListBrowserObjects", + Handler: _Chorus_ListBrowserObjects_Handler, + }, { MethodName: "GetProxyCredentials", Handler: _Chorus_GetProxyCredentials_Handler, diff --git a/proto/gen/openapi/chorus/chorus.swagger.json b/proto/gen/openapi/chorus/chorus.swagger.json index 442616fe..12c1a5eb 100644 --- a/proto/gen/openapi/chorus/chorus.swagger.json +++ b/proto/gen/openapi/chorus/chorus.swagger.json @@ -27,6 +27,105 @@ "application/json" ], "paths": { + "/browser/buckets": { + "get": { + "summary": "Lists buckets for an object browser.", + "operationId": "Chorus_ListBrowserBuckets", + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/chorusListBrowserBucketsResponse" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/googleRpcStatus" + } + } + }, + "parameters": [ + { + "name": "storage", + "in": "query", + "required": false, + "type": "string" + }, + { + "name": "user", + "in": "query", + "required": false, + "type": "string" + } + ], + "tags": [ + "Chorus" + ] + } + }, + "/browser/objects": { + "get": { + "summary": "Lists a page of objects and their original migration timestamps.", + "operationId": "Chorus_ListBrowserObjects", + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/chorusListBrowserObjectsResponse" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/googleRpcStatus" + } + } + }, + "parameters": [ + { + "name": "storage", + "in": "query", + "required": false, + "type": "string" + }, + { + "name": "user", + "in": "query", + "required": false, + "type": "string" + }, + { + "name": "bucket", + "in": "query", + "required": false, + "type": "string" + }, + { + "name": "prefix", + "in": "query", + "required": false, + "type": "string" + }, + { + "name": "after", + "in": "query", + "required": false, + "type": "string" + }, + { + "name": "limit", + "in": "query", + "required": false, + "type": "integer", + "format": "int64" + } + ], + "tags": [ + "Chorus" + ] + } + }, "/credentials": { "post": { "summary": "Sets user credentials for a given storage.", @@ -1117,6 +1216,25 @@ } } }, + "chorusBrowserObject": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "size": { + "type": "string", + "format": "uint64" + }, + "lastModified": { + "type": "string", + "format": "date-time" + }, + "sourceLastModified": { + "type": "string" + } + } + }, "chorusBucketRouting": { "type": "object", "properties": { @@ -1345,6 +1463,35 @@ } } }, + "chorusListBrowserBucketsResponse": { + "type": "object", + "properties": { + "buckets": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "chorusListBrowserObjectsResponse": { + "type": "object", + "properties": { + "objects": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/chorusBrowserObject" + } + }, + "nextAfter": { + "type": "string" + }, + "hasMore": { + "type": "boolean" + } + } + }, "chorusListDiffChecksResponse": { "type": "object", "properties": { diff --git a/proto/http.yaml b/proto/http.yaml index 7df7a387..9a3a9af5 100644 --- a/proto/http.yaml +++ b/proto/http.yaml @@ -29,6 +29,10 @@ http: get: /version - selector: chorus.Chorus.GetStorages get: /storage + - selector: chorus.Chorus.ListBrowserBuckets + get: /browser/buckets + - selector: chorus.Chorus.ListBrowserObjects + get: /browser/objects - selector: chorus.Chorus.GetProxyCredentials get: /proxy - selector: chorus.Chorus.SetUserCredentials diff --git a/service/worker/server.go b/service/worker/server.go index ed2f4723..07d93eba 100644 --- a/service/worker/server.go +++ b/service/worker/server.go @@ -264,7 +264,7 @@ func Start(ctx context.Context, app dom.AppInfo, conf *Config) error { } if conf.Api.Enabled { - chorusHandler := api.ChorusHandlers(credsSvc, rpc.NewProxyClient(appRedis), &app) + chorusHandler := api.ChorusHandlers(credsSvc, clientRegistry, rpc.NewProxyClient(appRedis), &app) diffHandler := api.DiffHandlers(credsSvc, checkSvc) var webhookConf *api.WebhookConfig if conf.Api.Webhook.Enabled { diff --git a/tools/chorctl/internal/api/rest.go b/tools/chorctl/internal/api/rest.go index de965ed2..f2801e17 100644 --- a/tools/chorctl/internal/api/rest.go +++ b/tools/chorctl/internal/api/rest.go @@ -19,7 +19,9 @@ package api import ( "bytes" "context" + "encoding/json" "errors" + "fmt" "io" "net/http" "net/url" @@ -30,7 +32,6 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/reflect/protoreflect" pb "github.com/clyso/chorus/proto/gen/go/chorus" ) @@ -58,6 +59,8 @@ var restRoutes = map[string]route{ // Chorus service: pb.Chorus_GetAppVersion_FullMethodName: {http.MethodGet, "/version"}, pb.Chorus_GetStorages_FullMethodName: {http.MethodGet, "/storage"}, + pb.Chorus_ListBrowserBuckets_FullMethodName: {http.MethodGet, "/browser/buckets"}, + pb.Chorus_ListBrowserObjects_FullMethodName: {http.MethodGet, "/browser/objects"}, pb.Chorus_GetProxyCredentials_FullMethodName: {http.MethodGet, "/proxy"}, pb.Chorus_SetUserCredentials_FullMethodName: {http.MethodPost, "/credentials"}, // Policy service - replication: @@ -81,10 +84,8 @@ var restRoutes = map[string]route{ pb.Policy_TestProxy_FullMethodName: {http.MethodPost, "/test-proxy"}, } -// Conn calls the chorus management API over HTTP/JSON via the grpc-gateway -// REST endpoints. It implements grpc.ClientConnInterface so that the -// generated gRPC client stubs (pb.NewChorusClient, pb.NewPolicyClient, ...) -// can be used as-is. +// Conn calls the chorus management API over HTTP/JSON via the grpc-gateway REST endpoints. +// It implements grpc.ClientConnInterface so generated gRPC client stubs can be used as-is. type Conn struct { httpClient *http.Client baseURL *url.URL @@ -102,14 +103,14 @@ func (c *Conn) Invoke(ctx context.Context, method string, args, reply any, _ ... return status.Errorf(codes.Internal, "request for %s is not a proto message", method) } + endpoint := c.baseURL.JoinPath(rt.path) var body io.Reader if rt.method == http.MethodGet { - // All GET routes take google.protobuf.Empty. Fail loudly if a future - // GET route carries request fields: they would require query - // parameter encoding, which is not implemented. - if !isEmptyMessage(reqMsg) { - return status.Errorf(codes.Internal, "GET route for %s cannot carry request fields", method) + query, err := encodeQuery(reqMsg) + if err != nil { + return status.Errorf(codes.Internal, "failed to encode query for %s: %v", method, err) } + endpoint.RawQuery = query.Encode() } else { data, err := protojson.Marshal(reqMsg) if err != nil { @@ -118,7 +119,7 @@ func (c *Conn) Invoke(ctx context.Context, method string, args, reply any, _ ... body = bytes.NewReader(data) } - httpReq, err := http.NewRequestWithContext(ctx, rt.method, c.baseURL.JoinPath(rt.path).String(), body) + httpReq, err := http.NewRequestWithContext(ctx, rt.method, endpoint.String(), body) if err != nil { return status.Errorf(codes.Internal, "failed to build request for %s: %v", method, err) } @@ -150,6 +151,39 @@ func (c *Conn) Invoke(ctx context.Context, method string, args, reply any, _ ... return nil } +// encodeQuery converts protobuf JSON fields to grpc-gateway GET query parameters. +// Scalar fields use their JSON names; repeated scalar fields become repeated query keys. +func encodeQuery(msg proto.Message) (url.Values, error) { + data, err := protojson.Marshal(msg) + if err != nil { + return nil, err + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return nil, err + } + + query := make(url.Values, len(fields)) + for name, raw := range fields { + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return nil, fmt.Errorf("decode field %q: %w", name, err) + } + if values, ok := value.([]any); ok { + for _, item := range values { + if item != nil { + query.Add(name, fmt.Sprint(item)) + } + } + continue + } + if value != nil { + query.Set(name, fmt.Sprint(value)) + } + } + return query, nil +} + func (c *Conn) NewStream(_ context.Context, _ *grpc.StreamDesc, method string, _ ...grpc.CallOption) (grpc.ClientStream, error) { return nil, status.Errorf(codes.Unimplemented, "streaming method %s is not supported by the REST client", method) } @@ -159,18 +193,7 @@ func (c *Conn) Close() error { return nil } -// isEmptyMessage reports whether the message has no populated fields. -func isEmptyMessage(m proto.Message) bool { - empty := true - m.ProtoReflect().Range(func(protoreflect.FieldDescriptor, protoreflect.Value) bool { - empty = false - return false - }) - return empty -} - -// transportErrCode maps HTTP client errors to gRPC codes preserving -// context cancellation semantics. +// transportErrCode maps HTTP client errors to gRPC codes preserving context cancellation semantics. func transportErrCode(err error) codes.Code { switch { case errors.Is(err, context.Canceled): @@ -182,16 +205,10 @@ func transportErrCode(err error) codes.Code { } } -// errorFromResponse converts a grpc-gateway error response into a gRPC status -// error. The gateway serializes errors as google.rpc.Status JSON -// ({"code":..,"message":..,"details":[..]}), which allows PrintGrpcError to -// present them the same way as before. +// errorFromResponse converts a grpc-gateway error response into a gRPC status error. func errorFromResponse(httpStatus int, body []byte) error { var st spb.Status err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(body, &st) - // Trust the body only if the code is a valid gRPC error code. This filters - // out non grpc-gateway JSON responses, e.g. {"code":401,...} produced by an - // auth proxy in front of the API. if err == nil && st.GetCode() > int32(codes.OK) && st.GetCode() <= int32(codes.Unauthenticated) { return status.ErrorProto(&st) } @@ -202,8 +219,6 @@ func errorFromResponse(httpStatus int, body []byte) error { return status.Errorf(codeFromHTTPStatus(httpStatus), "HTTP %d: %s", httpStatus, bytes.TrimSpace(body)) } -// codeFromHTTPStatus is a fallback for non grpc-gateway error responses, -// e.g. produced by a load balancer in front of the API. func codeFromHTTPStatus(httpStatus int) codes.Code { switch httpStatus { case http.StatusBadRequest: diff --git a/tools/chorctl/internal/api/rest_test.go b/tools/chorctl/internal/api/rest_test.go index 5aa61f91..297e82a4 100644 --- a/tools/chorctl/internal/api/rest_test.go +++ b/tools/chorctl/internal/api/rest_test.go @@ -37,18 +37,11 @@ import ( pb "github.com/clyso/chorus/proto/gen/go/chorus" ) -// apiServices lists the gRPC services exposed by the chorus management REST -// API and mapped in restRoutes. The Webhook service is excluded on purpose: -// its routes use path parameters and are not called by chorctl. var apiServices = []string{"chorus.Chorus", "chorus.Policy", "chorus.Diff"} -// newTestGateway starts an in-process HTTP server backed by the generated -// grpc-gateway registration code — the same code the worker serves REST with. -// It returns a Conn pointed at that server. func newTestGateway(t *testing.T, chorusSrv pb.ChorusServer) *Conn { t.Helper() r := require.New(t) - // Same marshaler options as the worker gateway, see pkg/api/grpc_http_gateway.go. mux := runtime.NewServeMux( runtime.WithMarshalerOption(runtime.MIMEWildcard, &runtime.JSONPb{ MarshalOptions: protojson.MarshalOptions{EmitUnpopulated: true}, @@ -87,15 +80,6 @@ func newMessage(t *testing.T, md protoreflect.MessageDescriptor) proto.Message { return mt.New().Interface() } -// TestRestRoutes_MatchGeneratedGateway verifies that every entry in restRoutes -// (HTTP verb + path) is routed by the generated grpc-gateway code to the -// expected gRPC method. The gateway is backed by Unimplemented* stubs, so a -// correctly routed request returns exactly -// "codes.Unimplemented: method not implemented", while a stale verb or -// path in restRoutes fails with the gateway routing error (NotFound). -// -// Together with the CI check that regenerates proto code and fails on -// uncommitted changes, this pins restRoutes to proto/http.yaml. func TestRestRoutes_MatchGeneratedGateway(t *testing.T) { conn := newTestGateway(t, pb.UnimplementedChorusServer{}) for fullMethod := range restRoutes { @@ -113,9 +97,6 @@ func TestRestRoutes_MatchGeneratedGateway(t *testing.T) { } } -// TestRestRoutes_CoverAllAPIMethods verifies that every unary method of the -// management API services has a restRoutes entry, so a new RPC cannot be -// silently missing from the chorctl REST client. func TestRestRoutes_CoverAllAPIMethods(t *testing.T) { r := require.New(t) for _, svcName := range apiServices { @@ -126,7 +107,6 @@ func TestRestRoutes_CoverAllAPIMethods(t *testing.T) { for i := 0; i < svc.Methods().Len(); i++ { m := svc.Methods().Get(i) if m.IsStreamingServer() || m.IsStreamingClient() { - // Streaming methods have no REST mapping, see Conn.NewStream. continue } fullMethod := fmt.Sprintf("/%s/%s", svcName, m.Name()) @@ -136,21 +116,63 @@ func TestRestRoutes_CoverAllAPIMethods(t *testing.T) { } } -// TestRestRoutes_GetRequestsAreEmpty verifies that GET routes take -// google.protobuf.Empty: Conn.Invoke does not implement query parameter -// encoding and drops the request message for GET (see Invoke). -func TestRestRoutes_GetRequestsAreEmpty(t *testing.T) { +type browserBucketsStub struct { + pb.UnimplementedChorusServer + request *pb.ListBrowserBucketsRequest +} + +func (s *browserBucketsStub) ListBrowserBuckets(_ context.Context, req *pb.ListBrowserBucketsRequest) (*pb.ListBrowserBucketsResponse, error) { + s.request = req + return &pb.ListBrowserBucketsResponse{Buckets: []string{"bucket-a"}}, nil +} + +func TestConn_InvokeGETEncodesRequestAsQuery(t *testing.T) { + r := require.New(t) + stub := &browserBucketsStub{} + conn := newTestGateway(t, stub) + var reply pb.ListBrowserBucketsResponse + err := conn.Invoke(t.Context(), pb.Chorus_ListBrowserBuckets_FullMethodName, &pb.ListBrowserBucketsRequest{ + Storage: "main storage", + User: "browser-user", + }, &reply) + r.NoError(err) + r.Equal("main storage", stub.request.GetStorage()) + r.Equal("browser-user", stub.request.GetUser()) + r.Equal([]string{"bucket-a"}, reply.GetBuckets()) +} + +func TestRestRoutes_GetRoutesEncodeRequestFields(t *testing.T) { r := require.New(t) for fullMethod, rt := range restRoutes { if rt.method != http.MethodGet { continue } md := methodDescriptor(t, fullMethod) - r.Equal(protoreflect.FullName("google.protobuf.Empty"), md.Input().FullName(), - "GET route %s has request fields which would be silently dropped; use a body method in proto/http.yaml or implement query params", fullMethod) + req := newMessage(t, md.Input()) + _, err := encodeQuery(req) + r.NoError(err, "GET route %s request cannot be encoded as query params", fullMethod) } } +func TestRestRoutes_GetRequestsUseProtoJSONNames(t *testing.T) { + r := require.New(t) + query, err := encodeQuery(&pb.ListBrowserObjectsRequest{ + Storage: "main", + User: "user", + Bucket: "bucket", + Prefix: "a/b", + After: "next", + Limit: 50, + }) + r.NoError(err) + r.Equal("main", query.Get("storage")) + r.Equal("user", query.Get("user")) + r.Equal("bucket", query.Get("bucket")) + r.Equal("a/b", query.Get("prefix")) + r.Equal("next", query.Get("after")) + r.Equal("50", query.Get("limit")) +} + type stubChorusServer struct { pb.UnimplementedChorusServer } @@ -159,8 +181,6 @@ func (stubChorusServer) GetAppVersion(context.Context, *emptypb.Empty) (*pb.GetA return &pb.GetAppVersionResponse{Version: "v1.2.3", Commit: "abc", Date: "2026-01-01"}, nil } -// TestConn_InvokeRoundTrip verifies the success path end to end through the -// generated gateway: request encoding, routing, and response decoding. func TestConn_InvokeRoundTrip(t *testing.T) { r := require.New(t) conn := newTestGateway(t, stubChorusServer{}) diff --git a/ui/src/components/dashboard/DashboardNav/DashboardNav.vue b/ui/src/components/dashboard/DashboardNav/DashboardNav.vue index b3389e44..2d9c89ab 100644 --- a/ui/src/components/dashboard/DashboardNav/DashboardNav.vue +++ b/ui/src/components/dashboard/DashboardNav/DashboardNav.vue @@ -48,6 +48,10 @@ name: RouteName.CHORUS_STORAGES, label: t(RouteName.CHORUS_STORAGES), }, + { + name: RouteName.CHORUS_OBJECT_BROWSER, + label: t(RouteName.CHORUS_OBJECT_BROWSER), + }, { name: RouteName.CHORUS_MONITORING, label: t(RouteName.CHORUS_MONITORING), diff --git a/ui/src/i18n/messages/common.ts b/ui/src/i18n/messages/common.ts index 6021c724..280a8420 100644 --- a/ui/src/i18n/messages/common.ts +++ b/ui/src/i18n/messages/common.ts @@ -29,6 +29,22 @@ export default { direction: 'Direction', user: 'User', buckets: 'Buckets', + objectBrowserTitle: 'Objects', + storage: 'Storage', + bucket: 'Bucket', + selectBucket: 'Select a bucket', + prefix: 'Prefix', + loadObjects: 'Load objects', + objectName: 'Object', + size: 'Size', + destinationModified: 'Destination modified', + originalTimestamp: 'Original timestamp', + loadMore: 'Load more', + noObjectsFound: 'No objects found.', + noS3Storages: 'No configured S3 storage is available.', + timestampNotRecorded: 'Not recorded', + loading: 'Loading...', + loadError: 'Unable to load objects.', }, [I18nLocale.DE]: { mainStorage: 'Hauptspeicher', // Consistent with your suggestion @@ -42,5 +58,21 @@ export default { direction: 'Richtung', user: 'Benutzer', buckets: 'Buckets', + objectBrowserTitle: 'Objekte', + storage: 'Speicher', + bucket: 'Bucket', + selectBucket: 'Bucket auswählen', + prefix: 'Präfix', + loadObjects: 'Objekte laden', + objectName: 'Objekt', + size: 'Größe', + destinationModified: 'Im Ziel geändert', + originalTimestamp: 'Ursprünglicher Zeitstempel', + loadMore: 'Mehr laden', + noObjectsFound: 'Keine Objekte gefunden.', + noS3Storages: 'Kein konfigurierter S3-Speicher verfügbar.', + timestampNotRecorded: 'Nicht aufgezeichnet', + loading: 'Wird geladen...', + loadError: 'Objekte konnten nicht geladen werden.', }, }; diff --git a/ui/src/i18n/messages/routes.ts b/ui/src/i18n/messages/routes.ts index 4e719a67..388852d9 100644 --- a/ui/src/i18n/messages/routes.ts +++ b/ui/src/i18n/messages/routes.ts @@ -26,6 +26,7 @@ const routes: I18nMessages = { [RouteName.LOGIN]: 'Login', [RouteName.CHORUS_HOME]: 'Home', [RouteName.CHORUS_STORAGES]: 'Storages', + [RouteName.CHORUS_OBJECT_BROWSER]: 'Objects', [RouteName.CHORUS_STORAGE_DETAILS]: 'Storage Details', [RouteName.CHORUS_REPLICATION]: 'Replication', [RouteName.CHORUS_ADD_REPLICATION]: 'Add Replication', @@ -41,6 +42,7 @@ const routes: I18nMessages = { [RouteName.LOGIN]: 'Login', [RouteName.CHORUS_HOME]: 'Startseite', [RouteName.CHORUS_STORAGES]: 'Speicher', + [RouteName.CHORUS_OBJECT_BROWSER]: 'Objekte', [RouteName.CHORUS_STORAGE_DETAILS]: 'Speicherdetails', [RouteName.CHORUS_REPLICATION]: 'Replikation', [RouteName.CHORUS_ADD_REPLICATION]: 'Replikation hinzufügen', diff --git a/ui/src/pages/ChorusObjectBrowserPage/ChorusObjectBrowserPage.vue b/ui/src/pages/ChorusObjectBrowserPage/ChorusObjectBrowserPage.vue new file mode 100644 index 00000000..199918a6 --- /dev/null +++ b/ui/src/pages/ChorusObjectBrowserPage/ChorusObjectBrowserPage.vue @@ -0,0 +1,468 @@ + + + + + + + \ No newline at end of file diff --git a/ui/src/router/routes.ts b/ui/src/router/routes.ts index 25eaeb8d..ae3222c0 100644 --- a/ui/src/router/routes.ts +++ b/ui/src/router/routes.ts @@ -29,6 +29,7 @@ import ChorusSetCredentialPage from '@/pages/ChorusSetCredentialPage/ChorusSetCr import ChorusDiffReportsPage from '@/pages/ChorusDiffReportsPage/ChorusDiffReportsPage.vue'; import ChorusAddDiffReportPage from '@/pages/ChorusAddDiffReportPage/ChorusAddDiffReportPage.vue'; import ChorusDiffReportDetailPage from '@/pages/ChorusDiffReportDetailPage/ChorusDiffReportDetailPage.vue'; +import ChorusObjectBrowserPage from '@/pages/ChorusObjectBrowserPage/ChorusObjectBrowserPage.vue'; export const routes: RouteRecordRaw[] = [ { @@ -71,6 +72,11 @@ export const routes: RouteRecordRaw[] = [ name: RouteName.CHORUS_STORAGES, component: ChorusStoragesPage, }, + { + path: 'objects', + name: RouteName.CHORUS_OBJECT_BROWSER, + component: ChorusObjectBrowserPage, + }, { path: 'storages/:storageName', name: RouteName.CHORUS_STORAGE_DETAILS, diff --git a/ui/src/services/ChorusService.ts b/ui/src/services/ChorusService.ts index 9b20bf1c..d155799f 100644 --- a/ui/src/services/ChorusService.ts +++ b/ui/src/services/ChorusService.ts @@ -31,6 +31,8 @@ import type { RoutingPolicyListRequest, RoutingPolicyListResponse, ChorusStorageListResponse, + ChorusBrowserBucketsResponse, + ChorusBrowserObjectsResponse, RoutingPolicyEditRequest, RoutingPolicyAddRequest, UserCredentialSetRequest, @@ -58,6 +60,33 @@ export abstract class ChorusService { return data; } + static async getBrowserBuckets( + params: { storage: string; user: string }, + ): Promise { + const { data } = await apiClient.get( + ApiHelper.getChorusAPIUrl('/browser/buckets'), + { params }, + ); + + return data; + } + + static async getBrowserObjects(params: { + storage: string; + user: string; + bucket: string; + prefix?: string; + after?: string; + limit?: number; + }): Promise { + const { data } = await apiClient.get( + ApiHelper.getChorusAPIUrl('/browser/objects'), + { params }, + ); + + return data; + } + static async compareBucket( payload: ChorusCompareBucketRequest, ): Promise { diff --git a/ui/src/utils/types/chorus.ts b/ui/src/utils/types/chorus.ts index 57d0f716..f9b31d0b 100644 --- a/ui/src/utils/types/chorus.ts +++ b/ui/src/utils/types/chorus.ts @@ -85,6 +85,23 @@ export interface ChorusStorageListResponse { storages: ChorusStorage[]; } +export interface ChorusBrowserBucketsResponse { + buckets: string[]; +} + +export interface ChorusBrowserObject { + key: string; + size: string; + lastModified: string; + sourceLastModified: string; +} + +export interface ChorusBrowserObjectsResponse { + objects: ChorusBrowserObject[]; + nextAfter: string; + hasMore: boolean; +} + export interface ChorusStorage { name: string; isMain: boolean; diff --git a/ui/src/utils/types/router.ts b/ui/src/utils/types/router.ts index 578722a3..9eb11de3 100644 --- a/ui/src/utils/types/router.ts +++ b/ui/src/utils/types/router.ts @@ -20,6 +20,7 @@ export enum RouteName { CHORUS_REPLICATION = 'CHORUS_REPLICATION', CHORUS_ADD_REPLICATION = 'CHORUS_ADD_REPLICATION', CHORUS_STORAGES = 'CHORUS_STORAGES', + CHORUS_OBJECT_BROWSER = 'CHORUS_OBJECT_BROWSER', CHORUS_STORAGE_DETAILS = 'CHORUS_STORAGE_DETAILS', CHORUS_MONITORING = 'CHORUS_MONITORING', CHORUS_ROUTING_POLICIES = 'CHORUS_ROUTING_POLICIES',