From 66298ed4bbb90bf111f9ff1390dafb266dc6b228 Mon Sep 17 00:00:00 2001 From: nfebe Date: Tue, 7 Jul 2026 18:20:16 +0100 Subject: [PATCH 1/5] feat(backup): Add remote S3-compatible backup destinations Backups can now be mirrored to remote S3-compatible object storage (AWS S3, Cloudflare R2, Backblaze B2, MinIO) on top of the always-local copy. A remote outage never fails a backup: the local copy is primary and uploads are best-effort. Object-storage credentials are held by the credential manager, written to a private file and masked in API responses, and referenced by destinations so no secret ever lands in the agent config. Destinations are configured through the existing config API and applied live. Listing, download, and restore transparently fall back to a remote copy when the local archive has been pruned, and local retention no longer deletes remote copies. A destination connectivity check is available, and what/how/where a deployment is backed up is now documented. --- config.example.yml | 15 ++ docs/BACKUPS.md | 57 ++++ go.mod | 17 +- go.sum | 22 ++ internal/api/backup_destinations.go | 128 +++++++++ internal/api/backup_handlers.go | 6 +- internal/api/config_handlers.go | 3 + internal/api/server.go | 14 + internal/api/storage_credential_handlers.go | 95 +++++++ internal/backup/manager.go | 34 ++- internal/backup/remotes.go | 284 ++++++++++++++++++++ internal/backup/remotes_test.go | 271 +++++++++++++++++++ internal/backup/store.go | 32 +++ internal/backup/store_s3.go | 169 ++++++++++++ internal/backup/types.go | 4 + internal/credentials/generic.go | 198 ++++++++++++++ internal/credentials/generic_test.go | 118 ++++++++ internal/credentials/manager.go | 28 +- pkg/config/backup_test.go | 38 +++ pkg/config/config.go | 25 ++ pkg/models/credential.go | 57 ++++ 21 files changed, 1587 insertions(+), 28 deletions(-) create mode 100644 docs/BACKUPS.md create mode 100644 internal/api/backup_destinations.go create mode 100644 internal/api/storage_credential_handlers.go create mode 100644 internal/backup/remotes.go create mode 100644 internal/backup/remotes_test.go create mode 100644 internal/backup/store.go create mode 100644 internal/backup/store_s3.go create mode 100644 internal/credentials/generic.go create mode 100644 internal/credentials/generic_test.go create mode 100644 pkg/config/backup_test.go create mode 100644 pkg/models/credential.go diff --git a/config.example.yml b/config.example.yml index 81ff82b..a0a679a 100644 --- a/config.example.yml +++ b/config.example.yml @@ -126,3 +126,18 @@ ai: api_key: your-gemini-api-key-here model: gemini-2.5-flash timeout: 1m0s +backup: + # Remote destinations that backups are mirrored to after the local copy is + # written. The local copy under /.flatrun/backups always + # remains the primary. Secrets live in the credential manager, never here: + # credential_id references an s3 credential managed via the API. + destinations: [] + # - name: s3-prod + # type: s3 + # endpoint: https://s3.us-east-1.amazonaws.com + # region: us-east-1 + # bucket: flatrun-backups + # prefix: agent-01 + # credential_id: + # use_path_style: false + # enabled: true diff --git a/docs/BACKUPS.md b/docs/BACKUPS.md new file mode 100644 index 0000000..99ef142 --- /dev/null +++ b/docs/BACKUPS.md @@ -0,0 +1,57 @@ +# Backups + +## What is backed up + +A backup captures a deployment as a self-contained archive. Each component is +best-effort: a component that cannot be read is logged and skipped, and the rest +of the backup still completes. + +| Component | Source | Included | +|-----------|--------|----------| +| Compose file | `docker-compose.yml` or `compose.yml` | Always | +| Env files | `.env`, `.env.flatrun` | Always | +| Deployment metadata | `.flatrun.yml` | Always | +| Mounted data | `data/`, `uploads/`, `storage/`, `config/`, `logs/` under the deployment | Always, when present | +| Container paths | Paths declared in the deployment's backup spec | When configured | +| Databases | MySQL (`mysqldump`) / PostgreSQL (`pg_dump`) dumps of declared services | When configured | + +Containers keep running during a backup. Optional pre and post hooks run inside +a service container around the capture. + +## How it is stored + +Components are staged, described by a `backup.json` manifest, and written as a +single gzip-compressed tar archive named `_.tar.gz`. + +## Where it is stored + +The primary copy is always local: + +``` +/.flatrun/backups//_.tar.gz +``` + +If one or more remote destinations are configured, the archive is also uploaded +to each after the local write. A remote upload failure is logged and never fails +the backup, whose local copy already succeeded. A backup's `locations` field +reports where it currently exists (`local` and/or destination names). Local +retention prunes only the on-disk copies; remote copies are governed by the +bucket's own lifecycle policy. + +## Remote (S3-compatible) destinations + +Any S3-compatible service works (AWS S3, Cloudflare R2, Backblaze B2, MinIO). + +1. Store the access key and secret as an S3 credential + (`POST /api/v1/storage-credentials`, `kind: s3`). Secrets are held by the + credential manager, written to a `0600` file, and never returned by the API + or stored in the agent config. +2. Add a destination under `backup.destinations` referencing that credential by + id (via the config API or `config.yml`). See `config.example.yml` for the + shape. +3. Validate reachability with `POST /api/v1/backup-destinations/test` before + relying on it; it performs a probe write and delete. + +Restore and download read from the local copy when present, otherwise stream the +archive from the first remote that holds it, so a backup remains usable after +its local copy has been pruned. diff --git a/go.mod b/go.mod index 6e7a6f2..61d6ad3 100644 --- a/go.mod +++ b/go.mod @@ -39,18 +39,23 @@ require ( github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/aws/aws-sdk-go-v2 v1.41.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.42.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect - github.com/aws/smithy-go v1.24.0 // indirect + github.com/aws/smithy-go v1.27.3 // indirect github.com/buger/goterm v1.0.4 // indirect github.com/bytedance/sonic v1.13.3 // indirect github.com/bytedance/sonic/loader v0.2.4 // indirect diff --git a/go.sum b/go.sum index ef647cd..2a0dc8e 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,10 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4= github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek= +github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E= github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8= github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI= github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE= @@ -32,16 +36,32 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBU github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 h1:3GUprIsfmGcC5SACIyB0e7E0BM1O1b3Erl5CePYIAeQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31/go.mod h1:7PuV1yl5e2xnUbm+RqvVg5i2iBM8EyijZNoI9wsOoOc= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I= github.com/aws/aws-sdk-go-v2/service/route53 v1.62.0 h1:80pDB3Tpmb2RCSZORrK9/3iQxsd+w6vSzVqpT1FGiwE= github.com/aws/aws-sdk-go-v2/service/route53 v1.62.0/go.mod h1:6EZUGGNLPLh5Unt30uEoA+KQcByERfXIkax9qrc80nA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0 h1:XptwLL+UHXgafYMIHTy59IRovLbhz3znkxY2uS/pbXU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do= github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ= github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU= github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw= @@ -52,6 +72,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk= github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= +github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= diff --git a/internal/api/backup_destinations.go b/internal/api/backup_destinations.go new file mode 100644 index 0000000..0be4b45 --- /dev/null +++ b/internal/api/backup_destinations.go @@ -0,0 +1,128 @@ +package api + +import ( + "bytes" + "fmt" + "log" + "net/http" + "strings" + + "github.com/flatrun/agent/internal/backup" + "github.com/flatrun/agent/pkg/config" + "github.com/flatrun/agent/pkg/models" + "github.com/gin-gonic/gin" +) + +// buildStore resolves a destination plus its referenced credential into a +// ready-to-use remote Store. +func (s *Server) buildStore(dest config.BackupDestination) (backup.Store, error) { + switch dest.Type { + case "s3", "": + cred, err := s.credentialsManager.GetGenericCredential(dest.CredentialID) + if err != nil { + return nil, fmt.Errorf("destination %q: %w", dest.Name, err) + } + if cred.Kind != models.CredentialKindS3 { + return nil, fmt.Errorf("destination %q: credential %q is not an s3 credential", dest.Name, dest.CredentialID) + } + return backup.NewS3Store(backup.S3Config{ + Name: dest.Name, + Endpoint: dest.Endpoint, + Region: dest.Region, + Bucket: dest.Bucket, + Prefix: dest.Prefix, + AccessKeyID: cred.Data["access_key_id"], + SecretKey: cred.Data["secret_access_key"], + UsePathStyle: dest.UsePathStyle, + }) + default: + return nil, fmt.Errorf("destination %q: unknown type %q", dest.Name, dest.Type) + } +} + +// applyBackupDestinations rebuilds the backup manager's remote stores from the +// current config. It is called at startup and by the runtime applier when the +// destinations config changes. +func (s *Server) applyBackupDestinations() error { + if s.backupManager == nil { + return nil + } + + var stores []backup.Store + var problems []string + for _, dest := range s.config.Backup.Destinations { + if !dest.IsEnabled() { + continue + } + store, err := s.buildStore(dest) + if err != nil { + problems = append(problems, err.Error()) + continue + } + stores = append(stores, store) + } + + s.backupManager.SetRemotes(stores) + + if len(problems) > 0 { + return fmt.Errorf("backup destinations not fully applied: %s", strings.Join(problems, "; ")) + } + log.Printf("Backup destinations applied: %d remote(s) active", len(stores)) + return nil +} + +// testBackupDestination validates connectivity to a destination by writing and +// deleting a small probe object. The destination may be given inline (for the +// configure-then-test flow) or by name for an already-saved destination. +func (s *Server) testBackupDestination(c *gin.Context) { + if s.backupManager == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Backup manager not enabled"}) + return + } + + var dest config.BackupDestination + if err := c.ShouldBindJSON(&dest); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + if dest.Name == "" && dest.Bucket == "" { + if named, ok := s.findDestinationByName(c.Query("name")); ok { + dest = named + } + } + + store, err := s.buildStore(dest) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) + return + } + + ctx := c.Request.Context() + probeKey := ".flatrun/connectivity-check" + probe := []byte("flatrun") + if err := store.Put(ctx, probeKey, bytes.NewReader(probe), int64(len(probe))); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "error": err.Error()}) + return + } + _ = store.Delete(ctx, probeKey) + + c.JSON(http.StatusOK, gin.H{"success": true, "message": "Destination reachable and writable"}) +} + +func (s *Server) listBackupDestinations(c *gin.Context) { + dests := s.config.Backup.Destinations + if dests == nil { + dests = []config.BackupDestination{} + } + c.JSON(http.StatusOK, gin.H{"destinations": dests}) +} + +func (s *Server) findDestinationByName(name string) (config.BackupDestination, bool) { + for _, d := range s.config.Backup.Destinations { + if d.Name == name { + return d, true + } + } + return config.BackupDestination{}, false +} diff --git a/internal/api/backup_handlers.go b/internal/api/backup_handlers.go index 75b44db..697a651 100644 --- a/internal/api/backup_handlers.go +++ b/internal/api/backup_handlers.go @@ -189,16 +189,16 @@ func (s *Server) downloadBackup(c *gin.Context) { return } - backupPath, err := s.backupManager.GetBackupPath(backupID) + reader, size, err := s.backupManager.OpenBackupArchive(c.Request.Context(), backupID) if err != nil { c.JSON(http.StatusNotFound, gin.H{"error": err.Error()}) return } + defer reader.Close() c.Header("Content-Description", "File Transfer") c.Header("Content-Disposition", "attachment; filename="+backupID+".tar.gz") - c.Header("Content-Type", "application/gzip") - c.File(backupPath) + c.DataFromReader(http.StatusOK, size, "application/gzip", reader, nil) } func (s *Server) getDeploymentBackupConfig(c *gin.Context) { diff --git a/internal/api/config_handlers.go b/internal/api/config_handlers.go index 7bed313..4a5c999 100644 --- a/internal/api/config_handlers.go +++ b/internal/api/config_handlers.go @@ -133,6 +133,9 @@ func (s *Server) runtimeAppliers() map[string]func(*Server) error { srv.manager.SetCleanupTimeout(srv.config.Cleanup.Timeout) return nil }, + "backup.destinations": func(srv *Server) error { + return srv.applyBackupDestinations() + }, "ai.enabled": rebuildAIProvider, "ai.base_url": rebuildAIProvider, "ai.api_key": rebuildAIProvider, diff --git a/internal/api/server.go b/internal/api/server.go index b0f5e28..3271706 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -348,6 +348,10 @@ func New(cfg *config.Config, configPath string) *Server { } if backupManager != nil { + if err := s.applyBackupDestinations(); err != nil { + log.Printf("Warning: %v", err) + } + executor := scheduler.NewExecutor(backupManager, manager) schedulerManager, err := scheduler.NewManager(cfg.DeploymentsPath, executor) if err != nil { @@ -619,6 +623,12 @@ func (s *Server) setupRoutes() { protected.DELETE("/credentials/:id", s.authMiddleware.RequirePermission(auth.PermRegistriesDelete), s.deleteCredential) protected.POST("/credentials/:id/test", s.authMiddleware.RequirePermission(auth.PermRegistriesRead), s.testCredential) + // Storage credential endpoints (S3 and other object-storage secrets) + protected.GET("/storage-credentials", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.listStorageCredentials) + protected.POST("/storage-credentials", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.createStorageCredential) + protected.PUT("/storage-credentials/:id", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.updateStorageCredential) + protected.DELETE("/storage-credentials/:id", s.authMiddleware.RequirePermission(auth.PermBackupsDelete), s.deleteStorageCredential) + // Security endpoints protected.GET("/security/stats", s.authMiddleware.RequirePermission(auth.PermSecurityRead), s.getSecurityStats) protected.GET("/security/events", s.authMiddleware.RequirePermission(auth.PermSecurityRead), s.listSecurityEvents) @@ -664,6 +674,10 @@ func (s *Server) setupRoutes() { protected.GET("/backups/jobs", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.listBackupJobs) protected.GET("/backups/jobs/:id", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.getBackupJob) + // Remote backup destinations + protected.GET("/backup-destinations", s.authMiddleware.RequirePermission(auth.PermBackupsRead), s.listBackupDestinations) + protected.POST("/backup-destinations/test", s.authMiddleware.RequirePermission(auth.PermBackupsWrite), s.testBackupDestination) + // Scheduler endpoints protected.GET("/scheduler/tasks", s.authMiddleware.RequirePermission(auth.PermSchedulerRead), s.listScheduledTasks) protected.GET("/scheduler/tasks/:id", s.authMiddleware.RequirePermission(auth.PermSchedulerRead), s.getScheduledTask) diff --git a/internal/api/storage_credential_handlers.go b/internal/api/storage_credential_handlers.go new file mode 100644 index 0000000..58505f6 --- /dev/null +++ b/internal/api/storage_credential_handlers.go @@ -0,0 +1,95 @@ +package api + +import ( + "fmt" + "net/http" + + "github.com/flatrun/agent/pkg/models" + "github.com/gin-gonic/gin" +) + +func (s *Server) listStorageCredentials(c *gin.Context) { + kind := models.CredentialKind(c.Query("kind")) + creds := s.credentialsManager.ListGenericCredentials(kind) + c.JSON(http.StatusOK, gin.H{"credentials": creds}) +} + +func (s *Server) createStorageCredential(c *gin.Context) { + var req struct { + Name string `json:"name" binding:"required"` + Kind string `json:"kind" binding:"required"` + Data map[string]string `json:"data"` + } + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + kind := models.CredentialKind(req.Kind) + if err := validateStorageCredentialData(kind, req.Data, true); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + cred, err := s.credentialsManager.CreateGenericCredential(req.Name, kind, req.Data) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusCreated, gin.H{"message": "Credential created", "credential": cred}) +} + +func (s *Server) updateStorageCredential(c *gin.Context) { + id := c.Param("id") + + var req struct { + Name string `json:"name"` + Data map[string]string `json:"data"` + } + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + cred, err := s.credentialsManager.UpdateGenericCredential(id, req.Name, req.Data) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{"message": "Credential updated", "credential": cred}) +} + +func (s *Server) deleteStorageCredential(c *gin.Context) { + id := c.Param("id") + + for _, dest := range s.config.Backup.Destinations { + if dest.CredentialID == id { + c.JSON(http.StatusConflict, gin.H{"error": "credential is in use by backup destination " + dest.Name}) + return + } + } + + if err := s.credentialsManager.DeleteGenericCredential(id); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{"message": "Credential deleted", "id": id}) +} + +func validateStorageCredentialData(kind models.CredentialKind, data map[string]string, requireSecret bool) error { + switch kind { + case models.CredentialKindS3: + if data["access_key_id"] == "" { + return fmt.Errorf("access_key_id is required") + } + if requireSecret && data["secret_access_key"] == "" { + return fmt.Errorf("secret_access_key is required") + } + return nil + default: + return fmt.Errorf("unknown credential kind: %s", kind) + } +} diff --git a/internal/backup/manager.go b/internal/backup/manager.go index ed203cf..72c7980 100644 --- a/internal/backup/manager.go +++ b/internal/backup/manager.go @@ -13,6 +13,7 @@ import ( "path/filepath" "sort" "strings" + "sync" "time" "github.com/flatrun/agent/pkg/version" @@ -22,6 +23,9 @@ type Manager struct { deploymentsPath string backupsPath string jobs *JobTracker + + remotesMu sync.RWMutex + remotes []Store } func NewManager(deploymentsPath string) (*Manager, error) { @@ -149,6 +153,9 @@ func (m *Manager) CreateBackup(ctx context.Context, deploymentName string, spec now := time.Now() backup.CompletedAt = &now + backup.Locations = []string{locationLocal} + backup.Locations = append(backup.Locations, m.mirrorToRemotes(ctx, deploymentName, backupID, archivePath, backup.Size)...) + log.Printf("Backup completed: %s (%d bytes)", backupID, backup.Size) return backup, nil } @@ -456,7 +463,7 @@ func (m *Manager) createArchive(sourceDir, destPath string) error { }) } -func (m *Manager) ListBackups(filter *BackupListFilter) ([]Backup, error) { +func (m *Manager) listLocalBackups(filter *BackupListFilter) ([]Backup, error) { var backups []Backup deploymentDirs, err := os.ReadDir(m.backupsPath) @@ -500,6 +507,7 @@ func (m *Manager) ListBackups(filter *BackupListFilter) ([]Backup, error) { Size: info.Size(), Path: filepath.Join(backupDir, file.Name()), CreatedAt: info.ModTime(), + Locations: []string{locationLocal}, } backups = append(backups, backup) @@ -517,7 +525,7 @@ func (m *Manager) ListBackups(filter *BackupListFilter) ([]Backup, error) { return backups, nil } -func (m *Manager) GetBackup(backupID string) (*Backup, error) { +func (m *Manager) getLocalBackup(backupID string) (*Backup, error) { parts := strings.SplitN(backupID, "_", 2) if len(parts) < 2 { return nil, fmt.Errorf("invalid backup ID format") @@ -538,11 +546,12 @@ func (m *Manager) GetBackup(backupID string) (*Backup, error) { Size: info.Size(), Path: backupPath, CreatedAt: info.ModTime(), + Locations: []string{locationLocal}, }, nil } -func (m *Manager) DeleteBackup(backupID string) error { - backup, err := m.GetBackup(backupID) +func (m *Manager) deleteLocalBackup(backupID string) error { + backup, err := m.getLocalBackup(backupID) if err != nil { return err } @@ -551,15 +560,18 @@ func (m *Manager) DeleteBackup(backupID string) error { } func (m *Manager) GetBackupPath(backupID string) (string, error) { - backup, err := m.GetBackup(backupID) + backup, err := m.getLocalBackup(backupID) if err != nil { return "", err } return backup.Path, nil } +// CleanupOldBackups prunes the local on-disk copies beyond keepCount. Retention +// applies to local disk only; remote copies are governed by the destination's +// own lifecycle policy and are never deleted here. func (m *Manager) CleanupOldBackups(deploymentName string, keepCount int) (int, error) { - backups, err := m.ListBackups(&BackupListFilter{DeploymentName: deploymentName}) + backups, err := m.listLocalBackups(&BackupListFilter{DeploymentName: deploymentName}) if err != nil { return 0, err } @@ -570,7 +582,7 @@ func (m *Manager) CleanupOldBackups(deploymentName string, keepCount int) (int, deleted := 0 for _, backup := range backups[keepCount:] { - if err := m.DeleteBackup(backup.ID); err != nil { + if err := m.deleteLocalBackup(backup.ID); err != nil { log.Printf("Failed to delete old backup %s: %v", backup.ID, err) continue } @@ -599,7 +611,13 @@ func (m *Manager) RestoreBackup(ctx context.Context, req *RestoreBackupRequest) } defer os.RemoveAll(tempDir) - if err := m.extractArchive(backup.Path, tempDir); err != nil { + archivePath, cleanup, err := m.ensureLocalArchive(ctx, backup) + if err != nil { + return err + } + defer cleanup() + + if err := m.extractArchive(archivePath, tempDir); err != nil { return fmt.Errorf("failed to extract backup: %w", err) } diff --git a/internal/backup/remotes.go b/internal/backup/remotes.go new file mode 100644 index 0000000..3397518 --- /dev/null +++ b/internal/backup/remotes.go @@ -0,0 +1,284 @@ +package backup + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "os" + "sort" + "strings" +) + +const locationLocal = "local" + +// SetRemotes replaces the set of remote destinations backups are mirrored to. +// It is called at startup and whenever the backup destination config changes. +func (m *Manager) SetRemotes(remotes []Store) { + m.remotesMu.Lock() + m.remotes = remotes + m.remotesMu.Unlock() +} + +func (m *Manager) getRemotes() []Store { + m.remotesMu.RLock() + defer m.remotesMu.RUnlock() + return append([]Store(nil), m.remotes...) +} + +func backupKey(deploymentName, backupID string) string { + return deploymentName + "/" + backupID + ".tar.gz" +} + +func parseBackupKey(key string) (deployment, backupID string, ok bool) { + parts := strings.SplitN(key, "/", 2) + if len(parts) != 2 || !strings.HasSuffix(parts[1], ".tar.gz") { + return "", "", false + } + return parts[0], strings.TrimSuffix(parts[1], ".tar.gz"), true +} + +func deploymentFromID(backupID string) string { + parts := strings.SplitN(backupID, "_", 2) + if len(parts) < 2 { + return "" + } + return parts[0] +} + +// mirrorToRemotes uploads a freshly written local archive to every configured +// remote, returning the names of the destinations that accepted it. A failed +// upload is logged and skipped so a remote outage never fails the backup, whose +// local copy already succeeded. +func (m *Manager) mirrorToRemotes(ctx context.Context, deploymentName, backupID, archivePath string, size int64) []string { + remotes := m.getRemotes() + if len(remotes) == 0 { + return nil + } + + key := backupKey(deploymentName, backupID) + var locations []string + for _, r := range remotes { + f, err := os.Open(archivePath) + if err != nil { + log.Printf("Backup: mirror to %s failed to open archive: %v", r.Name(), err) + continue + } + err = r.Put(ctx, key, f, size) + f.Close() + if err != nil { + log.Printf("Backup: mirror to %s failed: %v", r.Name(), err) + continue + } + locations = append(locations, r.Name()) + log.Printf("Backup mirrored: %s -> %s", backupID, r.Name()) + } + return locations +} + +// ListBackups returns backups from the local disk merged with those in every +// remote destination, deduped by ID with Locations tagged. A remote that fails +// to list is logged and skipped so the local listing still returns. +func (m *Manager) ListBackups(filter *BackupListFilter) ([]Backup, error) { + localFilter := *filter + localFilter.Limit = 0 + localFilter.Offset = 0 + + locals, err := m.listLocalBackups(&localFilter) + if err != nil { + return nil, err + } + + byID := make(map[string]*Backup, len(locals)) + for i := range locals { + b := locals[i] + byID[b.ID] = &b + } + + prefix := "" + if filter.DeploymentName != "" { + prefix = filter.DeploymentName + "/" + } + for _, r := range m.getRemotes() { + objs, err := r.List(context.Background(), prefix) + if err != nil { + log.Printf("Backup: list remote %s failed: %v", r.Name(), err) + continue + } + for _, obj := range objs { + dep, id, ok := parseBackupKey(obj.Key) + if !ok { + continue + } + if filter.DeploymentName != "" && dep != filter.DeploymentName { + continue + } + if existing, found := byID[id]; found { + existing.Locations = append(existing.Locations, r.Name()) + continue + } + byID[id] = &Backup{ + ID: id, + DeploymentName: dep, + Status: BackupStatusCompleted, + Size: obj.Size, + CreatedAt: obj.ModTime, + Locations: []string{r.Name()}, + } + } + } + + result := make([]Backup, 0, len(byID)) + for _, b := range byID { + result = append(result, *b) + } + sort.Slice(result, func(i, j int) bool { + return result[i].CreatedAt.After(result[j].CreatedAt) + }) + + if filter.Limit > 0 && len(result) > filter.Limit { + result = result[:filter.Limit] + } + + return result, nil +} + +// GetBackup resolves a backup by ID, preferring the local copy and falling back +// to the first remote that holds it. +func (m *Manager) GetBackup(backupID string) (*Backup, error) { + if b, err := m.getLocalBackup(backupID); err == nil { + return b, nil + } + + dep := deploymentFromID(backupID) + if dep == "" { + return nil, fmt.Errorf("invalid backup ID format") + } + + key := backupKey(dep, backupID) + for _, r := range m.getRemotes() { + info, err := r.Stat(context.Background(), key) + if err != nil { + if !errors.Is(err, ErrObjectNotFound) { + log.Printf("Backup: stat remote %s failed: %v", r.Name(), err) + } + continue + } + return &Backup{ + ID: backupID, + DeploymentName: dep, + Status: BackupStatusCompleted, + Size: info.Size, + CreatedAt: info.ModTime, + Locations: []string{r.Name()}, + }, nil + } + + return nil, fmt.Errorf("backup not found: %s", backupID) +} + +// DeleteBackup removes a backup from the local disk and every remote it may +// exist in. Remote deletes are idempotent, so a backup already absent remotely +// is not an error. +func (m *Manager) DeleteBackup(backupID string) error { + dep := deploymentFromID(backupID) + if dep == "" { + return fmt.Errorf("invalid backup ID format") + } + + localExisted := false + if _, err := m.getLocalBackup(backupID); err == nil { + if err := m.deleteLocalBackup(backupID); err != nil { + return err + } + localExisted = true + } + + remotes := m.getRemotes() + key := backupKey(dep, backupID) + var firstErr error + for _, r := range remotes { + if err := r.Delete(context.Background(), key); err != nil && firstErr == nil { + firstErr = err + } + } + if firstErr != nil { + return firstErr + } + + if !localExisted && len(remotes) == 0 { + return fmt.Errorf("backup not found: %s", backupID) + } + return nil +} + +// OpenBackupArchive returns a reader for the backup archive, from local disk if +// present, otherwise streamed from the first remote that holds it. +func (m *Manager) OpenBackupArchive(ctx context.Context, backupID string) (io.ReadCloser, int64, error) { + if b, err := m.getLocalBackup(backupID); err == nil { + f, err := os.Open(b.Path) + if err != nil { + return nil, 0, err + } + return f, b.Size, nil + } + + dep := deploymentFromID(backupID) + if dep == "" { + return nil, 0, fmt.Errorf("invalid backup ID format") + } + + key := backupKey(dep, backupID) + for _, r := range m.getRemotes() { + info, err := r.Stat(ctx, key) + if err != nil { + continue + } + rc, err := r.Open(ctx, key) + if err != nil { + continue + } + return rc, info.Size, nil + } + + return nil, 0, fmt.Errorf("backup not found: %s", backupID) +} + +// ensureLocalArchive returns a filesystem path to the backup archive, pulling it +// from a remote into a temp file when the local copy is gone. The returned +// cleanup removes any temp file it created. +func (m *Manager) ensureLocalArchive(ctx context.Context, backup *Backup) (string, func(), error) { + noop := func() {} + if backup.Path != "" { + if _, err := os.Stat(backup.Path); err == nil { + return backup.Path, noop, nil + } + } + + key := backupKey(backup.DeploymentName, backup.ID) + for _, r := range m.getRemotes() { + rc, err := r.Open(ctx, key) + if err != nil { + if !errors.Is(err, ErrObjectNotFound) { + log.Printf("Backup: fetch from remote %s failed: %v", r.Name(), err) + } + continue + } + tmp, err := os.CreateTemp("", "flatrun-remote-*.tar.gz") + if err != nil { + rc.Close() + return "", noop, err + } + _, err = io.Copy(tmp, rc) + rc.Close() + tmp.Close() + if err != nil { + os.Remove(tmp.Name()) + return "", noop, fmt.Errorf("failed to download backup from %s: %w", r.Name(), err) + } + return tmp.Name(), func() { os.Remove(tmp.Name()) }, nil + } + + return "", noop, fmt.Errorf("backup archive not available locally or in any remote: %s", backup.ID) +} diff --git a/internal/backup/remotes_test.go b/internal/backup/remotes_test.go new file mode 100644 index 0000000..d9c3b9f --- /dev/null +++ b/internal/backup/remotes_test.go @@ -0,0 +1,271 @@ +package backup + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +type fakeStore struct { + name string + mu sync.Mutex + objects map[string][]byte + modtimes map[string]time.Time + failList bool +} + +func newFakeStore(name string) *fakeStore { + return &fakeStore{name: name, objects: map[string][]byte{}, modtimes: map[string]time.Time{}} +} + +func (f *fakeStore) Name() string { return f.name } + +func (f *fakeStore) Put(_ context.Context, key string, r io.Reader, _ int64) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + f.mu.Lock() + defer f.mu.Unlock() + f.objects[key] = data + f.modtimes[key] = time.Now() + return nil +} + +func (f *fakeStore) Open(_ context.Context, key string) (io.ReadCloser, error) { + f.mu.Lock() + defer f.mu.Unlock() + data, ok := f.objects[key] + if !ok { + return nil, ErrObjectNotFound + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +func (f *fakeStore) List(_ context.Context, prefix string) ([]ObjectInfo, error) { + if f.failList { + return nil, fmt.Errorf("list failed") + } + f.mu.Lock() + defer f.mu.Unlock() + var out []ObjectInfo + for key, data := range f.objects { + if !strings.HasPrefix(key, prefix) { + continue + } + out = append(out, ObjectInfo{Key: key, Size: int64(len(data)), ModTime: f.modtimes[key]}) + } + return out, nil +} + +func (f *fakeStore) Delete(_ context.Context, key string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.objects, key) + delete(f.modtimes, key) + return nil +} + +func (f *fakeStore) Stat(_ context.Context, key string) (ObjectInfo, error) { + f.mu.Lock() + defer f.mu.Unlock() + data, ok := f.objects[key] + if !ok { + return ObjectInfo{}, ErrObjectNotFound + } + return ObjectInfo{Key: key, Size: int64(len(data)), ModTime: f.modtimes[key]}, nil +} + +func (f *fakeStore) has(key string) bool { + f.mu.Lock() + defer f.mu.Unlock() + _, ok := f.objects[key] + return ok +} + +func seedDeployment(t *testing.T, root, name string) { + t.Helper() + dir := filepath.Join(root, name) + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatalf("mkdir deployment: %v", err) + } + compose := "services:\n web:\n image: nginx\n" + if err := os.WriteFile(filepath.Join(dir, "docker-compose.yml"), []byte(compose), 0644); err != nil { + t.Fatalf("write compose: %v", err) + } +} + +func containsStr(s []string, want string) bool { + for _, v := range s { + if v == want { + return true + } + } + return false +} + +func TestCreateBackup_MirrorsToRemote(t *testing.T) { + m, tmpDir := setupTestManager(t) + defer os.RemoveAll(tmpDir) + + remote := newFakeStore("s3-test") + m.SetRemotes([]Store{remote}) + seedDeployment(t, tmpDir, "app") + + b, err := m.CreateBackup(context.Background(), "app", nil) + if err != nil { + t.Fatalf("create backup: %v", err) + } + + if !containsStr(b.Locations, locationLocal) || !containsStr(b.Locations, "s3-test") { + t.Fatalf("expected local and remote locations, got %v", b.Locations) + } + if !remote.has(backupKey("app", b.ID)) { + t.Fatalf("expected remote to hold %s", backupKey("app", b.ID)) + } +} + +func TestListAndGetBackup_RemoteOnlyAfterLocalPruned(t *testing.T) { + m, tmpDir := setupTestManager(t) + defer os.RemoveAll(tmpDir) + + remote := newFakeStore("s3-test") + m.SetRemotes([]Store{remote}) + seedDeployment(t, tmpDir, "app") + + b, err := m.CreateBackup(context.Background(), "app", nil) + if err != nil { + t.Fatalf("create backup: %v", err) + } + + // Simulate local retention pruning the on-disk copy. + if err := m.deleteLocalBackup(b.ID); err != nil { + t.Fatalf("prune local: %v", err) + } + + list, err := m.ListBackups(&BackupListFilter{DeploymentName: "app"}) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(list) != 1 { + t.Fatalf("expected 1 backup, got %d", len(list)) + } + if list[0].Locations[0] != "s3-test" || containsStr(list[0].Locations, locationLocal) { + t.Fatalf("expected remote-only location, got %v", list[0].Locations) + } + + got, err := m.GetBackup(b.ID) + if err != nil { + t.Fatalf("get remote-only backup: %v", err) + } + if got.Size != b.Size { + t.Fatalf("expected size %d, got %d", b.Size, got.Size) + } +} + +func TestDeleteBackup_RemovesFromLocalAndRemote(t *testing.T) { + m, tmpDir := setupTestManager(t) + defer os.RemoveAll(tmpDir) + + remote := newFakeStore("s3-test") + m.SetRemotes([]Store{remote}) + seedDeployment(t, tmpDir, "app") + + b, err := m.CreateBackup(context.Background(), "app", nil) + if err != nil { + t.Fatalf("create backup: %v", err) + } + + if err := m.DeleteBackup(b.ID); err != nil { + t.Fatalf("delete: %v", err) + } + if remote.has(backupKey("app", b.ID)) { + t.Fatal("expected remote copy to be deleted") + } + if _, err := m.getLocalBackup(b.ID); err == nil { + t.Fatal("expected local copy to be deleted") + } +} + +func TestRestoreBackup_FromRemoteWhenLocalMissing(t *testing.T) { + m, tmpDir := setupTestManager(t) + defer os.RemoveAll(tmpDir) + + remote := newFakeStore("s3-test") + m.SetRemotes([]Store{remote}) + seedDeployment(t, tmpDir, "app") + + b, err := m.CreateBackup(context.Background(), "app", nil) + if err != nil { + t.Fatalf("create backup: %v", err) + } + if err := m.deleteLocalBackup(b.ID); err != nil { + t.Fatalf("prune local: %v", err) + } + + // Wipe the deployment so restore must repopulate it from the remote copy. + if err := os.RemoveAll(filepath.Join(tmpDir, "app")); err != nil { + t.Fatalf("remove deployment: %v", err) + } + + err = m.RestoreBackup(context.Background(), &RestoreBackupRequest{BackupID: b.ID}) + if err != nil { + t.Fatalf("restore from remote: %v", err) + } + + if _, err := os.Stat(filepath.Join(tmpDir, "app", "docker-compose.yml")); err != nil { + t.Fatalf("expected compose file restored from remote: %v", err) + } +} + +func TestListBackups_RemoteListFailureDoesNotBreakLocal(t *testing.T) { + m, tmpDir := setupTestManager(t) + defer os.RemoveAll(tmpDir) + + remote := newFakeStore("s3-test") + remote.failList = true + m.SetRemotes([]Store{remote}) + seedDeployment(t, tmpDir, "app") + + if _, err := m.CreateBackup(context.Background(), "app", nil); err != nil { + t.Fatalf("create backup: %v", err) + } + + list, err := m.ListBackups(&BackupListFilter{DeploymentName: "app"}) + if err != nil { + t.Fatalf("list should tolerate remote failure: %v", err) + } + if len(list) != 1 || !containsStr(list[0].Locations, locationLocal) { + t.Fatalf("expected local backup despite remote list failure, got %v", list) + } +} + +func TestParseBackupKey(t *testing.T) { + dep, id, ok := parseBackupKey("app/app_20240101_120000.tar.gz") + if !ok || dep != "app" || id != "app_20240101_120000" { + t.Fatalf("parseBackupKey mismatch: dep=%q id=%q ok=%v", dep, id, ok) + } + if _, _, ok := parseBackupKey("no-slash.txt"); ok { + t.Fatal("expected parse failure for malformed key") + } +} + +func TestS3StoreRequiresBucketAndCreds(t *testing.T) { + if _, err := NewS3Store(S3Config{Name: "x"}); err == nil { + t.Fatal("expected error without bucket") + } + if _, err := NewS3Store(S3Config{Name: "x", Bucket: "b"}); err == nil { + t.Fatal("expected error without credentials") + } + if !errors.Is(ErrObjectNotFound, ErrObjectNotFound) { + t.Fatal("sentinel identity") + } +} diff --git a/internal/backup/store.go b/internal/backup/store.go new file mode 100644 index 0000000..c42c183 --- /dev/null +++ b/internal/backup/store.go @@ -0,0 +1,32 @@ +package backup + +import ( + "context" + "io" + "time" +) + +// ObjectInfo describes a stored backup archive in a remote destination. +type ObjectInfo struct { + Key string + Size int64 + ModTime time.Time +} + +// Store is a remote destination that backup archives are mirrored to. The local +// filesystem remains the primary copy and is handled directly by the Manager; +// a Store is only ever a secondary, remote target. +type Store interface { + // Name is the destination's configured name, surfaced as a backup location. + Name() string + // Put writes an object of the given size at key. + Put(ctx context.Context, key string, r io.Reader, size int64) error + // Open returns a reader for the object at key. + Open(ctx context.Context, key string) (io.ReadCloser, error) + // List returns objects whose key begins with prefix. + List(ctx context.Context, prefix string) ([]ObjectInfo, error) + // Delete removes the object at key. Missing objects are not an error. + Delete(ctx context.Context, key string) error + // Stat returns metadata for a single object. + Stat(ctx context.Context, key string) (ObjectInfo, error) +} diff --git a/internal/backup/store_s3.go b/internal/backup/store_s3.go new file mode 100644 index 0000000..046c20c --- /dev/null +++ b/internal/backup/store_s3.go @@ -0,0 +1,169 @@ +package backup + +import ( + "context" + "errors" + "fmt" + "io" + "path" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +// ErrObjectNotFound is returned by a Store when the requested key is absent. +var ErrObjectNotFound = errors.New("object not found") + +// S3Config holds everything needed to reach one S3-compatible bucket. It carries +// resolved secrets and is assembled by the caller from a destination plus its +// referenced credential; it is never persisted. +type S3Config struct { + Name string + Endpoint string + Region string + Bucket string + Prefix string + AccessKeyID string + SecretKey string + UsePathStyle bool +} + +type S3Store struct { + client *s3.Client + cfg S3Config +} + +func NewS3Store(cfg S3Config) (*S3Store, error) { + if cfg.Bucket == "" { + return nil, fmt.Errorf("s3 destination %q: bucket is required", cfg.Name) + } + if cfg.AccessKeyID == "" || cfg.SecretKey == "" { + return nil, fmt.Errorf("s3 destination %q: credentials are required", cfg.Name) + } + + region := cfg.Region + if region == "" { + region = "us-east-1" + } + + opts := s3.Options{ + Region: region, + Credentials: credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretKey, ""), + UsePathStyle: cfg.UsePathStyle, + } + if cfg.Endpoint != "" { + opts.BaseEndpoint = aws.String(cfg.Endpoint) + } + + return &S3Store{client: s3.New(opts), cfg: cfg}, nil +} + +func (s *S3Store) Name() string { return s.cfg.Name } + +// fullKey prepends the destination prefix to a relative backup key. +func (s *S3Store) fullKey(key string) string { + if s.cfg.Prefix == "" { + return key + } + return path.Join(s.cfg.Prefix, key) +} + +// relKey strips the destination prefix so returned keys match the Manager's +// relative /.tar.gz key space. +func (s *S3Store) relKey(full string) string { + if s.cfg.Prefix == "" { + return full + } + return strings.TrimPrefix(full, strings.TrimSuffix(s.cfg.Prefix, "/")+"/") +} + +func (s *S3Store) Put(ctx context.Context, key string, r io.Reader, size int64) error { + _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.cfg.Bucket), + Key: aws.String(s.fullKey(key)), + Body: r, + ContentLength: aws.Int64(size), + }) + if err != nil { + return fmt.Errorf("s3 put %s: %w", key, err) + } + return nil +} + +func (s *S3Store) Open(ctx context.Context, key string) (io.ReadCloser, error) { + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.cfg.Bucket), + Key: aws.String(s.fullKey(key)), + }) + if err != nil { + if isS3NotFound(err) { + return nil, ErrObjectNotFound + } + return nil, fmt.Errorf("s3 get %s: %w", key, err) + } + return out.Body, nil +} + +func (s *S3Store) List(ctx context.Context, prefix string) ([]ObjectInfo, error) { + var out []ObjectInfo + paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{ + Bucket: aws.String(s.cfg.Bucket), + Prefix: aws.String(s.fullKey(prefix)), + }) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("s3 list %s: %w", prefix, err) + } + for _, obj := range page.Contents { + key := s.relKey(aws.ToString(obj.Key)) + if !strings.HasSuffix(key, ".tar.gz") { + continue + } + out = append(out, ObjectInfo{ + Key: key, + Size: aws.ToInt64(obj.Size), + ModTime: aws.ToTime(obj.LastModified), + }) + } + } + return out, nil +} + +func (s *S3Store) Delete(ctx context.Context, key string) error { + _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.cfg.Bucket), + Key: aws.String(s.fullKey(key)), + }) + if err != nil && !isS3NotFound(err) { + return fmt.Errorf("s3 delete %s: %w", key, err) + } + return nil +} + +func (s *S3Store) Stat(ctx context.Context, key string) (ObjectInfo, error) { + out, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(s.cfg.Bucket), + Key: aws.String(s.fullKey(key)), + }) + if err != nil { + if isS3NotFound(err) { + return ObjectInfo{}, ErrObjectNotFound + } + return ObjectInfo{}, fmt.Errorf("s3 head %s: %w", key, err) + } + return ObjectInfo{ + Key: key, + Size: aws.ToInt64(out.ContentLength), + ModTime: aws.ToTime(out.LastModified), + }, nil +} + +func isS3NotFound(err error) bool { + var nsk *s3types.NoSuchKey + var nf *s3types.NotFound + return errors.As(err, &nsk) || errors.As(err, &nf) +} diff --git a/internal/backup/types.go b/internal/backup/types.go index 9d4d469..b57925a 100644 --- a/internal/backup/types.go +++ b/internal/backup/types.go @@ -31,6 +31,10 @@ type Backup struct { CreatedAt time.Time `json:"created_at"` CompletedAt *time.Time `json:"completed_at,omitempty"` ExpiresAt *time.Time `json:"expires_at,omitempty"` + // Locations lists where this backup exists: "local" and/or remote + // destination names. A backup may live remotely only if local retention + // has pruned the on-disk copy. + Locations []string `json:"locations,omitempty"` } type BackupMetadata struct { diff --git a/internal/credentials/generic.go b/internal/credentials/generic.go new file mode 100644 index 0000000..c63681c --- /dev/null +++ b/internal/credentials/generic.go @@ -0,0 +1,198 @@ +package credentials + +import ( + "fmt" + "os" + "time" + + "github.com/flatrun/agent/pkg/models" + "gopkg.in/yaml.v3" +) + +type genericCredentialsFile struct { + Credentials []*models.Credential `yaml:"credentials"` +} + +var knownCredentialKinds = map[models.CredentialKind]bool{ + models.CredentialKindS3: true, +} + +func (m *Manager) loadGenericCredentials() error { + data, err := os.ReadFile(m.genericCredsFile) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + var cf genericCredentialsFile + if err := yaml.Unmarshal(data, &cf); err != nil { + return err + } + + for _, cred := range cf.Credentials { + if cred.Data == nil { + cred.Data = map[string]string{} + } + m.genericCreds[cred.ID] = cred + } + + return nil +} + +func (m *Manager) saveGenericCredentials() error { + if err := os.MkdirAll(m.storagePath, 0700); err != nil { + return err + } + + creds := make([]*models.Credential, 0, len(m.genericCreds)) + for _, cred := range m.genericCreds { + creds = append(creds, cred) + } + + if len(creds) == 0 { + _ = os.Remove(m.genericCredsFile) + return nil + } + + cf := genericCredentialsFile{Credentials: creds} + data, err := yaml.Marshal(&cf) + if err != nil { + return err + } + + return os.WriteFile(m.genericCredsFile, data, 0600) +} + +// ListGenericCredentials returns generic credentials, optionally filtered by +// kind. Pass an empty kind to list all. +func (m *Manager) ListGenericCredentials(kind models.CredentialKind) []models.Credential { + m.mu.RLock() + defer m.mu.RUnlock() + + result := make([]models.Credential, 0, len(m.genericCreds)) + for _, cred := range m.genericCreds { + if kind != "" && cred.Kind != kind { + continue + } + result = append(result, *cred) + } + return result +} + +// GetGenericCredential returns the raw credential, including secret values, for +// internal use. API handlers marshal it to JSON, which masks secrets. +func (m *Manager) GetGenericCredential(id string) (*models.Credential, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + cred, ok := m.genericCreds[id] + if !ok { + return nil, fmt.Errorf("credential not found: %s", id) + } + clone := *cred + clone.Data = copyData(cred.Data) + return &clone, nil +} + +func (m *Manager) CreateGenericCredential(name string, kind models.CredentialKind, data map[string]string) (*models.Credential, error) { + if name == "" { + return nil, fmt.Errorf("credential name is required") + } + if !knownCredentialKinds[kind] { + return nil, fmt.Errorf("unknown credential kind: %s", kind) + } + + m.mu.Lock() + defer m.mu.Unlock() + + for _, cred := range m.genericCreds { + if cred.Name == name { + return nil, fmt.Errorf("credential with name %s already exists", name) + } + } + + now := time.Now() + cred := &models.Credential{ + ID: generateID(), + Name: name, + Kind: kind, + Data: copyData(data), + CreatedAt: now, + UpdatedAt: now, + } + + m.genericCreds[cred.ID] = cred + + if err := m.saveGenericCredentials(); err != nil { + delete(m.genericCreds, cred.ID) + return nil, err + } + + clone := *cred + clone.Data = copyData(cred.Data) + return &clone, nil +} + +// UpdateGenericCredential merges provided fields. A data value that is empty or +// equal to the mask sentinel leaves the stored value untouched, so a secret can +// be kept without the caller ever having to resend it. +func (m *Manager) UpdateGenericCredential(id, name string, data map[string]string) (*models.Credential, error) { + m.mu.Lock() + defer m.mu.Unlock() + + cred, ok := m.genericCreds[id] + if !ok { + return nil, fmt.Errorf("credential not found: %s", id) + } + + if name != "" && name != cred.Name { + for _, c := range m.genericCreds { + if c.ID != id && c.Name == name { + return nil, fmt.Errorf("credential with name %s already exists", name) + } + } + cred.Name = name + } + + if cred.Data == nil { + cred.Data = map[string]string{} + } + for k, v := range data { + if v == "" || v == models.CredentialMask { + continue + } + cred.Data[k] = v + } + + cred.UpdatedAt = time.Now() + + if err := m.saveGenericCredentials(); err != nil { + return nil, err + } + + clone := *cred + clone.Data = copyData(cred.Data) + return &clone, nil +} + +func (m *Manager) DeleteGenericCredential(id string) error { + m.mu.Lock() + defer m.mu.Unlock() + + if _, ok := m.genericCreds[id]; !ok { + return fmt.Errorf("credential not found: %s", id) + } + + delete(m.genericCreds, id) + return m.saveGenericCredentials() +} + +func copyData(in map[string]string) map[string]string { + out := make(map[string]string, len(in)) + for k, v := range in { + out[k] = v + } + return out +} diff --git a/internal/credentials/generic_test.go b/internal/credentials/generic_test.go new file mode 100644 index 0000000..df6f1f3 --- /dev/null +++ b/internal/credentials/generic_test.go @@ -0,0 +1,118 @@ +package credentials + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/flatrun/agent/pkg/models" +) + +func TestGenericCredential_CreateGetUpdateDelete(t *testing.T) { + m, tmpDir := setupTestManager(t) + defer os.RemoveAll(tmpDir) + + cred, err := m.CreateGenericCredential("prod-s3", models.CredentialKindS3, map[string]string{ + "access_key_id": "AKIAEXAMPLE", + "secret_access_key": "supersecret", + }) + if err != nil { + t.Fatalf("create: %v", err) + } + + got, err := m.GetGenericCredential(cred.ID) + if err != nil { + t.Fatalf("get: %v", err) + } + if got.Data["secret_access_key"] != "supersecret" { + t.Fatalf("expected raw secret from GetGenericCredential, got %q", got.Data["secret_access_key"]) + } + + // A masked or empty secret on update leaves the stored secret untouched, + // while other fields still update. + updated, err := m.UpdateGenericCredential(cred.ID, "prod-s3-renamed", map[string]string{ + "access_key_id": "AKIANEW", + "secret_access_key": models.CredentialMask, + }) + if err != nil { + t.Fatalf("update: %v", err) + } + if updated.Name != "prod-s3-renamed" { + t.Fatalf("expected rename, got %q", updated.Name) + } + if updated.Data["access_key_id"] != "AKIANEW" { + t.Fatalf("expected access key updated, got %q", updated.Data["access_key_id"]) + } + if updated.Data["secret_access_key"] != "supersecret" { + t.Fatalf("expected secret preserved through mask, got %q", updated.Data["secret_access_key"]) + } + + // A real new secret replaces the old one. + rotated, err := m.UpdateGenericCredential(cred.ID, "", map[string]string{"secret_access_key": "rotated"}) + if err != nil { + t.Fatalf("rotate: %v", err) + } + if rotated.Data["secret_access_key"] != "rotated" { + t.Fatalf("expected rotated secret, got %q", rotated.Data["secret_access_key"]) + } + + if err := m.DeleteGenericCredential(cred.ID); err != nil { + t.Fatalf("delete: %v", err) + } + if _, err := m.GetGenericCredential(cred.ID); err == nil { + t.Fatal("expected credential gone after delete") + } +} + +func TestGenericCredential_RejectsUnknownKind(t *testing.T) { + m, tmpDir := setupTestManager(t) + defer os.RemoveAll(tmpDir) + + if _, err := m.CreateGenericCredential("x", models.CredentialKind("ftp"), nil); err == nil { + t.Fatal("expected unknown kind to be rejected") + } +} + +func TestGenericCredential_PersistsAcrossReload(t *testing.T) { + m, tmpDir := setupTestManager(t) + defer os.RemoveAll(tmpDir) + + if _, err := m.CreateGenericCredential("prod", models.CredentialKindS3, map[string]string{ + "access_key_id": "AKIA", + "secret_access_key": "s3cr3t", + }); err != nil { + t.Fatalf("create: %v", err) + } + + reloaded := NewManager(tmpDir) + list := reloaded.ListGenericCredentials(models.CredentialKindS3) + if len(list) != 1 || list[0].Name != "prod" { + t.Fatalf("expected persisted credential after reload, got %#v", list) + } +} + +func TestGenericCredential_JSONMasksSecret(t *testing.T) { + cred := models.Credential{ + Name: "prod", + Kind: models.CredentialKindS3, + Data: map[string]string{ + "access_key_id": "AKIAVISIBLE", + "secret_access_key": "hidden", + }, + } + raw, err := json.Marshal(cred) + if err != nil { + t.Fatalf("marshal: %v", err) + } + s := string(raw) + if strings.Contains(s, "hidden") { + t.Fatalf("secret leaked into JSON: %s", s) + } + if !strings.Contains(s, "AKIAVISIBLE") { + t.Fatalf("access key id should be visible: %s", s) + } + if !strings.Contains(s, models.CredentialMask) { + t.Fatalf("expected mask sentinel in JSON: %s", s) + } +} diff --git a/internal/credentials/manager.go b/internal/credentials/manager.go index 1ad81c8..959836a 100644 --- a/internal/credentials/manager.go +++ b/internal/credentials/manager.go @@ -15,30 +15,36 @@ import ( ) type Manager struct { - mu sync.RWMutex - registryTypes map[string]*models.RegistryType - credentials map[string]*models.RegistryCredential - storagePath string - typesFilePath string - credsFilePath string + mu sync.RWMutex + registryTypes map[string]*models.RegistryType + credentials map[string]*models.RegistryCredential + genericCreds map[string]*models.Credential + storagePath string + typesFilePath string + credsFilePath string + genericCredsFile string } func NewManager(deploymentsPath string) *Manager { storagePath := filepath.Join(deploymentsPath, ".flatrun") typesFilePath := filepath.Join(storagePath, "registry-types.yml") credsFilePath := filepath.Join(storagePath, "credentials.yml") + genericCredsFile := filepath.Join(storagePath, "credentials-store.yml") m := &Manager{ - registryTypes: make(map[string]*models.RegistryType), - credentials: make(map[string]*models.RegistryCredential), - storagePath: storagePath, - typesFilePath: typesFilePath, - credsFilePath: credsFilePath, + registryTypes: make(map[string]*models.RegistryType), + credentials: make(map[string]*models.RegistryCredential), + genericCreds: make(map[string]*models.Credential), + storagePath: storagePath, + typesFilePath: typesFilePath, + credsFilePath: credsFilePath, + genericCredsFile: genericCredsFile, } m.initBuiltinTypes() _ = m.loadTypes() _ = m.loadCredentials() + _ = m.loadGenericCredentials() return m } diff --git a/pkg/config/backup_test.go b/pkg/config/backup_test.go new file mode 100644 index 0000000..730dfa4 --- /dev/null +++ b/pkg/config/backup_test.go @@ -0,0 +1,38 @@ +package config + +import "testing" + +func TestBackupDestination_IsEnabledDefaultsOn(t *testing.T) { + if !(BackupDestination{}).IsEnabled() { + t.Fatal("expected nil Enabled to default to on") + } + off := false + if (BackupDestination{Enabled: &off}).IsEnabled() { + t.Fatal("expected explicit false to disable") + } +} + +func TestBackupDestinations_RoundTripThroughRegistry(t *testing.T) { + cfg := &Config{} + setDefaults(cfg) + + dests := []BackupDestination{{ + Name: "s3-prod", + Type: "s3", + Endpoint: "https://s3.example.com", + Bucket: "flatrun-backups", + CredentialID: "abc123", + }} + + if err := Set(cfg, "backup.destinations", dests); err != nil { + t.Fatalf("set destinations: %v", err) + } + + if len(cfg.Backup.Destinations) != 1 || cfg.Backup.Destinations[0].Bucket != "flatrun-backups" { + t.Fatalf("destinations not applied: %#v", cfg.Backup.Destinations) + } + + if _, err := Get(cfg, "backup.destinations"); err != nil { + t.Fatalf("get destinations: %v", err) + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index cc956ff..c73955f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -42,6 +42,7 @@ type Config struct { Plans PlansConfig `yaml:"plans"` AI AIConfig `yaml:"ai"` Files FilesConfig `yaml:"files"` + Backup BackupConfig `yaml:"backup"` } type FilesConfig struct { @@ -49,6 +50,30 @@ type FilesConfig struct { ShowHidden *bool `yaml:"show_hidden" json:"show_hidden"` } +type BackupConfig struct { + Destinations []BackupDestination `yaml:"destinations" json:"destinations"` +} + +// BackupDestination is a remote object-storage target that backups are mirrored +// to after the local copy is written. Secrets are never stored here: CredentialID +// references an S3 credential held by the credential manager. +type BackupDestination struct { + Name string `yaml:"name" json:"name"` + Type string `yaml:"type" json:"type"` + Endpoint string `yaml:"endpoint" json:"endpoint"` + Region string `yaml:"region" json:"region"` + Bucket string `yaml:"bucket" json:"bucket"` + Prefix string `yaml:"prefix" json:"prefix"` + CredentialID string `yaml:"credential_id" json:"credential_id"` + UsePathStyle bool `yaml:"use_path_style" json:"use_path_style"` + // Pointer so an explicit false survives reloads; nil means enabled. + Enabled *bool `yaml:"enabled" json:"enabled"` +} + +func (d BackupDestination) IsEnabled() bool { + return d.Enabled == nil || *d.Enabled +} + type AIConfig struct { Enabled bool `yaml:"enabled" json:"enabled"` BaseURL string `yaml:"base_url" json:"base_url"` diff --git a/pkg/models/credential.go b/pkg/models/credential.go new file mode 100644 index 0000000..5a427a0 --- /dev/null +++ b/pkg/models/credential.go @@ -0,0 +1,57 @@ +package models + +import ( + "encoding/json" + "time" +) + +type CredentialKind string + +const ( + CredentialKindS3 CredentialKind = "s3" +) + +// Credential is a generic, kind-tagged secret held by the credential manager, +// separate from the registry-specific RegistryCredential. Secret-bearing keys +// in Data are masked when marshaled to JSON but written verbatim to the 0600 +// on-disk store. +type Credential struct { + ID string `json:"id" yaml:"id"` + Name string `json:"name" yaml:"name"` + Kind CredentialKind `json:"kind" yaml:"kind"` + Data map[string]string `json:"data" yaml:"data"` + CreatedAt time.Time `json:"created_at" yaml:"created_at"` + UpdatedAt time.Time `json:"updated_at" yaml:"updated_at"` +} + +// CredentialMask is returned in place of a secret value so callers can tell a +// secret is set without receiving it. A value equal to the mask on update is +// treated as "unchanged". +const CredentialMask = "********" + +// credentialSecretKeys lists the Data keys whose values must never leave the +// agent in a JSON response, per credential kind. +var credentialSecretKeys = map[CredentialKind]map[string]bool{ + CredentialKindS3: {"secret_access_key": true}, +} + +// IsSecretKey reports whether a Data key holds a secret for the given kind. +func IsSecretKey(kind CredentialKind, key string) bool { + return credentialSecretKeys[kind][key] +} + +func (c Credential) MarshalJSON() ([]byte, error) { + type alias Credential + secrets := credentialSecretKeys[c.Kind] + masked := make(map[string]string, len(c.Data)) + for k, v := range c.Data { + if secrets[k] && v != "" { + masked[k] = CredentialMask + continue + } + masked[k] = v + } + clone := alias(c) + clone.Data = masked + return json.Marshal(clone) +} From 03c81925c0ac39d9de8923ce58eb49bde43cbf66 Mon Sep 17 00:00:00 2001 From: nfebe Date: Wed, 8 Jul 2026 23:04:06 +0100 Subject: [PATCH 2/5] feat(templates): Add MinIO template and object store spec Introduces a MinIO app template so an S3-compatible object store can be deployed and run by FlatRun itself, with a per-deployment root credential and its data in a flat-file mount. Adds a spec defining the object store abstraction: one concept with two kinds, external (a store FlatRun only connects to) and managed (a store FlatRun runs from a template), which backup destinations fold into. Later slices cover an object browser, deployment consumption, and replication. --- docs/OBJECT_STORES.md | 96 ++++++++++++++++++++++++++++++ templates/minio/docker-compose.yml | 21 +++++++ templates/minio/metadata.yml | 23 +++++++ 3 files changed, 140 insertions(+) create mode 100644 docs/OBJECT_STORES.md create mode 100644 templates/minio/docker-compose.yml create mode 100644 templates/minio/metadata.yml diff --git a/docs/OBJECT_STORES.md b/docs/OBJECT_STORES.md new file mode 100644 index 0000000..ec4ff95 --- /dev/null +++ b/docs/OBJECT_STORES.md @@ -0,0 +1,96 @@ +# Object Stores + +## What this is + +An **object store** is a named, S3-compatible endpoint FlatRun knows about. One +abstraction covers every store, whoever runs it, so backups, deployments, and the +UI treat them all the same: each resolves to `(endpoint, region, bucket, credential)` +and is reached through one S3 client. + +This supersedes the earlier standalone "backup destination": a backup destination +is now just an object store used as a backup target. + +## Two kinds + +A store's `kind` records **who runs the store**, nothing else: + +- **external**: a store running somewhere else that FlatRun only connects to. You + provide an endpoint and credentials. FlatRun does not manage its lifecycle. + Examples: AWS S3, Cloudflare R2, Backblaze B2, an existing MinIO. +- **managed**: a store FlatRun runs itself, deployed from a template as a container + on the host (MinIO, SeaweedFS, Garage). Because it is a normal FlatRun deployment, + FlatRun knows its endpoint, issues its credentials, and can start, stop, and back + it up like any other deployment. Its data lives in a flat-file bind mount. + +Everything downstream (browsing objects, using a store as a backup target, mounting +it into a deployment) is identical across kinds. + +## Data model + +``` +ObjectStore + id string + name string + kind "external" | "managed" + endpoint string # empty means AWS default + region string + bucket string + prefix string # optional key prefix + credential_id string # references an s3 credential in the credential manager + use_path_style bool + deployment string # managed only: the deployment that runs the container + enabled *bool # nil means enabled + backup_target bool # whether backups mirror to this store +``` + +Secrets never live on the store record. `credential_id` points at an S3 credential +held by the credential manager (written 0600, masked in API responses). For a managed +store, the credential is the one issued to its container at deploy time. + +## How backups use it + +The backup manager mirrors each new archive to every enabled store with +`backup_target = true`, on top of the always-local copy. List, download, and restore +fall back to a store when the local archive has been pruned. This is the existing +mirror mechanism; only the source of the target list changes from "destinations" to +"object stores marked as backup targets". + +## Managed stores via templates + +A managed store is created through the normal deploy flow using an object-store +template. A template is a directory under `agent/templates//` with +`metadata.yml` and `docker-compose.yml`, auto-discovered from the embedded set. + +Deploying one: +1. User picks an object-store template (MinIO, SeaweedFS) and deploys it. The + template generates a root credential per deployment. +2. FlatRun registers a `managed` object store linked to that deployment, deriving + the endpoint from the container and storing the issued credential. +3. The store then appears alongside external stores and can be used as a backup + target or, later, mounted into other deployments. + +Data stays in the deployment's flat-file `./data` mount, consistent with the +no-hidden-volumes philosophy. + +## Slices + +1. **Abstraction + managed template** (this cycle): the store model with `kind`, + a MinIO template (SeaweedFS and Garage to follow), and folding backup + destinations into stores. The Object Stores UI lists external and managed stores + and can add an external store or deploy a managed one. +2. **Object browser**: list and manage buckets and objects in a store from the UI + (upload, download, delete), the "visualization" half of the feature. +3. **Deployment consumption**: inject a store's endpoint and issued credentials into + another deployment's environment so apps can use it directly. +4. **Replication**: sync one store to another (managed to external for offsite, or + external to managed for local cache), on a schedule. + +## API sketch + +- `GET /object-stores` list stores (external and managed), non-secret. +- `POST /object-stores` register an external store. +- `PUT /object-stores/:id`, `DELETE /object-stores/:id`. +- `POST /object-stores/:id/test` connectivity probe (write and delete). +- `POST /object-stores/managed` deploy a managed store from a template, returning the + new store plus the deploy job. +- S3 credentials continue through `/storage-credentials`. diff --git a/templates/minio/docker-compose.yml b/templates/minio/docker-compose.yml new file mode 100644 index 0000000..22d15ab --- /dev/null +++ b/templates/minio/docker-compose.yml @@ -0,0 +1,21 @@ +name: ${NAME} +services: + minio: + image: minio/minio:latest + container_name: ${NAME} + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-flatrun} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-changeme} + volumes: + - ./data:/data + expose: + - "9000" + - "9001" + networks: + - proxy + restart: unless-stopped + +networks: + proxy: + external: true diff --git a/templates/minio/metadata.yml b/templates/minio/metadata.yml new file mode 100644 index 0000000..f1fb4f0 --- /dev/null +++ b/templates/minio/metadata.yml @@ -0,0 +1,23 @@ +name: MinIO +description: S3-compatible object storage you host yourself +icon: pi pi-database +logo: https://cdn.simpleicons.org/minio +category: infrastructure +priority: 68 +container_port: 9000 +mounts: + - id: data + name: Data + container_path: /data + description: Buckets and objects + type: file + required: true +env: + file: .env + template: | + MINIO_ROOT_USER=flatrun + MINIO_ROOT_PASSWORD= + secrets: + - key: MINIO_ROOT_PASSWORD + encoding: alphanumeric + bytes: 24 From 5e7d5c9905742578a3d958be1b7175a93ccfa6c3 Mon Sep 17 00:00:00 2001 From: nfebe Date: Wed, 8 Jul 2026 23:04:44 +0100 Subject: [PATCH 3/5] refactor(deps): Promote aws-sdk s3 to a direct dependency --- go.mod | 4 ++-- go.sum | 12 ------------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 61d6ad3..5fa613e 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,11 @@ module github.com/flatrun/agent go 1.25.0 require ( + github.com/aws/aws-sdk-go-v2 v1.42.1 github.com/aws/aws-sdk-go-v2/config v1.32.6 github.com/aws/aws-sdk-go-v2/credentials v1.19.6 github.com/aws/aws-sdk-go-v2/service/route53 v1.62.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0 github.com/cloudflare/cloudflare-go v0.116.0 github.com/compose-spec/compose-go/v2 v2.10.1 github.com/creack/pty v1.1.24 @@ -39,7 +41,6 @@ require ( github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/aws/aws-sdk-go-v2 v1.42.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect @@ -50,7 +51,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.105.0 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect diff --git a/go.sum b/go.sum index 2a0dc8e..fc5541c 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,6 @@ github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9 github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4= -github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek= github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY= @@ -34,26 +32,18 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRt github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 h1:3GUprIsfmGcC5SACIyB0e7E0BM1O1b3Erl5CePYIAeQ= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31/go.mod h1:7PuV1yl5e2xnUbm+RqvVg5i2iBM8EyijZNoI9wsOoOc= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU= @@ -70,8 +60,6 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlV github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0= github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70= github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk= -github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= -github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= From 57c06000c7eca9896e58065e7dd6b4ce0bacb17b Mon Sep 17 00:00:00 2001 From: nfebe Date: Wed, 8 Jul 2026 23:06:58 +0100 Subject: [PATCH 4/5] feat(backup): Record object store kind (external vs managed) --- config.example.yml | 1 + pkg/config/config.go | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/config.example.yml b/config.example.yml index a0a679a..2d4acd3 100644 --- a/config.example.yml +++ b/config.example.yml @@ -134,6 +134,7 @@ backup: destinations: [] # - name: s3-prod # type: s3 + # kind: external # external (connect only) or managed (FlatRun runs it) # endpoint: https://s3.us-east-1.amazonaws.com # region: us-east-1 # bucket: flatrun-backups diff --git a/pkg/config/config.go b/pkg/config/config.go index c73955f..d6b189e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -54,12 +54,19 @@ type BackupConfig struct { Destinations []BackupDestination `yaml:"destinations" json:"destinations"` } -// BackupDestination is a remote object-storage target that backups are mirrored -// to after the local copy is written. Secrets are never stored here: CredentialID -// references an S3 credential held by the credential manager. +// BackupDestination is an object store that backups are mirrored to after the +// local copy is written. Secrets are never stored here: CredentialID references +// an S3 credential held by the credential manager. +// +// Kind records who runs the store: "external" (a store FlatRun only connects to) +// or "managed" (a store FlatRun runs itself, deployed from a template). For a +// managed store, Deployment names the deployment that runs the container. See +// docs/OBJECT_STORES.md. type BackupDestination struct { Name string `yaml:"name" json:"name"` Type string `yaml:"type" json:"type"` + Kind string `yaml:"kind" json:"kind"` + Deployment string `yaml:"deployment,omitempty" json:"deployment,omitempty"` Endpoint string `yaml:"endpoint" json:"endpoint"` Region string `yaml:"region" json:"region"` Bucket string `yaml:"bucket" json:"bucket"` @@ -74,6 +81,14 @@ func (d BackupDestination) IsEnabled() bool { return d.Enabled == nil || *d.Enabled } +// StoreKind returns the store kind, defaulting to "external" when unset. +func (d BackupDestination) StoreKind() string { + if d.Kind == "" { + return "external" + } + return d.Kind +} + type AIConfig struct { Enabled bool `yaml:"enabled" json:"enabled"` BaseURL string `yaml:"base_url" json:"base_url"` From 0e51f1ae42fa2a55a5a58e2872a4520b8972ca99 Mon Sep 17 00:00:00 2001 From: nfebe Date: Wed, 8 Jul 2026 23:54:48 +0100 Subject: [PATCH 5/5] test(api): Cover object store credential and destination endpoints --- internal/api/object_stores_test.go | 201 +++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 internal/api/object_stores_test.go diff --git a/internal/api/object_stores_test.go b/internal/api/object_stores_test.go new file mode 100644 index 0000000..19b9906 --- /dev/null +++ b/internal/api/object_stores_test.go @@ -0,0 +1,201 @@ +package api + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/flatrun/agent/internal/auth" + "github.com/flatrun/agent/internal/backup" + "github.com/flatrun/agent/internal/credentials" + "github.com/flatrun/agent/pkg/config" + "github.com/flatrun/agent/pkg/models" + "github.com/gin-gonic/gin" +) + +func setupObjectStoreTestServer(t *testing.T) (*Server, *gin.Engine, func()) { + gin.SetMode(gin.TestMode) + + tmpDir, err := os.MkdirTemp("", "objstore_test") + if err != nil { + t.Fatalf("temp dir: %v", err) + } + + cfg := &config.Config{ + DeploymentsPath: tmpDir, + Auth: config.AuthConfig{ + Enabled: true, + JWTSecret: "test-jwt-secret-key-for-testing", + }, + } + os.Setenv("FLATRUN_ADMIN_PASSWORD", "testadminpass") + + authManager, err := auth.NewManager(tmpDir, &cfg.Auth, true) + if err != nil { + os.RemoveAll(tmpDir) + t.Fatalf("auth manager: %v", err) + } + + backupManager, err := backup.NewManager(tmpDir) + if err != nil { + authManager.Close() + os.RemoveAll(tmpDir) + t.Fatalf("backup manager: %v", err) + } + + server := &Server{ + config: cfg, + authManager: authManager, + credentialsManager: credentials.NewManager(tmpDir), + backupManager: backupManager, + } + + router := gin.New() + mw := auth.NewMiddlewareWithManager(&cfg.Auth, authManager) + api := router.Group("/api") + api.POST("/auth/login", mw.Login) + + protected := api.Group("") + protected.Use(mw.RequireAuth()) + protected.GET("/storage-credentials", mw.RequirePermission(auth.PermBackupsRead), server.listStorageCredentials) + protected.POST("/storage-credentials", mw.RequirePermission(auth.PermBackupsWrite), server.createStorageCredential) + protected.PUT("/storage-credentials/:id", mw.RequirePermission(auth.PermBackupsWrite), server.updateStorageCredential) + protected.DELETE("/storage-credentials/:id", mw.RequirePermission(auth.PermBackupsDelete), server.deleteStorageCredential) + protected.GET("/backup-destinations", mw.RequirePermission(auth.PermBackupsRead), server.listBackupDestinations) + + cleanup := func() { + authManager.Close() + os.RemoveAll(tmpDir) + os.Unsetenv("FLATRUN_ADMIN_PASSWORD") + } + return server, router, cleanup +} + +func objStoreLogin(t *testing.T, router *gin.Engine) string { + body, _ := json.Marshal(map[string]string{"username": "admin", "password": "testadminpass"}) + req := httptest.NewRequest(http.MethodPost, "/api/auth/login", bytes.NewBuffer(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("login failed: %d %s", w.Code, w.Body.String()) + } + var resp map[string]any + _ = json.Unmarshal(w.Body.Bytes(), &resp) + return resp["token"].(string) +} + +func osReq(t *testing.T, router *gin.Engine, method, path, token string, body any) *httptest.ResponseRecorder { + var buf bytes.Buffer + if body != nil { + _ = json.NewEncoder(&buf).Encode(body) + } + req := httptest.NewRequest(method, path, &buf) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + return w +} + +func TestStorageCredential_CreateListMaskDelete(t *testing.T) { + _, router, cleanup := setupObjectStoreTestServer(t) + defer cleanup() + token := objStoreLogin(t, router) + + created := osReq(t, router, http.MethodPost, "/api/storage-credentials", token, map[string]any{ + "name": "prod-r2", + "kind": "s3", + "data": map[string]string{"access_key_id": "AKIATESTVALUE", "secret_access_key": "topsecret"}, + }) + if created.Code != http.StatusCreated { + t.Fatalf("create: %d %s", created.Code, created.Body.String()) + } + var createdResp struct { + Credential models.Credential `json:"credential"` + } + _ = json.Unmarshal(created.Body.Bytes(), &createdResp) + id := createdResp.Credential.ID + if id == "" { + t.Fatal("expected credential id") + } + + list := osReq(t, router, http.MethodGet, "/api/storage-credentials?kind=s3", token, nil) + if list.Code != http.StatusOK { + t.Fatalf("list: %d", list.Code) + } + body := list.Body.String() + if bytes.Contains([]byte(body), []byte("topsecret")) { + t.Fatalf("secret leaked through the API: %s", body) + } + if !bytes.Contains([]byte(body), []byte(models.CredentialMask)) { + t.Fatalf("expected masked secret in response: %s", body) + } + if !bytes.Contains([]byte(body), []byte("AKIATESTVALUE")) { + t.Fatalf("access key id should be visible: %s", body) + } + + upd := osReq(t, router, http.MethodPut, "/api/storage-credentials/"+id, token, map[string]any{ + "name": "prod-r2-renamed", + "data": map[string]string{"secret_access_key": models.CredentialMask}, + }) + if upd.Code != http.StatusOK { + t.Fatalf("update: %d %s", upd.Code, upd.Body.String()) + } + + del := osReq(t, router, http.MethodDelete, "/api/storage-credentials/"+id, token, nil) + if del.Code != http.StatusOK { + t.Fatalf("delete: %d %s", del.Code, del.Body.String()) + } +} + +func TestStorageCredential_DeleteInUseConflicts(t *testing.T) { + server, router, cleanup := setupObjectStoreTestServer(t) + defer cleanup() + token := objStoreLogin(t, router) + + created := osReq(t, router, http.MethodPost, "/api/storage-credentials", token, map[string]any{ + "name": "in-use", + "kind": "s3", + "data": map[string]string{"access_key_id": "AKIATESTVALUE", "secret_access_key": "topsecret"}, + }) + var createdResp struct { + Credential models.Credential `json:"credential"` + } + _ = json.Unmarshal(created.Body.Bytes(), &createdResp) + id := createdResp.Credential.ID + + server.config.Backup.Destinations = []config.BackupDestination{ + {Name: "s3-prod", Type: "s3", Bucket: "b", CredentialID: id}, + } + + del := osReq(t, router, http.MethodDelete, "/api/storage-credentials/"+id, token, nil) + if del.Code != http.StatusConflict { + t.Fatalf("expected 409 for in-use credential, got %d %s", del.Code, del.Body.String()) + } +} + +func TestBackupDestinations_ListReportsKind(t *testing.T) { + server, router, cleanup := setupObjectStoreTestServer(t) + defer cleanup() + token := objStoreLogin(t, router) + + server.config.Backup.Destinations = []config.BackupDestination{ + {Name: "s3-prod", Type: "s3", Kind: "external", Bucket: "flatrun-backups", CredentialID: "abc"}, + } + + list := osReq(t, router, http.MethodGet, "/api/backup-destinations", token, nil) + if list.Code != http.StatusOK { + t.Fatalf("list: %d %s", list.Code, list.Body.String()) + } + var resp struct { + Destinations []config.BackupDestination `json:"destinations"` + } + _ = json.Unmarshal(list.Body.Bytes(), &resp) + if len(resp.Destinations) != 1 || resp.Destinations[0].Bucket != "flatrun-backups" || resp.Destinations[0].Kind != "external" { + t.Fatalf("unexpected destinations: %#v", resp.Destinations) + } +}