Use S3Proxy to access object storage so we can swap out cloud storage engines

charts/massdriver/templates/_helpers.tpl

@@ -126,12 +126,4 @@ so we have to handle the "double" base64 encoding gracefully
 
 {{- define "massdriver.phxSigningSalt" -}}
   {{- include "massdriver.getValueFromSecret" (dict "Namespace" .Release.Namespace "Name" (printf "%s-massdriver-envs" (include "massdriver.fullname" .)) "Length" 20 "Key" "PHX_SIGNING_SALT") }}
-{{- end -}}
-
-{{- define "massdriver.minio.password" -}}
-  {{ include "massdriver.getValueFromSecret" (dict "Namespace" .Release.Namespace "Name" (printf "%s-minio" (include "massdriver.fullname" .)) "Length" 40 "Key" "rootPassword") }}
-{{- end }}
-
-{{- define "massdriver.minio.username" -}}
-  {{ include "massdriver.getValueFromSecret" (dict "Namespace" .Release.Namespace "Name" (printf "%s-minio" (include "massdriver.fullname" .)) "Length" 20 "Key" "rootUser") }}
-{{- end }}
+{{- end -}}

charts/massdriver/templates/massdriver/configmap-envs.yaml

@@ -6,16 +6,9 @@ metadata:
     {{- include "massdriver.labels" . | nindent 4 }}
     app.kubernetes.io/component: massdriver
 data:
-  {{- if eq .Values.massdriver.blobStorage.type "minio" }}
-  BLOB_STORAGE_HOST: "{{ include "massdriver.fullname" . }}-minio.{{ .Release.Namespace }}.svc"
-  BLOB_STORAGE_PORT: "{{ toString .Values.minio.service.port }}"
+  BLOB_STORAGE_HOST: "{{ include "massdriver.fullname" . }}-s3proxy.{{ .Release.Namespace }}.svc"
+  BLOB_STORAGE_PORT: "80"
   BLOB_STORAGE_SCHEME: http
-  {{- else if eq .Values.massdriver.blobStorage.type "s3" }}
-  AWS_REGION: {{ .Values.massdriver.blobStorage.s3.region }}
-  BLOB_STORAGE_HOST: "{{ printf "s3.%s.amazonaws.com" .Values.massdriver.blobStorage.s3.region }}"
-  BLOB_STORAGE_PORT: "443"
-  BLOB_STORAGE_SCHEME: "https"
-  {{- end }}
   DATABASE_SSL: "true"
   FORCE_V2_LOGGING: "true"
   LOG_LEVEL: {{ .Values.massdriver.logLevel | quote }}

charts/massdriver/templates/massdriver/deployment.yaml

@@ -20,7 +20,6 @@ spec:
         configmap.massdriver-envs/checksum: {{ include (print $.Template.BasePath "/massdriver/configmap-envs.yaml") . | sha256sum }}
         configmap.ui-envs/checksum: {{ include (print $.Template.BasePath "/massdriver/configmap-ui.yaml") . | sha256sum }}
         secret.massdriver-envs/checksum: {{ include (print $.Template.BasePath "/massdriver/secret-envs.yaml") . | sha256sum }}
-        secret.minio/checksum: {{ include (print $.Template.BasePath "/massdriver/secret-minio.yaml") . | sha256sum }}
       labels:
         {{- include "massdriver.labels" . | nindent 8 }}
         {{- with .Values.massdriver.podLabels }}
@@ -49,10 +48,6 @@ spec:
                 name: {{ include "massdriver.fullname" . }}-massdriver-envs
             - secretRef:
                 name: {{ include "massdriver.fullname" . }}-massdriver-envs
-            {{- if and .Values.minio.enabled (eq .Values.massdriver.blobStorage.type "minio") }}
-            - secretRef:
-                name: {{ include "massdriver.fullname" . }}-massdriver-minio-auth
-            {{- end }}
           ports:
             - name: http
               containerPort: {{ .Values.massdriver.port }}

charts/massdriver/templates/massdriver/job-db-migration.yaml

@@ -37,10 +37,6 @@ spec:
                 name: {{ include "massdriver.fullname" . }}-massdriver-envs
             - secretRef:
                 name: {{ include "massdriver.fullname" . }}-massdriver-envs
-            {{- if and .Values.minio.enabled (eq .Values.massdriver.blobStorage.type "minio") }}
-            - secretRef:
-                name: {{ include "massdriver.fullname" . }}-massdriver-minio-auth
-            {{- end }}
           args: 
             - eval
             - Massdriver.Release.migrate

charts/massdriver/templates/massdriver/secret-envs.yaml

@@ -6,10 +6,9 @@ metadata:
     {{- include "massdriver.labels" . | nindent 4 }}
     app.kubernetes.io/component: massdriver
 data:
-  {{- if and (eq .Values.massdriver.blobStorage.type "s3") (not (empty .Values.massdriver.blobStorage.s3.accessKeyId)) (not (empty .Values.massdriver.blobStorage.s3.secretAccessKey)) }}
-  AWS_ACCESS_KEY_ID: {{ .Values.massdriver.blobStorage.s3.accessKeyId | b64enc | quote }}
-  AWS_SECRET_ACCESS_KEY: {{ .Values.massdriver.blobStorage.s3.secretAccessKey | b64enc | quote }}
-  {{- end }}
+  AWS_ACCESS_KEY_ID: {{ .Values.massdriver.blobStorage.username | b64enc | quote }}
+  AWS_SECRET_ACCESS_KEY: {{ .Values.massdriver.blobStorage.password | b64enc | quote }}
+  AWS_REGION: {{ "us-east-1" | b64enc | quote }}
   JWT_SECRET: {{ include "massdriver.jwtSecret" . | b64enc | quote }}
   LICENSE_KEY: {{ .Values.licenseKey | b64enc | quote }}
   MD_CLOAK_KEY: {{ include "massdriver.cloakKey" . | b64enc | quote }}

charts/massdriver/templates/massdriver/secret-minio.yaml

@@ -1,6 +1,4 @@
 {{- if .Values.minio.enabled -}}
-{{- $minioUser := include "massdriver.minio.username" . }}
-{{- $minioPassword := include "massdriver.minio.password" . }}
 apiVersion: v1
 kind: Secret
 metadata: 
@@ -9,20 +7,6 @@ metadata:
     {{- include "massdriver.labels" . | nindent 4 }}
     app.kubernetes.io/component: minio
 data:
-  rootUser: {{ $minioUser | b64enc | quote }}
-  rootPassword: {{ $minioPassword | b64enc | quote }}
----
-# this secret is used by massdriver to authenticate with minio - it is here so that the secret generator will be consistent on install
-{{- if eq .Values.massdriver.blobStorage.type "minio" }}
-apiVersion: v1
-kind: Secret
-metadata: 
-  name: {{ include "massdriver.fullname" . }}-massdriver-minio-auth
-  labels:
-    {{- include "massdriver.labels" . | nindent 4 }}
-    app.kubernetes.io/component: minio
-data:
-  AWS_ACCESS_KEY_ID: {{ $minioUser | b64enc | quote }}
-  AWS_SECRET_ACCESS_KEY: {{ $minioPassword | b64enc | quote }}
-{{- end }}
-{{- end }}
+  rootUser: {{ .Values.massdriver.blobStorage.minio.username | b64enc | quote }}
+  rootPassword: {{ .Values.massdriver.blobStorage.minio.password | b64enc | quote }}
+{{- end -}}

charts/massdriver/templates/s3proxy/deployment.yaml

@@ -0,0 +1,63 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "massdriver.fullname" . }}-s3proxy
+  labels:
+    {{- include "massdriver.labels" . | nindent 4 }}
+    app.kubernetes.io/component: s3proxy
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      {{- include "massdriver.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: s3proxy
+  template:
+    metadata:
+      annotations:
+        secret.s3proxy-config/checksum: {{ include (print $.Template.BasePath "/s3proxy/secret-envs.yaml") . | sha256sum }}
+      labels:
+        {{- include "massdriver.labels" . | nindent 8 }}
+        app.kubernetes.io/component: s3proxy
+    spec:
+      serviceAccountName: {{ include "massdriver.fullname" . }}-s3proxy
+      containers:
+        - name: s3proxy
+          image: "andrewgaul/s3proxy:3.0.0"
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 8080
+              protocol: TCP
+          livenessProbe:
+            tcpSocket:
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            tcpSocket:
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 3
+          resources:
+            limits:
+              cpu: 500m
+              memory: 512Mi
+            requests:
+              cpu: 100m
+              memory: 128Mi
+          envFrom:
+            - secretRef:
+                name: {{ include "massdriver.fullname" . }}-s3proxy-envs
+          securityContext:
+            capabilities:
+              drop:
+              - ALL
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 65534
+      securityContext:
+        fsGroup: 65534

charts/massdriver/templates/s3proxy/secret-envs.yaml

@@ -0,0 +1,62 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "massdriver.fullname" . }}-s3proxy-envs
+  labels:
+    {{- include "massdriver.labels" . | nindent 4 }}
+    app.kubernetes.io/component: s3proxy
+type: Opaque
+stringData:
+  # S3Proxy configuration
+  S3PROXY_ENDPOINT: "http://0.0.0.0:8080"
+  S3PROXY_AUTHORIZATION: "aws-v2-or-v4"
+  S3PROXY_IDENTITY: {{ .Values.massdriver.blobStorage.username | quote }}
+  S3PROXY_CREDENTIAL: {{ .Values.massdriver.blobStorage.password | quote }}
+  S3PROXY_VIRTUALHOST: ""
+  S3PROXY_IGNORE_UNKNOWN_HEADERS: "true"
+  S3PROXY_CORS_ALLOW_ALL: "false"
+
+  # Logging configuration for debugging
+  LOG_LEVEL: {{ .Values.massdriver.blobStorage.logLevel | quote }}
+  JETTY_LOG_LEVEL: {{ .Values.massdriver.blobStorage.logLevel | quote }}
+
+  # JClouds HTTP client performance optimizations (applies to all providers)
+  JCLOUDS_CONNECTION_TIMEOUT: "5000"
+  JCLOUDS_SO_TIMEOUT: "10000"
+  JCLOUDS_MAX_CONNECTIONS: "200"
+  JCLOUDS_MAX_CONNECTIONS_PER_ROUTE: "20"
+
+  {{- if eq .Values.massdriver.blobStorage.type "minio" }}
+  # MinIO backend configuration (using S3 provider)
+  JCLOUDS_PROVIDER: "aws-s3"
+  JCLOUDS_IDENTITY: {{ .Values.massdriver.blobStorage.minio.username | quote }}
+  JCLOUDS_CREDENTIAL: {{ .Values.massdriver.blobStorage.minio.password | quote }}
+  JCLOUDS_ENDPOINT: "http://{{ include "massdriver.fullname" . }}-minio.{{ .Release.Namespace }}.svc:{{ toString .Values.minio.service.port }}"
+  {{- else if eq .Values.massdriver.blobStorage.type "s3" }}
+  # AWS S3 Configuration
+  JCLOUDS_PROVIDER: "aws-s3"
+  JCLOUDS_IDENTITY: {{ .Values.massdriver.blobStorage.s3.accessKeyId | quote }}
+  JCLOUDS_CREDENTIAL: {{ .Values.massdriver.blobStorage.s3.secretAccessKey | quote }}
+  JCLOUDS_ENDPOINT: "{{ printf "https://s3.%s.amazonaws.com" .Values.massdriver.blobStorage.s3.region }}"
+  JCLOUDS_REGION: {{ .Values.massdriver.blobStorage.s3.region | quote }}
+  {{- else if eq .Values.massdriver.blobStorage.type "gcs" }}
+  # Google Cloud Storage Configuration
+  JCLOUDS_PROVIDER: "google-cloud-storage"
+  JCLOUDS_IDENTITY: {{ .Values.massdriver.blobStorage.gcs.serviceAccountEmail | quote }}
+  JCLOUDS_CREDENTIAL: {{ .Values.massdriver.blobStorage.gcs.privateKey | quote }}
+  JCLOUDS_ENDPOINT: "https://storage.googleapis.com"
+  {{- else if eq .Values.massdriver.blobStorage.type "azureblob" }}
+  # Azure Blob Storage Configuration
+  JCLOUDS_PROVIDER: "azureblob-sdk"
+  JCLOUDS_ENDPOINT: "https://{{ .Values.massdriver.blobStorage.azureblob.storageAccountName }}.blob.core.windows.net"
+  {{- if not (empty .Values.massdriver.blobStorage.azureblob.storageAccountKey) }}
+  JCLOUDS_IDENTITY: {{ .Values.massdriver.blobStorage.azureblob.storageAccountName | quote }}
+  JCLOUDS_CREDENTIAL: {{ .Values.massdriver.blobStorage.azureblob.storageAccountKey | quote }}
+  {{- else }}
+  JCLOUDS_IDENTITY: ""
+  JCLOUDS_CREDENTIAL: ""
+  AZURE_TENANT_ID: {{ .Values.massdriver.blobStorage.azureblob.tenantId | quote }}
+  AZURE_CLIENT_ID: {{ .Values.massdriver.blobStorage.azureblob.clientId | quote }}
+  AZURE_CLIENT_SECRET: {{ .Values.massdriver.blobStorage.azureblob.clientSecret | quote }}
+  {{- end }}
+  {{- end }}

charts/massdriver/templates/s3proxy/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "massdriver.fullname" . }}-s3proxy
+  labels:
+    {{- include "massdriver.labels" . | nindent 4 }}
+    app.kubernetes.io/component: s3proxy
+spec:
+  type: ClusterIP
+  ports:
+    - port: 80
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "massdriver.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: s3proxy

charts/massdriver/templates/s3proxy/serviceaccount.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "massdriver.fullname" . }}-s3proxy
+  labels:
+    {{- include "massdriver.labels" . | nindent 4 }}
+    app.kubernetes.io/component: s3proxy
+  {{- with .Values.massdriver.blobStorage.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: true

charts/massdriver/values.yaml

@@ -57,25 +57,67 @@ massdriver:
 
   # Configuration for blob storage
   blobStorage:
-    # options are "minio" or "s3". If using "s3", make sure you set the correct bucket names and s3 values below
+
+    # S3Proxy credentials for authenticated access
+    username: massdriver
+    password: s3proxypassword
+
+    # Log level for S3Proxy (debug, info, warn, error)
+    logLevel: info
+
+    # This is where you specify your desired object storage backend.
+    # Options are "minio", "s3", "gcs", or "azureblob". For whichever type you choose,
+    # be sure to fill out the corresponding configuration section below.
     type: minio
 
     # Bucket names for Massdriver to use. The default values will work with the included MinIO deployment.
-    # If you are using your own S3 buckets you will need to update the values.
-    # The massdriver bucket is used for bundle and OCI storage
+    # If you are using your own S3 buckets you will need to update the values to be the actual s3 bucket names (make sure they exist).
+    # If you are using Azure Blob Storage, the bucket names correspond to container names in the storage account (make sure they exist).
+    # If you are using GCS, the bucket names correspond to GCS bucket names (make sure they exist).
+    # The massdriver bucket is used for bundle/OCI storage and deployment logs
     massdriverBucket: massdriver
     # The state bucket is used for OpenTofu/Terraform remote state storage
     stateBucket: state
 
+    # Additional annotations to add to the ServiceAccount used by S3Proxy
+    # This is useful for adding dynamic IAM role bindings
+    annotations: {}
+
+    # Required if using "minio" for blobStorage.type
+    minio:
+      username: minioaccesskey
+      password: miniosecretkey
+
     # Required if using "s3" for blobStorage.type
-    # NOTE: Be sure to update Argo Workflows artifact repository if you wish to S3 for Argo artifacts as well
     s3:
       region: us-east-1
-      # S3 access is better granted via IAM roles (IRSA for EKS) by setting the proper annotations on the service account
-      # If you aren't running in EKS or prefer to use access keys, you can set them here
+      # S3 access can be granted via access keys or IAM roles (IRSA for EKS) by setting the proper annotations on the service account
+      # If you are using access keys, set them here. If not, leave them blank and ensure the service account has the proper annotations for role binding.
       accessKeyId: ""
       secretAccessKey: ""
 
+    # Required if using "gcs" for blobStorage.type
+    gcs:
+      serviceAccountEmail: ""
+      privateKey: |-
+        -----BEGIN PRIVATE KEY-----
+        ...
+        -----END PRIVATE KEY-----
+
+    # Required if using "azureblob" for blobStorage.type
+    azureblob:
+      region: eastus
+      storageAccountName: ""
+
+      # You can authenticate with either storage account key or via service principal.
+      # If using storage account key, set the value here and leave the service principal values blank.
+      storageAccountKey: ""
+
+      # If using service principal authentication, leave storageAccountKey blank and set the values below
+      clientId: ""
+      clientSecret: ""
+      tenantId: ""
+
   replicaCount: 2
 
   image: