Skip to content

Commit 425ddd8

Browse files
chrisghillchrisghill
authored
Use S3Proxy to access object storage so we can swap out cloud storage engines (#29)
* working for aws and azure, gcp next * GCS seems to be working * convert to a secret * add log levels back, cleanup comments * final updates to values - everything tested and working * final changes
1 parent 4843a9f commit 425ddd8

File tree

11 files changed

+212
-57
lines changed

11 files changed

+212
-57
lines changed

‎charts/massdriver/templates/_helpers.tpl‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -126,12 +126,4 @@ so we have to handle the "double" base64 encoding gracefully
126126

127127
{{- define "massdriver.phxSigningSalt" -}}
128128
{{- include "massdriver.getValueFromSecret" (dict "Namespace" .Release.Namespace "Name" (printf "%s-massdriver-envs" (include "massdriver.fullname" .)) "Length" 20 "Key" "PHX_SIGNING_SALT") }}
129-
{{- end -}}
130-
131-
{{- define "massdriver.minio.password" -}}
132-
{{ include "massdriver.getValueFromSecret" (dict "Namespace" .Release.Namespace "Name" (printf "%s-minio" (include "massdriver.fullname" .)) "Length" 40 "Key" "rootPassword") }}
133-
{{- end }}
134-
135-
{{- define "massdriver.minio.username" -}}
136-
{{ include "massdriver.getValueFromSecret" (dict "Namespace" .Release.Namespace "Name" (printf "%s-minio" (include "massdriver.fullname" .)) "Length" 20 "Key" "rootUser") }}
137-
{{- end }}
129+
{{- end -}}

‎charts/massdriver/templates/massdriver/configmap-envs.yaml‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,9 @@ metadata:
66
{{- include "massdriver.labels" . | nindent 4 }}
77
app.kubernetes.io/component: massdriver
88
data:
9-
{{- if eq .Values.massdriver.blobStorage.type "minio" }}
10-
BLOB_STORAGE_HOST: "{{ include "massdriver.fullname" . }}-minio.{{ .Release.Namespace }}.svc"
11-
BLOB_STORAGE_PORT: "{{ toString .Values.minio.service.port }}"
9+
BLOB_STORAGE_HOST: "{{ include "massdriver.fullname" . }}-s3proxy.{{ .Release.Namespace }}.svc"
10+
BLOB_STORAGE_PORT: "80"
1211
BLOB_STORAGE_SCHEME: http
13-
{{- else if eq .Values.massdriver.blobStorage.type "s3" }}
14-
AWS_REGION: {{ .Values.massdriver.blobStorage.s3.region }}
15-
BLOB_STORAGE_HOST: "{{ printf "s3.%s.amazonaws.com" .Values.massdriver.blobStorage.s3.region }}"
16-
BLOB_STORAGE_PORT: "443"
17-
BLOB_STORAGE_SCHEME: "https"
18-
{{- end }}
1912
DATABASE_SSL: "true"
2013
FORCE_V2_LOGGING: "true"
2114
LOG_LEVEL: {{ .Values.massdriver.logLevel | quote }}

‎charts/massdriver/templates/massdriver/deployment.yaml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ spec:
2020
configmap.massdriver-envs/checksum: {{ include (print $.Template.BasePath "/massdriver/configmap-envs.yaml") . | sha256sum }}
2121
configmap.ui-envs/checksum: {{ include (print $.Template.BasePath "/massdriver/configmap-ui.yaml") . | sha256sum }}
2222
secret.massdriver-envs/checksum: {{ include (print $.Template.BasePath "/massdriver/secret-envs.yaml") . | sha256sum }}
23-
secret.minio/checksum: {{ include (print $.Template.BasePath "/massdriver/secret-minio.yaml") . | sha256sum }}
2423
labels:
2524
{{- include "massdriver.labels" . | nindent 8 }}
2625
{{- with .Values.massdriver.podLabels }}
@@ -49,10 +48,6 @@ spec:
4948
name: {{ include "massdriver.fullname" . }}-massdriver-envs
5049
- secretRef:
5150
name: {{ include "massdriver.fullname" . }}-massdriver-envs
52-
{{- if and .Values.minio.enabled (eq .Values.massdriver.blobStorage.type "minio") }}
53-
- secretRef:
54-
name: {{ include "massdriver.fullname" . }}-massdriver-minio-auth
55-
{{- end }}
5651
ports:
5752
- name: http
5853
containerPort: {{ .Values.massdriver.port }}

‎charts/massdriver/templates/massdriver/job-db-migration.yaml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,6 @@ spec:
3737
name: {{ include "massdriver.fullname" . }}-massdriver-envs
3838
- secretRef:
3939
name: {{ include "massdriver.fullname" . }}-massdriver-envs
40-
{{- if and .Values.minio.enabled (eq .Values.massdriver.blobStorage.type "minio") }}
41-
- secretRef:
42-
name: {{ include "massdriver.fullname" . }}-massdriver-minio-auth
43-
{{- end }}
4440
args:
4541
- eval
4642
- Massdriver.Release.migrate

‎charts/massdriver/templates/massdriver/secret-envs.yaml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,9 @@ metadata:
66
{{- include "massdriver.labels" . | nindent 4 }}
77
app.kubernetes.io/component: massdriver
88
data:
9-
{{- if and (eq .Values.massdriver.blobStorage.type "s3") (not (empty .Values.massdriver.blobStorage.s3.accessKeyId)) (not (empty .Values.massdriver.blobStorage.s3.secretAccessKey)) }}
10-
AWS_ACCESS_KEY_ID: {{ .Values.massdriver.blobStorage.s3.accessKeyId | b64enc | quote }}
11-
AWS_SECRET_ACCESS_KEY: {{ .Values.massdriver.blobStorage.s3.secretAccessKey | b64enc | quote }}
12-
{{- end }}
9+
AWS_ACCESS_KEY_ID: {{ .Values.massdriver.blobStorage.username | b64enc | quote }}
10+
AWS_SECRET_ACCESS_KEY: {{ .Values.massdriver.blobStorage.password | b64enc | quote }}
11+
AWS_REGION: {{ "us-east-1" | b64enc | quote }}
1312
JWT_SECRET: {{ include "massdriver.jwtSecret" . | b64enc | quote }}
1413
LICENSE_KEY: {{ .Values.licenseKey | b64enc | quote }}
1514
MD_CLOAK_KEY: {{ include "massdriver.cloakKey" . | b64enc | quote }}

‎charts/massdriver/templates/massdriver/secret-minio.yaml‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
{{- if .Values.minio.enabled -}}
2-
{{- $minioUser := include "massdriver.minio.username" . }}
3-
{{- $minioPassword := include "massdriver.minio.password" . }}
42
apiVersion: v1
53
kind: Secret
64
metadata:
@@ -9,20 +7,6 @@ metadata:
97
{{- include "massdriver.labels" . | nindent 4 }}
108
app.kubernetes.io/component: minio
119
data:
12-
rootUser: {{ $minioUser | b64enc | quote }}
13-
rootPassword: {{ $minioPassword | b64enc | quote }}
14-
---
15-
# this secret is used by massdriver to authenticate with minio - it is here so that the secret generator will be consistent on install
16-
{{- if eq .Values.massdriver.blobStorage.type "minio" }}
17-
apiVersion: v1
18-
kind: Secret
19-
metadata:
20-
name: {{ include "massdriver.fullname" . }}-massdriver-minio-auth
21-
labels:
22-
{{- include "massdriver.labels" . | nindent 4 }}
23-
app.kubernetes.io/component: minio
24-
data:
25-
AWS_ACCESS_KEY_ID: {{ $minioUser | b64enc | quote }}
26-
AWS_SECRET_ACCESS_KEY: {{ $minioPassword | b64enc | quote }}
27-
{{- end }}
28-
{{- end }}
10+
rootUser: {{ .Values.massdriver.blobStorage.minio.username | b64enc | quote }}
11+
rootPassword: {{ .Values.massdriver.blobStorage.minio.password | b64enc | quote }}
12+
{{- end -}}

‎charts/massdriver/templates/s3proxy/deployment.yaml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: {{ include "massdriver.fullname" . }}-s3proxy
5+
labels:
6+
{{- include "massdriver.labels" . | nindent 4 }}
7+
app.kubernetes.io/component: s3proxy
8+
spec:
9+
replicas: 2
10+
selector:
11+
matchLabels:
12+
{{- include "massdriver.selectorLabels" . | nindent 6 }}
13+
app.kubernetes.io/component: s3proxy
14+
template:
15+
metadata:
16+
annotations:
17+
secret.s3proxy-config/checksum: {{ include (print $.Template.BasePath "/s3proxy/secret-envs.yaml") . | sha256sum }}
18+
labels:
19+
{{- include "massdriver.labels" . | nindent 8 }}
20+
app.kubernetes.io/component: s3proxy
21+
spec:
22+
serviceAccountName: {{ include "massdriver.fullname" . }}-s3proxy
23+
containers:
24+
- name: s3proxy
25+
image: "andrewgaul/s3proxy:3.0.0"
26+
imagePullPolicy: IfNotPresent
27+
ports:
28+
- name: http
29+
containerPort: 8080
30+
protocol: TCP
31+
livenessProbe:
32+
tcpSocket:
33+
port: http
34+
initialDelaySeconds: 30
35+
periodSeconds: 10
36+
timeoutSeconds: 5
37+
failureThreshold: 3
38+
readinessProbe:
39+
tcpSocket:
40+
port: http
41+
initialDelaySeconds: 10
42+
periodSeconds: 5
43+
timeoutSeconds: 3
44+
failureThreshold: 3
45+
resources:
46+
limits:
47+
cpu: 500m
48+
memory: 512Mi
49+
requests:
50+
cpu: 100m
51+
memory: 128Mi
52+
envFrom:
53+
- secretRef:
54+
name: {{ include "massdriver.fullname" . }}-s3proxy-envs
55+
securityContext:
56+
capabilities:
57+
drop:
58+
- ALL
59+
readOnlyRootFilesystem: false
60+
runAsNonRoot: true
61+
runAsUser: 65534
62+
securityContext:
63+
fsGroup: 65534

‎charts/massdriver/templates/s3proxy/secret-envs.yaml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
apiVersion: v1
2+
kind: Secret
3+
metadata:
4+
name: {{ include "massdriver.fullname" . }}-s3proxy-envs
5+
labels:
6+
{{- include "massdriver.labels" . | nindent 4 }}
7+
app.kubernetes.io/component: s3proxy
8+
type: Opaque
9+
data:
10+
# S3Proxy configuration
11+
S3PROXY_ENDPOINT: {{ "http://0.0.0.0:8080" | b64enc | quote }}
12+
S3PROXY_AUTHORIZATION: {{ "aws-v4" | b64enc | quote }}
13+
S3PROXY_IDENTITY: {{ .Values.massdriver.blobStorage.username | b64enc | quote }}
14+
S3PROXY_CREDENTIAL: {{ .Values.massdriver.blobStorage.password | b64enc | quote }}
15+
S3PROXY_VIRTUALHOST: {{ "" | b64enc | quote }}
16+
S3PROXY_IGNORE_UNKNOWN_HEADERS: {{ "true" | b64enc | quote }}
17+
S3PROXY_CORS_ALLOW_ALL: {{ "false" | b64enc | quote }}
18+
19+
# Logging configuration for debugging
20+
LOG_LEVEL: {{ .Values.massdriver.blobStorage.logLevel | b64enc | quote }}
21+
JETTY_LOG_LEVEL: {{ .Values.massdriver.blobStorage.logLevel | b64enc | quote }}
22+
23+
{{- if eq .Values.massdriver.blobStorage.type "minio" }}
24+
# MinIO backend configuration (using S3 SDK provider for path-style bucket addressing)
25+
JCLOUDS_PROVIDER: {{ "aws-s3-sdk" | b64enc | quote }}
26+
JCLOUDS_IDENTITY: {{ .Values.massdriver.blobStorage.minio.username | b64enc | quote }}
27+
JCLOUDS_CREDENTIAL: {{ .Values.massdriver.blobStorage.minio.password | b64enc | quote }}
28+
JCLOUDS_ENDPOINT: {{ printf "http://%s-minio.%s.svc:%s" (include "massdriver.fullname" .) .Release.Namespace (toString .Values.minio.service.port) | b64enc | quote }}
29+
{{- else if eq .Values.massdriver.blobStorage.type "s3" }}
30+
# AWS S3 Configuration
31+
JCLOUDS_PROVIDER: {{ "aws-s3" | b64enc | quote }}
32+
JCLOUDS_IDENTITY: {{ .Values.massdriver.blobStorage.s3.accessKeyId | b64enc | quote }}
33+
JCLOUDS_CREDENTIAL: {{ .Values.massdriver.blobStorage.s3.secretAccessKey | b64enc | quote }}
34+
JCLOUDS_ENDPOINT: {{ printf "https://s3.%s.amazonaws.com" .Values.massdriver.blobStorage.s3.region | b64enc | quote }}
35+
JCLOUDS_REGION: {{ .Values.massdriver.blobStorage.s3.region | b64enc | quote }}
36+
{{- else if eq .Values.massdriver.blobStorage.type "gcs" }}
37+
# Google Cloud Storage Configuration
38+
JCLOUDS_PROVIDER: {{ "google-cloud-storage" | b64enc | quote }}
39+
JCLOUDS_IDENTITY: {{ .Values.massdriver.blobStorage.gcs.serviceAccountEmail | b64enc | quote }}
40+
JCLOUDS_CREDENTIAL: {{ .Values.massdriver.blobStorage.gcs.privateKey | b64enc | quote }}
41+
JCLOUDS_ENDPOINT: {{ "https://storage.googleapis.com" | b64enc | quote }}
42+
{{- else if eq .Values.massdriver.blobStorage.type "azureblob" }}
43+
# Azure Blob Storage Configuration
44+
JCLOUDS_PROVIDER: {{ "azureblob-sdk" | b64enc | quote }}
45+
JCLOUDS_ENDPOINT: {{ printf "https://%s.blob.core.windows.net" .Values.massdriver.blobStorage.azureblob.storageAccountName | b64enc | quote }}
46+
{{- if not (empty .Values.massdriver.blobStorage.azureblob.storageAccountKey) }}
47+
JCLOUDS_IDENTITY: {{ .Values.massdriver.blobStorage.azureblob.storageAccountName | b64enc | quote }}
48+
JCLOUDS_CREDENTIAL: {{ .Values.massdriver.blobStorage.azureblob.storageAccountKey | b64enc | quote }}
49+
{{- else }}
50+
JCLOUDS_IDENTITY: {{ "" | b64enc | quote }}
51+
JCLOUDS_CREDENTIAL: {{ "" | b64enc | quote }}
52+
AZURE_TENANT_ID: {{ .Values.massdriver.blobStorage.azureblob.tenantId | b64enc | quote }}
53+
AZURE_CLIENT_ID: {{ .Values.massdriver.blobStorage.azureblob.clientId | b64enc | quote }}
54+
AZURE_CLIENT_SECRET: {{ .Values.massdriver.blobStorage.azureblob.clientSecret | b64enc | quote }}
55+
{{- end }}
56+
{{- end }}

‎charts/massdriver/templates/s3proxy/service.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: {{ include "massdriver.fullname" . }}-s3proxy
5+
labels:
6+
{{- include "massdriver.labels" . | nindent 4 }}
7+
app.kubernetes.io/component: s3proxy
8+
spec:
9+
type: ClusterIP
10+
ports:
11+
- port: 80
12+
targetPort: http
13+
protocol: TCP
14+
name: http
15+
selector:
16+
{{- include "massdriver.selectorLabels" . | nindent 4 }}
17+
app.kubernetes.io/component: s3proxy

‎charts/massdriver/templates/s3proxy/serviceaccount.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: {{ include "massdriver.fullname" . }}-s3proxy
5+
labels:
6+
{{- include "massdriver.labels" . | nindent 4 }}
7+
app.kubernetes.io/component: s3proxy
8+
{{- with .Values.massdriver.blobStorage.serviceAccount.annotations }}
9+
annotations:
10+
{{- toYaml . | nindent 4 }}
11+
{{- end }}
12+
automountServiceAccountToken: true

0 commit comments

Comments
 (0)