WPB-27255: move email sending to background-worker

changelog.d/0-release-notes/wpb-27255-email-background-worker.md

@@ -0,0 +1,14 @@
+Outbound email delivery has moved from **brig** to the **background-worker**.
+brig no longer sends email directly: it enqueues every outbound
+message (verification, activation, password-reset, invitation, new-client,
+account-deletion, SAML IdP-change, provider and enterprise-audit mail) on the
+existing `background-jobs` RabbitMQ queue, and the background-worker performs
+the actual SMTP/SES send. Operators must configure the new
+`background-worker.config.email` block (SES **or** SMTP, the same shape as
+brig's `emailSMS.email`) and, for SES, the worker's AWS region and
+credentials (`AWS_REGION` and `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`).
+The `background-jobs` queue is a durable quorum queue, so transient
+background-worker downtime does not lose mail: undelivered jobs are
+requeued until a worker picks them up. When rolling out, deploy the updated
+background-worker before (or alongside) the updated brig so that the new
+`send-email` jobs are consumed as soon as they appear.

charts/wire-server/templates/background-worker/configmap.yaml

@@ -80,6 +80,22 @@ data:
       {{- end }}
     {{- end }}
 
+    email:
+    {{- if .useSES }}
+      sesQueue: {{ required "Missing value: background-worker.config.aws.sesQueue" .aws.sesQueue }}
+      sesEndpoint: {{ .aws.sesEndpoint | quote }}
+    {{- else }}
+      smtpEndpoint:
+        host: {{ .smtp.host }}
+        port: {{ .smtp.port }}
+      smtpConnType: {{ .smtp.connType }}
+      {{- if .smtp.username }}
+      smtpCredentials:
+        smtpUsername: {{ .smtp.username }}
+        smtpPassword: {{ .smtp.passwordFile }}
+      {{- end }}
+    {{- end }}
+
     migrateConversations: {{ .migrateConversations }}
     migrateConversationCodes: {{ .migrateConversationCodes }}
     migrateTeamFeatures: {{ .migrateTeamFeatures }}

charts/wire-server/templates/background-worker/deployment.yaml

@@ -109,6 +109,20 @@ spec:
           {{ toYaml .Values.additionalVolumeMounts | nindent 10 }}
           {{- end }}
           env:
+          {{- if hasKey $backgroundWorker.secrets "awsKeyId" }}
+          - name: AWS_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                name: background-worker
+                key: awsKeyId
+          - name: AWS_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: background-worker
+                key: awsSecretKey
+          {{- end }}
+          - name: AWS_REGION
+            value: "{{ $backgroundWorker.config.aws.region }}"
           - name: RABBITMQ_USERNAME
             valueFrom:
               secretKeyRef:

charts/wire-server/templates/background-worker/secret.yaml

@@ -14,6 +14,13 @@ data:
   for_helm_linting: {{ required "No .secrets found in configuration. Did you forget to helm <command> -f path/to/secrets.yaml ?" $backgroundWorker.secrets | quote | b64enc | quote }}
 
   {{- with $backgroundWorker.secrets }}
+  {{- if .awsKeyId }}
+  awsKeyId: {{ .awsKeyId | b64enc | quote }}
+  awsSecretKey: {{ .awsSecretKey | b64enc | quote }}
+  {{- end }}
+  {{- if (not $backgroundWorker.config.useSES) }}
+  smtp-password.txt: {{ .smtpPassword | b64enc | quote }}
+  {{- end }}
   rabbitmqUsername: {{ .rabbitmq.username | b64enc | quote }}
   rabbitmqPassword: {{ .rabbitmq.password | b64enc | quote }}
   {{- end }}

charts/wire-server/values.yaml

@@ -1074,6 +1074,28 @@ background-worker:
       # Cron schedule for the cleanup job (0 * * * * = every hour)
       schedule: "0 * * * *"
 
+    # Email transport for the background-worker (delivers the email jobs
+    # enqueued by brig). Same SES/SMTP shape as brig's emailSMS.email.
+    # `useSES` selects the transport: when true, the worker sends via AWS SES
+    # using `aws.sesQueue`/`aws.sesEndpoint` together with the AWS_ACCESS_KEY_ID
+    # / AWS_SECRET_ACCESS_KEY secrets and the AWS_REGION below. When false, it
+    # sends via SMTP using the `smtp.*` settings.
+    useSES: true
+    aws:
+      region: "eu-west-1"
+      sesEndpoint: https://email.eu-west-1.amazonaws.com
+      # sesQueue is required when useSES is true (deployment-specific), e.g.:
+      # sesQueue: wire-brig-events
+    # SMTP transport (used when useSES is false). The ConfigMap renders these
+    # into the `email` block; mirrors brig's `smtp` settings.
+    smtp:
+      passwordFile: /etc/wire/background-worker/secrets/smtp-password.txt
+
+  # Optional secret keys (see templates/background-worker/secret.yaml):
+  #   awsKeyId: <aws-access-key-id>          # required for SES; rendered as AWS_ACCESS_KEY_ID
+  #   awsSecretKey: <aws-secret-access-key>   # required for SES; rendered as AWS_SECRET_ACCESS_KEY
+  #   smtpPassword: <smtp-password>           # mounted at /etc/wire/background-worker/secrets/smtp-password.txt;
+  #                                          # consumed via config.smtp.passwordFile (mirrors brig/galley)
   secrets: {}
 
   podSecurityContext:

docs/src/developer/reference/config-options.md

@@ -2403,3 +2403,66 @@ Notes
 - `jobs.workerThreads` controls the number of worker threads in each job queue. The default is `1`; increasing it allows jobs in that queue to run in parallel when their group keys permit it.
 - Both job queues share the same PostgreSQL pool. Increasing `jobs.workerThreads` can increase the number of connections needed when more jobs run concurrently, but it does not create a permanently dedicated connection per thread or queue.
 - The job runner is poll-only and does not require an additional PostgreSQL listener connection.
+
+## Background worker: Email sending
+
+The background-worker delivers the email jobs enqueued by brig. It requires an
+`email` transport (AWS SES or SMTP), the same shape brig uses for
+`emailSMS.email`. Configuration is supplied via Helm under
+`background-worker.config` and rendered into the `email` block of
+`background-worker.yaml`.
+
+The transport is selected by `background-worker.config.useSES`:
+
+- `useSES: true` (default) renders an SES block. `aws.sesQueue` is required and
+  `aws.sesEndpoint` selects the SES endpoint. The worker also needs the
+  `AWS_REGION`, `AWS_ACCESS_KEY_ID`, and `AWS_SECRET_ACCESS_KEY` environment
+  variables, injected from `background-worker.config.aws.region` and the
+  `awsKeyId`/`awsSecretKey` secrets (the same pattern brig uses).
+- `useSES: false` renders an SMTP block using the `smtp.*` settings. The SMTP
+  password is read from the file named by `smtp.passwordFile` (mounted from the
+  `smtpPassword` secret).
+
+Rendered config (`background-worker.yaml`):
+
+```yaml
+# SES:
+email:
+  sesQueue: wire-brig-events
+  sesEndpoint: https://email.eu-west-1.amazonaws.com
+# SMTP (xor SES):
+# email:
+#   smtpEndpoint: { host: smtp.example.com, port: 587 }
+#   smtpConnType: tls
+#   smtpCredentials:
+#     smtpUsername: wire
+#     smtpPassword: /etc/wire/background-worker/secrets/smtp-password.txt
+```
+
+Helm values (under `background-worker`):
+
+```yaml
+config:
+  useSES: true
+  aws:
+    region: "eu-west-1"
+    sesEndpoint: https://email.eu-west-1.amazonaws.com
+    sesQueue: wire-brig-events   # required when useSES is true
+  smtp:
+    passwordFile: /etc/wire/background-worker/secrets/smtp-password.txt
+secrets:
+  awsKeyId: <aws-access-key-id>          # SES only
+  awsSecretKey: <aws-secret-access-key>  # SES only
+  smtpPassword: <smtp-password>          # SMTP only
+```
+
+Notes
+
+- `email` is required: the worker fails to start without a transport.
+- For SES, the worker reads `AWS_REGION` from `config.aws.region` and the AWS
+  credentials from the `awsKeyId`/`awsSecretKey` secrets, mirroring brig.
+- For SMTP, the password is mounted at
+  `/etc/wire/background-worker/secrets/smtp-password.txt` (from the
+  `smtpPassword` secret); `config.smtp.passwordFile` must point at it.
+- The `background-jobs` queue is durable, so transient worker downtime does not
+  lose email jobs; an updated worker picks up messages an older one requeued.

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -697,7 +697,15 @@ background-worker:
       tlsCaSecretRef:
         name: "rabbitmq-certificate"
         key: "ca.crt"
+    # Email transport: the worker sends email via SES in CI (mirrors brig).
+    useSES: true
+    aws:
+      region: "eu-west-1"
+      sesEndpoint: http://fake-aws-ses:4569
+      sesQueue: integration-brig-events
   secrets:
+    awsKeyId: dummykey
+    awsSecretKey: dummysecret
     rabbitmq:
       username: {{ .Values.rabbitmqUsername }}
       password: {{ .Values.rabbitmqPassword }}

libs/wire-api/src/Wire/API/BackgroundJobs.hs

@@ -30,22 +30,26 @@ import Data.Schema
 import Imports
 import Network.AMQP qualified as Q
 import Network.AMQP.Types qualified as QT
+import Wire.API.BackgroundJobs.Email (SendEmailJob)
 import Wire.Arbitrary (Arbitrary (..), GenericUniform (..))
 
 data BackgroundJobPayload
   = BackgroundJobSyncUserGroupAndChannel SyncUserGroupAndChannel
   | BackgroundJobSyncUserGroup SyncUserGroup
+  | BackgroundJobSendEmail !SendEmailJob
   deriving stock (Eq, Show, Generic)
   deriving (Arbitrary) via GenericUniform BackgroundJobPayload
 
 backgroundJobPayloadLabel :: BackgroundJobPayload -> Text
 backgroundJobPayloadLabel p = case backgroundJobPayloadTag p of
   BackgroundJobSyncUserGroupAndChannelTag -> "sync-user-group-and-channel"
   BackgroundJobSyncUserGroupTag -> "sync-user-group"
+  BackgroundJobSendEmailTag -> "send-email"
 
 data BackgroundJobPayloadTag
   = BackgroundJobSyncUserGroupAndChannelTag
   | BackgroundJobSyncUserGroupTag
+  | BackgroundJobSendEmailTag
   deriving stock (Eq, Ord, Bounded, Enum, Show, Generic)
   deriving (Arbitrary) via GenericUniform BackgroundJobPayloadTag
 
@@ -54,14 +58,16 @@ instance ToSchema BackgroundJobPayloadTag where
     enum @Text $
       mconcat
         [ element "sync-user-group-and-channel" BackgroundJobSyncUserGroupAndChannelTag,
-          element "sync-user-group" BackgroundJobSyncUserGroupTag
+          element "sync-user-group" BackgroundJobSyncUserGroupTag,
+          element "send-email" BackgroundJobSendEmailTag
         ]
 
 backgroundJobPayloadTag :: BackgroundJobPayload -> BackgroundJobPayloadTag
 backgroundJobPayloadTag =
   \case
     BackgroundJobSyncUserGroupAndChannel {} -> BackgroundJobSyncUserGroupAndChannelTag
     BackgroundJobSyncUserGroup {} -> BackgroundJobSyncUserGroupTag
+    BackgroundJobSendEmail {} -> BackgroundJobSendEmailTag
 
 backgroundJobPayloadTagSchema :: ObjectSchema SwaggerDoc BackgroundJobPayloadTag
 backgroundJobPayloadTagSchema = field "type" schema
@@ -116,6 +122,7 @@ backgroundJobPayloadObjectSchema =
     backgroundJobPayloadDataSchema = \case
       BackgroundJobSyncUserGroupAndChannelTag -> tag _BackgroundJobSyncUserGroupAndChannel (field "payload" schema)
       BackgroundJobSyncUserGroupTag -> tag _BackgroundJobSyncUserGroup (field "payload" schema)
+      BackgroundJobSendEmailTag -> tag _BackgroundJobSendEmail (field "payload" schema)
 
 instance ToSchema BackgroundJobPayload where
   schema = object backgroundJobPayloadObjectSchema