diff --git a/charts/d2e-services/charts/d2e-core/templates/core-caddy-deployment.yaml b/charts/d2e-services/charts/d2e-core/templates/core-caddy-deployment.yaml index 89748a7ccd..acf1caa150 100644 --- a/charts/d2e-services/charts/d2e-core/templates/core-caddy-deployment.yaml +++ b/charts/d2e-services/charts/d2e-core/templates/core-caddy-deployment.yaml @@ -95,10 +95,18 @@ spec: - name: caddy-local-pki secret: secretName: "{{ .Release.Name }}-{{ .Chart.Name }}-caddy" - # Internal CA for `tls_trust_pool file` on the trex upstreams. Note this - # is the chart-level secret, not the -caddy one above: the -caddy secret - # holds Caddy's own edge PKI, not the internal CA. CA only -- Caddy is a - # client on these hops, never the server. + # Internal TLS material from the chart-level secret, not the -caddy one + # above: the -caddy secret holds Caddy's own edge PKI (SERVER__CRT, the + # environment's real public certificate), not the internal CA. + # + # ca.pem -- Caddy as a *client*, for `tls_trust_pool file` when proxying + # to the trex/idp upstreams. + # cert.pem/key.pem -- Caddy as a *server*, for the + # https://gateway..svc.cluster.local site block. Internal + # callers (WebAPI fetching the advertised jwks_uri) dial the + # gateway by its cluster name, which SERVER__CRT does not + # cover; without a certificate for it Caddy aborts the + # handshake and WebAPI's OIDC bean fails to build. - name: d2e-tls secret: secretName: "{{ .Release.Name }}-{{ .Chart.Name }}" @@ -106,3 +114,10 @@ spec: - key: TLS__INTERNAL__CA_CRT path: ca.pem mode: 0444 + - key: TLS__INTERNAL__CRT + path: cert.pem + mode: 0444 + # Owner-only: this is a private key, and Caddy runs as root here. + - key: TLS__INTERNAL__KEY + path: key.pem + mode: 0400 diff --git a/charts/d2e-services/charts/d2e-core/templates/core-config.yaml b/charts/d2e-services/charts/d2e-core/templates/core-config.yaml index e81f9133e4..4e0b050e9b 100644 --- a/charts/d2e-services/charts/d2e-core/templates/core-config.yaml +++ b/charts/d2e-services/charts/d2e-core/templates/core-config.yaml @@ -147,6 +147,47 @@ data: } } + # Internal callers reach the gateway by its in-cluster Service name over TLS. + # Logto advertises jwks_uri as https://gateway..svc.cluster.local/oidc/jwks + # -- endpoint URLs follow the TLS hop into idp (and X-Forwarded-Proto https), + # while `issuer` stays the plaintext ENDPOINT. WebAPI reads that jwks_uri and + # dials it, so this name must be servable over 443. + # + # Without this block nothing on 443 matches the cluster name: the only other + # https site is {$CADDY__D2E__INTERNAL_FQDN}, so Caddy finds no certificate + # and aborts with an internal_error alert one message after Client Hello -- + # before any certificate is offered, which is why `curl -k` fails identically. + # WebAPI then cannot build OidcAuthConfig$OpenidDirect, returns 500, and every + # analytics-svc call that proxies to WebAPI 500s with it. + # + # It serves the internal certificate, not SERVER__CRT: SERVER__CRT is the + # environment's real public certificate and has no reason to carry + # cluster-internal SANs. (CI hides this by feeding the same self-signed cert + # to both SERVER__CRT and TLS__INTERNAL__CRT, so its gateway happens to cover + # the cluster name.) Routes mirror the :8080 block above so internal callers + # get the same behaviour over either scheme. + https://gateway.{{ .Release.Namespace }}.svc.cluster.local { + tls /etc/d2e/tls/cert.pem /etc/d2e/tls/key.pem + + @oidc { + path /oidc/* + } + handle @oidc { + reverse_proxy https://idp.{{ .Release.Namespace }}.svc.cluster.local { + transport http { + tls + tls_trust_pool file /etc/d2e/tls/ca.pem + } + } + } + handle /check-liveness* { + respond "OK-INTERNAL-TLS" + } + handle { + respond 404 + } + } + https://localhost { tls { issuer internal { diff --git a/charts/d2e-services/templates/d2e-deployment.yaml b/charts/d2e-services/templates/d2e-deployment.yaml index 0e1ee4c8e8..976d09bcb6 100644 --- a/charts/d2e-services/templates/d2e-deployment.yaml +++ b/charts/d2e-services/templates/d2e-deployment.yaml @@ -90,6 +90,9 @@ spec: requests: cpu: 50m memory: 50Mi + limits: + cpu: 100m + memory: 100Mi volumeMounts: - mountPath: /etc/d2e/tls name: d2e-tls @@ -123,7 +126,10 @@ spec: cp -rfv /data/plugins/bunfig.toml /usr/src/data/plugins/bunfig.toml && exit 0", ] - name: logto-check - image: busybox:latest + # Not busybox: its wget has no --ca-certificate, only the + # --no-check-certificate bypass. The trex image is already pulled for + # this pod and ships curl, which verifies chain and hostname by default. + image: ghcr.io/ohdsi/d2e-trex:{{ .Values.global.image.tag }} resources: {{- toYaml .Values.global.resources.logtoCheck | nindent 12 }} volumeMounts: @@ -134,7 +140,7 @@ spec: [ "sh", "-c", - "while :;do sleep 20 && wget --ca-certificate=/etc/d2e/tls/ca.pem -O success https://idp.{{ .Release.Namespace }}.svc.cluster.local:443/api/status && if test -f success; then exit 0; fi; done", + "until curl -sf --cacert /etc/d2e/tls/ca.pem https://idp.{{ .Release.Namespace }}.svc.cluster.local:443/api/status >/dev/null; do echo 'waiting for logto ...'; sleep 20; done", ] - name: logto-post-init env: @@ -441,11 +447,39 @@ spec: # Three runtimes share this container and each finds its trust # anchors somewhere different. The OS store covers shell/curl and, # via DENO_TLS_CA_STORE=system, the Deno edge functions. GraalVM - # WebAPI ignores the OS store entirely and reads WEBAPI_TRUST_CERTS. - # All three are required: a pod-level curl check passes while the - # other two runtimes still fail. - cp /etc/d2e/tls/ca.pem /usr/local/share/ca-certificates/d2e-internal-ca.crt + # WebAPI ignores the OS store entirely and reads WEBAPI_TRUST_CERTS + # (which points at this same file). All three are required: a + # pod-level curl check passes while the other two runtimes fail. + # + # Assert rather than hope. This copy previously ran against a path + # the container did not mount: the cp failed, update-ca-certificates + # had nothing to add, and the process started with public roots only. + # Every internal TLS call then failed -- UnknownIssuer from rustls, + # PKIX from GraalVM -- while the `curl -fk` liveness probe still went + # green, so the pod reported Ready. Starting untrusted is worse than + # not starting, so each step below is checked. + ca=/etc/d2e/tls/ca.pem + if [ ! -s "$ca" ]; then + echo "FATAL: $ca is missing or empty -- the d2e-tls volume is not mounted, or global.secrets.TLS__INTERNAL__CA_CRT is empty. Refusing to start without the internal trust anchor." >&2 + exit 1 + fi + if ! grep -q '^-----BEGIN CERTIFICATE-----$' "$ca"; then + echo "FATAL: $ca is not a PEM certificate. Refusing to start without the internal trust anchor." >&2 + exit 1 + fi + cp "$ca" /usr/local/share/ca-certificates/d2e-internal-ca.crt || { + echo "FATAL: could not install $ca into the OS trust store." >&2 + exit 1 + } update-ca-certificates + # The anchor is self-signed, so verifying it against the freshly + # rebuilt bundle proves it is actually in the store -- not merely + # that the file was copied and the rebuild exited 0. + if ! openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt "$ca" >/dev/null 2>&1; then + echo "FATAL: the internal CA is not present in the OS trust store after update-ca-certificates. Refusing to start untrusted." >&2 + exit 1 + fi + echo "internal CA installed into OS trust store: $(openssl x509 -in "$ca" -noout -subject)" exec /usr/src/entrypoint.sh ${TREX_OTEL_ENABLED:+--enable-otel} env: - name: RUST_BACKTRACE @@ -1170,6 +1204,12 @@ spec: # webapi-init container, which ran the same files through psql. - mountPath: /usr/src/atlas-db-init name: atlas-db-init + # The internal CA, read by the args above (OS trust store, for + # shell/curl and DENO_TLS_CA_STORE=system) and by the GraalVM WebAPI + # via WEBAPI_TRUST_CERTS. Without this mount all three go untrusted. + - mountPath: /etc/d2e/tls + name: d2e-tls + readOnly: true {{- if .Values.features.materializeCohorts }} - mountPath: /usr/src/cert/client.crt name: materialize-cohorts-client-pki diff --git a/charts/d2e-services/templates/dataflow-worker-deployment.yaml b/charts/d2e-services/templates/dataflow-worker-deployment.yaml index bcbc950593..ae5ae5e577 100644 --- a/charts/d2e-services/templates/dataflow-worker-deployment.yaml +++ b/charts/d2e-services/templates/dataflow-worker-deployment.yaml @@ -33,12 +33,16 @@ spec: # claimName: trex-pvc initContainers: - name: dependencies-check - image: busybox:latest + # Not busybox: its wget has no --ca-certificate, only the + # --no-check-certificate bypass. The worker image is already pulled for + # this pod and ships curl (see its Dockerfile), which verifies chain and + # hostname by default. + image: ghcr.io/ohdsi/d2e-dataflow-gen-worker:{{ .Values.global.image.tag }} command: [ "sh", "-c", - "while :;do sleep {{ .Values.dataflowWorker.readinessSleep }} && wget --ca-certificate=/etc/d2e/tls/ca.pem -O success-trex https://trex.{{ .Release.Namespace }}.svc.cluster.local:443/trex/_internal/health && wget --ca-certificate=/etc/d2e/tls/ca.pem -O success-dataflow https://dataflow-gen.{{ .Release.Namespace }}.svc.cluster.local:443/d2e/api/health && if test -f success-trex; then if test -f success-dataflow; then exit 0; fi; fi; done", + "until curl -sf --cacert /etc/d2e/tls/ca.pem https://trex.{{ .Release.Namespace }}.svc.cluster.local:443/trex/_internal/health >/dev/null && curl -sf --cacert /etc/d2e/tls/ca.pem https://dataflow-gen.{{ .Release.Namespace }}.svc.cluster.local:443/d2e/api/health >/dev/null; do echo 'waiting for trex and dataflow-gen ...'; sleep {{ .Values.dataflowWorker.readinessSleep }}; done", ] volumeMounts: - mountPath: /etc/d2e/tls @@ -301,6 +305,21 @@ spec: value: "10" - name: ROUNDING_TARGET value: "10" + # Trust the internal self-signed CA across every Python HTTP stack, + # matching docker-compose (alp-dataflow-gen-worker). The args above + # put the CA into the OS bundle; these point urllib/requests/httpx at + # that bundle. Without them only clients that explicitly pass + # certifi.where() are covered -- stdlib urlopen (run-flow.sh's + # deployment lookup) and websockets (the prefect events subscriber) + # read OpenSSL's defaults, which under `pixi run` resolve inside the + # env prefix (/ssl/cert.pem) and never see the CA. Flows also + # exec in their own per-plugin pixi env provisioned at run time, so a + # startup-time append to this env's certifi cannot reach them; an + # inherited env var can. + - name: SSL_CERT_FILE + value: /etc/ssl/certs/ca-certificates.crt + - name: REQUESTS_CA_BUNDLE + value: /etc/ssl/certs/ca-certificates.crt command: - /entrypoint.sh - pixi @@ -312,8 +331,11 @@ spec: args: - >- rm -rf /root/.prefect/storage/run_cache.db /root/.prefect/storage/events.db; - cp /etc/d2e/tls/ca.pem /usr/local/share/ca-certificates/d2e-internal-ca.crt; + if [ ! -s /etc/d2e/tls/ca.pem ]; then echo "FATAL: /etc/d2e/tls/ca.pem is missing or empty -- the d2e-tls volume is not mounted, or global.secrets.TLS__INTERNAL__CA_CRT is empty. Refusing to start without the internal trust anchor." >&2; exit 1; fi; + grep -q '^-----BEGIN CERTIFICATE-----$' /etc/d2e/tls/ca.pem || { echo "FATAL: /etc/d2e/tls/ca.pem is not a PEM certificate. Refusing to start untrusted." >&2; exit 1; }; + cp /etc/d2e/tls/ca.pem /usr/local/share/ca-certificates/d2e-internal-ca.crt || { echo "FATAL: could not install the internal CA into the OS trust store." >&2; exit 1; }; update-ca-certificates; + openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt /etc/d2e/tls/ca.pem >/dev/null 2>&1 || { echo "FATAL: the internal CA is not present in /etc/ssl/certs/ca-certificates.crt after update-ca-certificates. SSL_CERT_FILE/REQUESTS_CA_BUNDLE point at that bundle, so every Python HTTP stack would fail verification. Refusing to start untrusted." >&2; exit 1; }; python -c 'import certifi; open(certifi.where(), "a").write("\n" + open("/etc/d2e/tls/ca.pem").read() + "\n")'; prefect work-pool create "${PREFECT_POOL:-process-pool}" --type ${WORKPOOL_TYPE:-process}; prefect worker start --pool "${PREFECT_POOL:-process-pool}" --type ${WORKPOOL_TYPE:-process} diff --git a/services/alp-logto/Dockerfile b/services/alp-logto/Dockerfile index e4ec70cf7a..02b1e54343 100644 --- a/services/alp-logto/Dockerfile +++ b/services/alp-logto/Dockerfile @@ -1 +1 @@ -FROM ghcr.io/data2evidence/logto-with-logto-schema@sha256:dde8283c3ab072019568881d061674012378b16bfed06672d4de5b7ccef38758 +FROM ghcr.io/data2evidence/logto-with-logto-schema@sha256:b15f139b08744afa7e68bbaa11cd5d33644fcb18cb50cc55ec7175e7c41cbdff