fix: TLS Fixes in helm chart

charts/d2e-services/charts/d2e-core/templates/core-caddy-deployment.yaml

@@ -95,14 +95,29 @@ spec:
         - name: caddy-local-pki
           secret:
             secretName: "{{ .Release.Name }}-{{ .Chart.Name }}-caddy"
-        # Internal CA for `tls_trust_pool file` on the trex upstreams. Note this
-        # is the chart-level secret, not the -caddy one above: the -caddy secret
-        # holds Caddy's own edge PKI, not the internal CA. CA only -- Caddy is a
-        # client on these hops, never the server.
+        # Internal TLS material from the chart-level secret, not the -caddy one
+        # above: the -caddy secret holds Caddy's own edge PKI (SERVER__CRT, the
+        # environment's real public certificate), not the internal CA.
+        #
+        # ca.pem  -- Caddy as a *client*, for `tls_trust_pool file` when proxying
+        #            to the trex/idp upstreams.
+        # cert.pem/key.pem -- Caddy as a *server*, for the
+        #            https://gateway.<ns>.svc.cluster.local site block. Internal
+        #            callers (WebAPI fetching the advertised jwks_uri) dial the
+        #            gateway by its cluster name, which SERVER__CRT does not
+        #            cover; without a certificate for it Caddy aborts the
+        #            handshake and WebAPI's OIDC bean fails to build.
         - name: d2e-tls
           secret:
             secretName: "{{ .Release.Name }}-{{ .Chart.Name }}"
             items:
               - key: TLS__INTERNAL__CA_CRT
                 path: ca.pem
                 mode: 0444
+              - key: TLS__INTERNAL__CRT
+                path: cert.pem
+                mode: 0444
+              # Owner-only: this is a private key, and Caddy runs as root here.
+              - key: TLS__INTERNAL__KEY
+                path: key.pem
+                mode: 0400

charts/d2e-services/charts/d2e-core/templates/core-config.yaml

@@ -147,6 +147,47 @@ data:
         }
     }
 
+    # Internal callers reach the gateway by its in-cluster Service name over TLS.
+    # Logto advertises jwks_uri as https://gateway.<ns>.svc.cluster.local/oidc/jwks
+    # -- endpoint URLs follow the TLS hop into idp (and X-Forwarded-Proto https),
+    # while `issuer` stays the plaintext ENDPOINT. WebAPI reads that jwks_uri and
+    # dials it, so this name must be servable over 443.
+    #
+    # Without this block nothing on 443 matches the cluster name: the only other
+    # https site is {$CADDY__D2E__INTERNAL_FQDN}, so Caddy finds no certificate
+    # and aborts with an internal_error alert one message after Client Hello --
+    # before any certificate is offered, which is why `curl -k` fails identically.
+    # WebAPI then cannot build OidcAuthConfig$OpenidDirect, returns 500, and every
+    # analytics-svc call that proxies to WebAPI 500s with it.
+    #
+    # It serves the internal certificate, not SERVER__CRT: SERVER__CRT is the
+    # environment's real public certificate and has no reason to carry
+    # cluster-internal SANs. (CI hides this by feeding the same self-signed cert
+    # to both SERVER__CRT and TLS__INTERNAL__CRT, so its gateway happens to cover
+    # the cluster name.) Routes mirror the :8080 block above so internal callers
+    # get the same behaviour over either scheme.
+    https://gateway.{{ .Release.Namespace }}.svc.cluster.local {
+        tls /etc/d2e/tls/cert.pem /etc/d2e/tls/key.pem
+
+        @oidc {
+            path /oidc/*
+        }
+        handle @oidc {
+            reverse_proxy https://idp.{{ .Release.Namespace }}.svc.cluster.local {
+            	transport http {
+                    tls
+                    tls_trust_pool file /etc/d2e/tls/ca.pem
+                }
+            }
+        }
+        handle /check-liveness* {
+            respond "OK-INTERNAL-TLS"
+        }
+        handle {
+            respond 404
+        }
+    }
+
     https://localhost {
        tls {
             issuer internal {

charts/d2e-services/templates/d2e-deployment.yaml

@@ -90,6 +90,9 @@ spec:
             requests:
               cpu: 50m
               memory: 50Mi
+            limits: 
+              cpu: 100m
+              memory: 100Mi
           volumeMounts:
             - mountPath: /etc/d2e/tls
               name: d2e-tls
@@ -123,7 +126,10 @@ spec:
               cp -rfv /data/plugins/bunfig.toml /usr/src/data/plugins/bunfig.toml && exit 0",
             ]
         - name: logto-check
-          image: busybox:latest
+          # Not busybox: its wget has no --ca-certificate, only the
+          # --no-check-certificate bypass. The trex image is already pulled for
+          # this pod and ships curl, which verifies chain and hostname by default.
+          image: ghcr.io/ohdsi/d2e-trex:{{ .Values.global.image.tag }}
           resources:
             {{- toYaml .Values.global.resources.logtoCheck | nindent 12 }}
           volumeMounts:
@@ -134,7 +140,7 @@ spec:
             [
               "sh",
               "-c",
-              "while :;do sleep 20 && wget --ca-certificate=/etc/d2e/tls/ca.pem -O success https://idp.{{ .Release.Namespace }}.svc.cluster.local:443/api/status && if test -f success; then exit 0; fi; done",
+              "until curl -sf --cacert /etc/d2e/tls/ca.pem https://idp.{{ .Release.Namespace }}.svc.cluster.local:443/api/status >/dev/null; do echo 'waiting for logto ...'; sleep 20; done",
             ]
         - name: logto-post-init
           env:
@@ -441,11 +447,39 @@ spec:
               # Three runtimes share this container and each finds its trust
               # anchors somewhere different. The OS store covers shell/curl and,
               # via DENO_TLS_CA_STORE=system, the Deno edge functions. GraalVM
-              # WebAPI ignores the OS store entirely and reads WEBAPI_TRUST_CERTS.
-              # All three are required: a pod-level curl check passes while the
-              # other two runtimes still fail.
-              cp /etc/d2e/tls/ca.pem /usr/local/share/ca-certificates/d2e-internal-ca.crt
+              # WebAPI ignores the OS store entirely and reads WEBAPI_TRUST_CERTS
+              # (which points at this same file). All three are required: a
+              # pod-level curl check passes while the other two runtimes fail.
+              #
+              # Assert rather than hope. This copy previously ran against a path
+              # the container did not mount: the cp failed, update-ca-certificates
+              # had nothing to add, and the process started with public roots only.
+              # Every internal TLS call then failed -- UnknownIssuer from rustls,
+              # PKIX from GraalVM -- while the `curl -fk` liveness probe still went
+              # green, so the pod reported Ready. Starting untrusted is worse than
+              # not starting, so each step below is checked.
+              ca=/etc/d2e/tls/ca.pem
+              if [ ! -s "$ca" ]; then
+                echo "FATAL: $ca is missing or empty -- the d2e-tls volume is not mounted, or global.secrets.TLS__INTERNAL__CA_CRT is empty. Refusing to start without the internal trust anchor." >&2
+                exit 1
+              fi
+              if ! grep -q '^-----BEGIN CERTIFICATE-----$' "$ca"; then
+                echo "FATAL: $ca is not a PEM certificate. Refusing to start without the internal trust anchor." >&2
+                exit 1
+              fi
+              cp "$ca" /usr/local/share/ca-certificates/d2e-internal-ca.crt || {
+                echo "FATAL: could not install $ca into the OS trust store." >&2
+                exit 1
+              }
               update-ca-certificates
+              # The anchor is self-signed, so verifying it against the freshly
+              # rebuilt bundle proves it is actually in the store -- not merely
+              # that the file was copied and the rebuild exited 0.
+              if ! openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt "$ca" >/dev/null 2>&1; then
+                echo "FATAL: the internal CA is not present in the OS trust store after update-ca-certificates. Refusing to start untrusted." >&2
+                exit 1
+              fi
+              echo "internal CA installed into OS trust store: $(openssl x509 -in "$ca" -noout -subject)"
               exec /usr/src/entrypoint.sh ${TREX_OTEL_ENABLED:+--enable-otel}
           env:
             - name: RUST_BACKTRACE
@@ -1170,6 +1204,12 @@ spec:
             # webapi-init container, which ran the same files through psql.
             - mountPath: /usr/src/atlas-db-init
               name: atlas-db-init
+            # The internal CA, read by the args above (OS trust store, for
+            # shell/curl and DENO_TLS_CA_STORE=system) and by the GraalVM WebAPI
+            # via WEBAPI_TRUST_CERTS. Without this mount all three go untrusted.
+            - mountPath: /etc/d2e/tls
+              name: d2e-tls
+              readOnly: true
             {{- if .Values.features.materializeCohorts }}
             - mountPath: /usr/src/cert/client.crt
               name: materialize-cohorts-client-pki

charts/d2e-services/templates/dataflow-worker-deployment.yaml

@@ -33,12 +33,16 @@ spec:
       #       claimName: trex-pvc
       initContainers:
         - name: dependencies-check
-          image: busybox:latest
+          # Not busybox: its wget has no --ca-certificate, only the
+          # --no-check-certificate bypass. The worker image is already pulled for
+          # this pod and ships curl (see its Dockerfile), which verifies chain and
+          # hostname by default.
+          image: ghcr.io/ohdsi/d2e-dataflow-gen-worker:{{ .Values.global.image.tag }}
           command:
             [
               "sh",
               "-c",
-              "while :;do sleep {{ .Values.dataflowWorker.readinessSleep }} && wget --ca-certificate=/etc/d2e/tls/ca.pem -O success-trex https://trex.{{ .Release.Namespace }}.svc.cluster.local:443/trex/_internal/health && wget --ca-certificate=/etc/d2e/tls/ca.pem -O success-dataflow https://dataflow-gen.{{ .Release.Namespace }}.svc.cluster.local:443/d2e/api/health && if test -f success-trex; then if test -f success-dataflow; then exit 0; fi; fi; done",
+              "until curl -sf --cacert /etc/d2e/tls/ca.pem https://trex.{{ .Release.Namespace }}.svc.cluster.local:443/trex/_internal/health >/dev/null && curl -sf --cacert /etc/d2e/tls/ca.pem https://dataflow-gen.{{ .Release.Namespace }}.svc.cluster.local:443/d2e/api/health >/dev/null; do echo 'waiting for trex and dataflow-gen ...'; sleep {{ .Values.dataflowWorker.readinessSleep }}; done",
             ]
           volumeMounts:
             - mountPath: /etc/d2e/tls
@@ -301,6 +305,21 @@ spec:
               value: "10"
             - name: ROUNDING_TARGET
               value: "10"
+            # Trust the internal self-signed CA across every Python HTTP stack,
+            # matching docker-compose (alp-dataflow-gen-worker). The args above
+            # put the CA into the OS bundle; these point urllib/requests/httpx at
+            # that bundle. Without them only clients that explicitly pass
+            # certifi.where() are covered -- stdlib urlopen (run-flow.sh's
+            # deployment lookup) and websockets (the prefect events subscriber)
+            # read OpenSSL's defaults, which under `pixi run` resolve inside the
+            # env prefix (<env>/ssl/cert.pem) and never see the CA. Flows also
+            # exec in their own per-plugin pixi env provisioned at run time, so a
+            # startup-time append to this env's certifi cannot reach them; an
+            # inherited env var can.
+            - name: SSL_CERT_FILE
+              value: /etc/ssl/certs/ca-certificates.crt
+            - name: REQUESTS_CA_BUNDLE
+              value: /etc/ssl/certs/ca-certificates.crt
           command:
             - /entrypoint.sh
             - pixi
@@ -312,8 +331,11 @@ spec:
           args:
             - >-
               rm -rf /root/.prefect/storage/run_cache.db /root/.prefect/storage/events.db;
-              cp /etc/d2e/tls/ca.pem /usr/local/share/ca-certificates/d2e-internal-ca.crt;
+              if [ ! -s /etc/d2e/tls/ca.pem ]; then echo "FATAL: /etc/d2e/tls/ca.pem is missing or empty -- the d2e-tls volume is not mounted, or global.secrets.TLS__INTERNAL__CA_CRT is empty. Refusing to start without the internal trust anchor." >&2; exit 1; fi;
+              grep -q '^-----BEGIN CERTIFICATE-----$' /etc/d2e/tls/ca.pem || { echo "FATAL: /etc/d2e/tls/ca.pem is not a PEM certificate. Refusing to start untrusted." >&2; exit 1; };
+              cp /etc/d2e/tls/ca.pem /usr/local/share/ca-certificates/d2e-internal-ca.crt || { echo "FATAL: could not install the internal CA into the OS trust store." >&2; exit 1; };
               update-ca-certificates;
+              openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt /etc/d2e/tls/ca.pem >/dev/null 2>&1 || { echo "FATAL: the internal CA is not present in /etc/ssl/certs/ca-certificates.crt after update-ca-certificates. SSL_CERT_FILE/REQUESTS_CA_BUNDLE point at that bundle, so every Python HTTP stack would fail verification. Refusing to start untrusted." >&2; exit 1; };
               python -c 'import certifi; open(certifi.where(), "a").write("\n" + open("/etc/d2e/tls/ca.pem").read() + "\n")';
               prefect work-pool create "${PREFECT_POOL:-process-pool}" --type ${WORKPOOL_TYPE:-process};
               prefect worker start --pool "${PREFECT_POOL:-process-pool}" --type ${WORKPOOL_TYPE:-process}

services/alp-logto/Dockerfile

@@ -1 +1 @@
-FROM ghcr.io/data2evidence/logto-with-logto-schema@sha256:dde8283c3ab072019568881d061674012378b16bfed06672d4de5b7ccef38758
+FROM ghcr.io/data2evidence/logto-with-logto-schema@sha256:b15f139b08744afa7e68bbaa11cd5d33644fcb18cb50cc55ec7175e7c41cbdff