From b400f35c06568ad7b0817b4584a49e7eebca1b73 Mon Sep 17 00:00:00 2001 From: santan maddi Date: Tue, 25 Aug 2026 13:07:08 +0800 Subject: [PATCH 1/6] fix(chart): use curl for the TLS readiness checks, not busybox wget logto-check failed with: wget: unrecognized option '--ca-certificate=/etc/d2e/tls/ca.pem' That flag is GNU wget; the containers run busybox:latest, whose wget offers only --no-check-certificate. So the check could not verify at all, and the only busybox-compatible spelling would have been a verification bypass -- the exact thing this branch removes everywhere else. Both affected init containers now use curl --cacert, which verifies chain and hostname by default, on an image already pulled for their own pod: the trex image for logto-check, the worker image for dependencies-check (its Dockerfile installs curl). No new image pull, no second tag to track. The loops become `until ; do sleep; done` rather than polling and testing for a marker file, which also drops the success-trex/success-dataflow temp files. Verified curl exists and supports --cacert by running both images, rather than assuming it -- assuming a flag existed is what caused this. Co-Authored-By: Claude Opus 5 (1M context) --- charts/d2e-services/templates/d2e-deployment.yaml | 7 +++++-- .../templates/dataflow-worker-deployment.yaml | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/charts/d2e-services/templates/d2e-deployment.yaml b/charts/d2e-services/templates/d2e-deployment.yaml index 0e1ee4c8e8..4eb198afc4 100644 --- a/charts/d2e-services/templates/d2e-deployment.yaml +++ b/charts/d2e-services/templates/d2e-deployment.yaml @@ -123,7 +123,10 @@ spec: cp -rfv /data/plugins/bunfig.toml /usr/src/data/plugins/bunfig.toml && exit 0", ] - name: logto-check - image: busybox:latest + # Not busybox: its wget has no --ca-certificate, only the + # --no-check-certificate bypass. The trex image is already pulled for + # this pod and ships curl, which verifies chain and hostname by default. + image: ghcr.io/ohdsi/d2e-trex:{{ .Values.global.image.tag }} resources: {{- toYaml .Values.global.resources.logtoCheck | nindent 12 }} volumeMounts: @@ -134,7 +137,7 @@ spec: [ "sh", "-c", - "while :;do sleep 20 && wget --ca-certificate=/etc/d2e/tls/ca.pem -O success https://idp.{{ .Release.Namespace }}.svc.cluster.local:443/api/status && if test -f success; then exit 0; fi; done", + "until curl -sf --cacert /etc/d2e/tls/ca.pem https://idp.{{ .Release.Namespace }}.svc.cluster.local:443/api/status >/dev/null; do echo 'waiting for logto ...'; sleep 20; done", ] - name: logto-post-init env: diff --git a/charts/d2e-services/templates/dataflow-worker-deployment.yaml b/charts/d2e-services/templates/dataflow-worker-deployment.yaml index bcbc950593..57c1545b79 100644 --- a/charts/d2e-services/templates/dataflow-worker-deployment.yaml +++ b/charts/d2e-services/templates/dataflow-worker-deployment.yaml @@ -33,12 +33,16 @@ spec: # claimName: trex-pvc initContainers: - name: dependencies-check - image: busybox:latest + # Not busybox: its wget has no --ca-certificate, only the + # --no-check-certificate bypass. The worker image is already pulled for + # this pod and ships curl (see its Dockerfile), which verifies chain and + # hostname by default. + image: ghcr.io/ohdsi/d2e-dataflow-gen-worker:{{ .Values.global.image.tag }} command: [ "sh", "-c", - "while :;do sleep {{ .Values.dataflowWorker.readinessSleep }} && wget --ca-certificate=/etc/d2e/tls/ca.pem -O success-trex https://trex.{{ .Release.Namespace }}.svc.cluster.local:443/trex/_internal/health && wget --ca-certificate=/etc/d2e/tls/ca.pem -O success-dataflow https://dataflow-gen.{{ .Release.Namespace }}.svc.cluster.local:443/d2e/api/health && if test -f success-trex; then if test -f success-dataflow; then exit 0; fi; fi; done", + "until curl -sf --cacert /etc/d2e/tls/ca.pem https://trex.{{ .Release.Namespace }}.svc.cluster.local:443/trex/_internal/health >/dev/null && curl -sf --cacert /etc/d2e/tls/ca.pem https://dataflow-gen.{{ .Release.Namespace }}.svc.cluster.local:443/d2e/api/health >/dev/null; do echo 'waiting for trex and dataflow-gen ...'; sleep {{ .Values.dataflowWorker.readinessSleep }}; done", ] volumeMounts: - mountPath: /etc/d2e/tls From 5e64afc3e96a6e9ed7bff36ca75c8992d524496d Mon Sep 17 00:00:00 2001 From: mwaiyee Date: Tue, 25 Aug 2026 14:44:44 +0800 Subject: [PATCH 2/6] add resource limits --- charts/d2e-services/templates/d2e-deployment.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/d2e-services/templates/d2e-deployment.yaml b/charts/d2e-services/templates/d2e-deployment.yaml index 4eb198afc4..5098c96524 100644 --- a/charts/d2e-services/templates/d2e-deployment.yaml +++ b/charts/d2e-services/templates/d2e-deployment.yaml @@ -90,6 +90,9 @@ spec: requests: cpu: 50m memory: 50Mi + limits: + cpu: 100m + memory: 100Mi volumeMounts: - mountPath: /etc/d2e/tls name: d2e-tls From 02befefca853802ecf21c68ec9905487aac067a9 Mon Sep 17 00:00:00 2001 From: santan maddi Date: Tue, 25 Aug 2026 18:53:23 +0800 Subject: [PATCH 3/6] fix(charts): mount the internal CA into trex and fail loudly without it The trex container copied the internal CA from /etc/d2e/tls/ca.pem and pointed WEBAPI_TRUST_CERTS at the same path, but never mounted the d2e-tls volume -- the only container referencing that path without it. The cp failed, update-ca-certificates had nothing to add, and the process started with public roots only, so every internal TLS call died: UnknownIssuer from rustls (Deno edge functions, DENO_TLS_CA_STORE=system) and PKIX from the GraalVM WebAPI. Nothing caught it. The args have no set -e, so the failed cp did not stop startup; the liveness probe is `curl -fk`, which disables the very check that was broken; and trex has no readiness probe, so the pod reported Ready. CI runs a real `helm install --wait` plus `kubectl wait --for=condition=Ready` and went green throughout. Mount d2e-tls read-only at /etc/d2e/tls, matching the five sibling containers that already do, which fixes the OS-store and WEBAPI_TRUST_CERTS paths together since both read that one file. Then assert instead of hoping: the file is non-empty, is PEM, the copy succeeded, and -- the part that actually matters -- the anchor is present in the rebuilt bundle. update-ca-certificates exits 0 over an empty directory, so "it ran" proves nothing; verifying the self-signed CA against ca-certificates.crt proves membership. Starting untrusted is worse than not starting. Verified in containers: volume absent -> exit 1; valid CA -> "1 added", entrypoint reached; non-PEM -> exit 1; and the old script with the volume absent -> exit 0, started anyway. End to end, curl with neither -k nor --cacert reached idp.d2e.svc.cluster.local and trex.d2e.svc.cluster.local at HTTP 200, ssl_verify_result 0 -- chain and hostname verified from the OS store alone. TLS__EXTRA__CA_CRTS parity with compose is still missing and is left for a separate change. --- .../templates/d2e-deployment.yaml | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/charts/d2e-services/templates/d2e-deployment.yaml b/charts/d2e-services/templates/d2e-deployment.yaml index 4eb198afc4..94aa125ccd 100644 --- a/charts/d2e-services/templates/d2e-deployment.yaml +++ b/charts/d2e-services/templates/d2e-deployment.yaml @@ -444,11 +444,39 @@ spec: # Three runtimes share this container and each finds its trust # anchors somewhere different. The OS store covers shell/curl and, # via DENO_TLS_CA_STORE=system, the Deno edge functions. GraalVM - # WebAPI ignores the OS store entirely and reads WEBAPI_TRUST_CERTS. - # All three are required: a pod-level curl check passes while the - # other two runtimes still fail. - cp /etc/d2e/tls/ca.pem /usr/local/share/ca-certificates/d2e-internal-ca.crt + # WebAPI ignores the OS store entirely and reads WEBAPI_TRUST_CERTS + # (which points at this same file). All three are required: a + # pod-level curl check passes while the other two runtimes fail. + # + # Assert rather than hope. This copy previously ran against a path + # the container did not mount: the cp failed, update-ca-certificates + # had nothing to add, and the process started with public roots only. + # Every internal TLS call then failed -- UnknownIssuer from rustls, + # PKIX from GraalVM -- while the `curl -fk` liveness probe still went + # green, so the pod reported Ready. Starting untrusted is worse than + # not starting, so each step below is checked. + ca=/etc/d2e/tls/ca.pem + if [ ! -s "$ca" ]; then + echo "FATAL: $ca is missing or empty -- the d2e-tls volume is not mounted, or global.secrets.TLS__INTERNAL__CA_CRT is empty. Refusing to start without the internal trust anchor." >&2 + exit 1 + fi + if ! grep -q '^-----BEGIN CERTIFICATE-----$' "$ca"; then + echo "FATAL: $ca is not a PEM certificate. Refusing to start without the internal trust anchor." >&2 + exit 1 + fi + cp "$ca" /usr/local/share/ca-certificates/d2e-internal-ca.crt || { + echo "FATAL: could not install $ca into the OS trust store." >&2 + exit 1 + } update-ca-certificates + # The anchor is self-signed, so verifying it against the freshly + # rebuilt bundle proves it is actually in the store -- not merely + # that the file was copied and the rebuild exited 0. + if ! openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt "$ca" >/dev/null 2>&1; then + echo "FATAL: the internal CA is not present in the OS trust store after update-ca-certificates. Refusing to start untrusted." >&2 + exit 1 + fi + echo "internal CA installed into OS trust store: $(openssl x509 -in "$ca" -noout -subject)" exec /usr/src/entrypoint.sh ${TREX_OTEL_ENABLED:+--enable-otel} env: - name: RUST_BACKTRACE @@ -1173,6 +1201,12 @@ spec: # webapi-init container, which ran the same files through psql. - mountPath: /usr/src/atlas-db-init name: atlas-db-init + # The internal CA, read by the args above (OS trust store, for + # shell/curl and DENO_TLS_CA_STORE=system) and by the GraalVM WebAPI + # via WEBAPI_TRUST_CERTS. Without this mount all three go untrusted. + - mountPath: /etc/d2e/tls + name: d2e-tls + readOnly: true {{- if .Values.features.materializeCohorts }} - mountPath: /usr/src/cert/client.crt name: materialize-cohorts-client-pki From 13dd96c4a0e2ba854df9301b88a6448f05480ff6 Mon Sep 17 00:00:00 2001 From: santan maddi Date: Tue, 25 Aug 2026 23:58:38 +0800 Subject: [PATCH 4/6] fix(charts): point the dataflow worker's Python stacks at the CA bundle Flow runs crashed with ssl.SSLCertVerificationError (CERTIFICATE_VERIFY_FAILED, unable to get local issuer certificate) from prefect/events/clients.py -> websockets, plus a preceding "run-flow: deployment lookup failed" from run-flow.sh's urllib call. docker-compose sets SSL_CERT_FILE and REQUESTS_CA_BUNDLE on alp-dataflow-gen-worker to /etc/ssl/certs/ca-certificates.crt; the chart never got those two lines. Both setups use the same https PREFECT_API_URL, so that was the entire difference -- which is why compose and the DQD run on CI were fine while k8s failed. Without them only clients that explicitly pass certifi.where() are covered (prefect's httpx client, requests). stdlib urlopen and websockets use OpenSSL's defaults, which under `pixi run` resolve inside the env prefix (/ssl/cert.pem, capath empty) and never see the internal CA -- so neither update-ca-certificates nor the certifi append reaches them. Worse, flows exec in their own per-plugin pixi env provisioned at run time, so no startup-time append can ever reach them; an inherited env var can. Also fail loudly, as with trex: assert the CA is present, is PEM, copies, and actually lands in /etc/ssl/certs/ca-certificates.crt. That last check matters more here than anywhere else, because the two new env vars point every Python HTTP stack at that bundle -- if the anchor is missing from it, verification fails everywhere. Verified in ghcr.io/ohdsi/d2e-dataflow-gen-worker:develop against a server presenting the real leaf. With the chart's rendered args and env: stdlib urlopen, requests and websockets all pass. Guard cases: volume absent -> exit 1, non-PEM -> exit 1, valid CA -> proceeds. Most directly, in /var/lib/d2e-flows/d2e-flows/baked/.pixi/envs/default -- the exact env from the traceback -- the crash reproduces without the env vars and is gone with them. --- .../templates/dataflow-worker-deployment.yaml | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/charts/d2e-services/templates/dataflow-worker-deployment.yaml b/charts/d2e-services/templates/dataflow-worker-deployment.yaml index 57c1545b79..ae5ae5e577 100644 --- a/charts/d2e-services/templates/dataflow-worker-deployment.yaml +++ b/charts/d2e-services/templates/dataflow-worker-deployment.yaml @@ -305,6 +305,21 @@ spec: value: "10" - name: ROUNDING_TARGET value: "10" + # Trust the internal self-signed CA across every Python HTTP stack, + # matching docker-compose (alp-dataflow-gen-worker). The args above + # put the CA into the OS bundle; these point urllib/requests/httpx at + # that bundle. Without them only clients that explicitly pass + # certifi.where() are covered -- stdlib urlopen (run-flow.sh's + # deployment lookup) and websockets (the prefect events subscriber) + # read OpenSSL's defaults, which under `pixi run` resolve inside the + # env prefix (/ssl/cert.pem) and never see the CA. Flows also + # exec in their own per-plugin pixi env provisioned at run time, so a + # startup-time append to this env's certifi cannot reach them; an + # inherited env var can. + - name: SSL_CERT_FILE + value: /etc/ssl/certs/ca-certificates.crt + - name: REQUESTS_CA_BUNDLE + value: /etc/ssl/certs/ca-certificates.crt command: - /entrypoint.sh - pixi @@ -316,8 +331,11 @@ spec: args: - >- rm -rf /root/.prefect/storage/run_cache.db /root/.prefect/storage/events.db; - cp /etc/d2e/tls/ca.pem /usr/local/share/ca-certificates/d2e-internal-ca.crt; + if [ ! -s /etc/d2e/tls/ca.pem ]; then echo "FATAL: /etc/d2e/tls/ca.pem is missing or empty -- the d2e-tls volume is not mounted, or global.secrets.TLS__INTERNAL__CA_CRT is empty. Refusing to start without the internal trust anchor." >&2; exit 1; fi; + grep -q '^-----BEGIN CERTIFICATE-----$' /etc/d2e/tls/ca.pem || { echo "FATAL: /etc/d2e/tls/ca.pem is not a PEM certificate. Refusing to start untrusted." >&2; exit 1; }; + cp /etc/d2e/tls/ca.pem /usr/local/share/ca-certificates/d2e-internal-ca.crt || { echo "FATAL: could not install the internal CA into the OS trust store." >&2; exit 1; }; update-ca-certificates; + openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt /etc/d2e/tls/ca.pem >/dev/null 2>&1 || { echo "FATAL: the internal CA is not present in /etc/ssl/certs/ca-certificates.crt after update-ca-certificates. SSL_CERT_FILE/REQUESTS_CA_BUNDLE point at that bundle, so every Python HTTP stack would fail verification. Refusing to start untrusted." >&2; exit 1; }; python -c 'import certifi; open(certifi.where(), "a").write("\n" + open("/etc/d2e/tls/ca.pem").read() + "\n")'; prefect work-pool create "${PREFECT_POOL:-process-pool}" --type ${WORKPOOL_TYPE:-process}; prefect worker start --pool "${PREFECT_POOL:-process-pool}" --type ${WORKPOOL_TYPE:-process} From e137ab0d0f6d38430734e0d27dd5af2b3a25551d Mon Sep 17 00:00:00 2001 From: santan maddi Date: Wed, 26 Aug 2026 11:07:59 +0800 Subject: [PATCH 5/6] update logto image --- services/alp-logto/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/alp-logto/Dockerfile b/services/alp-logto/Dockerfile index e4ec70cf7a..02b1e54343 100644 --- a/services/alp-logto/Dockerfile +++ b/services/alp-logto/Dockerfile @@ -1 +1 @@ -FROM ghcr.io/data2evidence/logto-with-logto-schema@sha256:dde8283c3ab072019568881d061674012378b16bfed06672d4de5b7ccef38758 +FROM ghcr.io/data2evidence/logto-with-logto-schema@sha256:b15f139b08744afa7e68bbaa11cd5d33644fcb18cb50cc55ec7175e7c41cbdff From 80458384a1ab1dd52c906d6f01ace600074916a0 Mon Sep 17 00:00:00 2001 From: santan maddi Date: Thu, 27 Aug 2026 10:19:12 +0800 Subject: [PATCH 6/6] fix(charts): serve the internal certificate for the gateway's cluster name Caddy had no certificate for gateway..svc.cluster.local and aborted the handshake ('no certificate available' -> 'Received fatal alert: internal_error'). WebAPI dials that name for the jwks_uri advertised by OIDC discovery, so its OIDC bean failed to build and every analytics-svc call proxying to WebAPI returned 500. Caddy is the server on this hop, so no client-side trust setting helps; it already trusts the internal CA as a client for its upstreams. Add the internal cert/key to its d2e-tls mount and a site block serving that name, with routes mirroring the existing :8080 block. SERVER__CRT is the environment's public certificate and has no cluster-internal SANs; CI hid this by using one self-signed cert for both. Verified: caddy validate passes on caddy:2.11-alpine with two independent SNI policies; deployed and confirmed from trex that the jwks_uri returns HTTP 200 with tls_verify=0. The rendered Caddyfile is byte-identical to the running ConfigMap. --- .../templates/core-caddy-deployment.yaml | 23 +++++++++-- .../d2e-core/templates/core-config.yaml | 41 +++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/charts/d2e-services/charts/d2e-core/templates/core-caddy-deployment.yaml b/charts/d2e-services/charts/d2e-core/templates/core-caddy-deployment.yaml index 89748a7ccd..acf1caa150 100644 --- a/charts/d2e-services/charts/d2e-core/templates/core-caddy-deployment.yaml +++ b/charts/d2e-services/charts/d2e-core/templates/core-caddy-deployment.yaml @@ -95,10 +95,18 @@ spec: - name: caddy-local-pki secret: secretName: "{{ .Release.Name }}-{{ .Chart.Name }}-caddy" - # Internal CA for `tls_trust_pool file` on the trex upstreams. Note this - # is the chart-level secret, not the -caddy one above: the -caddy secret - # holds Caddy's own edge PKI, not the internal CA. CA only -- Caddy is a - # client on these hops, never the server. + # Internal TLS material from the chart-level secret, not the -caddy one + # above: the -caddy secret holds Caddy's own edge PKI (SERVER__CRT, the + # environment's real public certificate), not the internal CA. + # + # ca.pem -- Caddy as a *client*, for `tls_trust_pool file` when proxying + # to the trex/idp upstreams. + # cert.pem/key.pem -- Caddy as a *server*, for the + # https://gateway..svc.cluster.local site block. Internal + # callers (WebAPI fetching the advertised jwks_uri) dial the + # gateway by its cluster name, which SERVER__CRT does not + # cover; without a certificate for it Caddy aborts the + # handshake and WebAPI's OIDC bean fails to build. - name: d2e-tls secret: secretName: "{{ .Release.Name }}-{{ .Chart.Name }}" @@ -106,3 +114,10 @@ spec: - key: TLS__INTERNAL__CA_CRT path: ca.pem mode: 0444 + - key: TLS__INTERNAL__CRT + path: cert.pem + mode: 0444 + # Owner-only: this is a private key, and Caddy runs as root here. + - key: TLS__INTERNAL__KEY + path: key.pem + mode: 0400 diff --git a/charts/d2e-services/charts/d2e-core/templates/core-config.yaml b/charts/d2e-services/charts/d2e-core/templates/core-config.yaml index e81f9133e4..4e0b050e9b 100644 --- a/charts/d2e-services/charts/d2e-core/templates/core-config.yaml +++ b/charts/d2e-services/charts/d2e-core/templates/core-config.yaml @@ -147,6 +147,47 @@ data: } } + # Internal callers reach the gateway by its in-cluster Service name over TLS. + # Logto advertises jwks_uri as https://gateway..svc.cluster.local/oidc/jwks + # -- endpoint URLs follow the TLS hop into idp (and X-Forwarded-Proto https), + # while `issuer` stays the plaintext ENDPOINT. WebAPI reads that jwks_uri and + # dials it, so this name must be servable over 443. + # + # Without this block nothing on 443 matches the cluster name: the only other + # https site is {$CADDY__D2E__INTERNAL_FQDN}, so Caddy finds no certificate + # and aborts with an internal_error alert one message after Client Hello -- + # before any certificate is offered, which is why `curl -k` fails identically. + # WebAPI then cannot build OidcAuthConfig$OpenidDirect, returns 500, and every + # analytics-svc call that proxies to WebAPI 500s with it. + # + # It serves the internal certificate, not SERVER__CRT: SERVER__CRT is the + # environment's real public certificate and has no reason to carry + # cluster-internal SANs. (CI hides this by feeding the same self-signed cert + # to both SERVER__CRT and TLS__INTERNAL__CRT, so its gateway happens to cover + # the cluster name.) Routes mirror the :8080 block above so internal callers + # get the same behaviour over either scheme. + https://gateway.{{ .Release.Namespace }}.svc.cluster.local { + tls /etc/d2e/tls/cert.pem /etc/d2e/tls/key.pem + + @oidc { + path /oidc/* + } + handle @oidc { + reverse_proxy https://idp.{{ .Release.Namespace }}.svc.cluster.local { + transport http { + tls + tls_trust_pool file /etc/d2e/tls/ca.pem + } + } + } + handle /check-liveness* { + respond "OK-INTERNAL-TLS" + } + handle { + respond 404 + } + } + https://localhost { tls { issuer internal {