[Feature]: Merge flip-omop-db into the monorepo as the trust/omop-db build source

.github/workflows/docker_build_omop_db.yml

@@ -0,0 +1,146 @@
+# Copyright (c) Guy's and St Thomas' NHS Foundation Trust & King's College London
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Build and Push Docker Image for OMOP DB
+
+on:
+  workflow_dispatch:
+  # Publish only after the omop-db test suite ("Trust - OMOP DB CI") completes
+  # on main/develop; the job-level `if` below gates on its success. The image
+  # is vocab-free (FLIP#842) — nothing licensed enters the build context, which
+  # is what makes this CI build possible at all.
+  workflow_run:
+    workflows: ["Trust - OMOP DB CI"]
+    types: [completed]
+    branches: [main, develop]
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-push:
+    # Only publish when the upstream test run passed. workflow_dispatch (manual)
+    # has no workflow_run context, so allow it through unconditionally.
+    # The repository guard skips publishing on forks, which cannot push to
+    # ghcr.io/londonaicentre and would otherwise show a spurious CI failure.
+    if: >-
+      (github.event_name == 'workflow_dispatch' ||
+      github.event.workflow_run.conclusion == 'success') &&
+      github.repository == 'londonaicentre/FLIP'
+    runs-on: ubuntu-latest
+    permissions: # override top-level read-only default to allow GHCR push
+      contents: read
+      packages: write
+    defaults:
+      run:
+        working-directory: ./trust/omop-db
+    env:
+      REGISTRY: ghcr.io
+      IMAGE_NAME: londonaicentre/omop-db
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          # workflow_run.* is empty on manual workflow_dispatch runs, so fall
+          # back to the dispatched ref so manual builds check out correctly.
+          repository: ${{ github.event.workflow_run.head_repository.full_name || github.repository }}
+          ref: ${{ github.event.workflow_run.head_sha || github.sha }}
+
+      - name: Determine tags
+        id: tags
+        env:
+          GH_REF_NAME: ${{ github.ref_name }}
+          GH_EVENT_NAME: ${{ github.event_name }}
+          GH_REF: ${{ github.ref }}
+          GH_SHA: ${{ github.sha }}
+          GH_WR_BRANCH: ${{ github.event.workflow_run.head_branch }}
+          GH_WR_EVENT: ${{ github.event.workflow_run.event }}
+          GH_WR_SHA: ${{ github.event.workflow_run.head_sha }}
+        run: |
+          # On a workflow_run trigger, github.sha / github.ref_name point at the
+          # default branch (where this workflow definition lives), NOT the commit
+          # that was tested and checked out. Use the triggering run's head_sha /
+          # head_branch so the :<sha> and :<branch> tags match the built code.
+          if [[ "$GH_EVENT_NAME" == "workflow_run" ]]; then
+            SHA="$GH_WR_SHA"
+            REF_NAME="$GH_WR_BRANCH"
+          else
+            SHA="$GH_SHA"
+            REF_NAME="$GH_REF_NAME"
+          fi
+
+          TAGS="${REGISTRY}/${IMAGE_NAME}:${SHA}"
+
+          # An empty sha would silently publish a mutable literal `sha-` tag — refuse.
+          [[ -n "$SHA" ]] || { echo "::error::empty commit SHA — cannot compute the sha-<short7> tag"; exit 1; }
+
+          # Immutable short-SHA tag (FLIP#751), pushed uniformly on every publish.
+          # Length 7 must match the tag resolution in deploy/providers/AWS/Makefile.
+          TAGS="${TAGS},${REGISTRY}/${IMAGE_NAME}:sha-${SHA:0:7}"
+
+          # Branch name sanitization
+          SAFE_REF_NAME=$(echo "$REF_NAME" | sed 's/[^a-zA-Z0-9]/-/g')
+          TAGS="${TAGS},${REGISTRY}/${IMAGE_NAME}:${SAFE_REF_NAME}"
+
+          # Branch number (if starts with number)
+          if [[ "$REF_NAME" =~ ^[0-9]+ ]]; then
+              BRANCH_NUM=$(echo "$REF_NAME" | grep -oE '^[0-9]+')
+              TAGS="${TAGS},${REGISTRY}/${IMAGE_NAME}:${BRANCH_NUM}"
+          fi
+
+          # PR Number (if PR)
+          if [[ "$GH_EVENT_NAME" == "pull_request" ]]; then
+             PR_NUMBER=$(echo "$GH_REF" | awk -F / '{print $3}')
+             TAGS="${TAGS},${REGISTRY}/${IMAGE_NAME}:pr-${PR_NUMBER}"
+          fi
+
+          # Determine if this is a merge/push to main or develop
+          BRANCH_NAME=""
+          if [[ "$GH_EVENT_NAME" == "workflow_run" && "$GH_WR_EVENT" == "push" ]]; then
+            BRANCH_NAME="$GH_WR_BRANCH"
+          elif [[ "$GH_EVENT_NAME" == "push" ]]; then
+            BRANCH_NAME="$GH_REF_NAME"
+          fi
+
+          if [[ "$BRANCH_NAME" == "main" ]]; then
+            TAGS="${TAGS},${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:prod,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest"
+          elif [[ "$BRANCH_NAME" == "develop" ]]; then
+            TAGS="${TAGS},${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:stag"
+          fi
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+          echo "Generated tags: ${TAGS}"
+
+      - name: Log in to GitHub Container Registry
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_ACTOR: ${{ github.actor }}
+        run: echo "$GH_TOKEN" | docker login $REGISTRY -u "$GH_ACTOR" --password-stdin
+
+      - name: Build Docker image
+        env:
+          DOCKER_TAGS: ${{ steps.tags.outputs.tags }}
+        run: |
+          IFS=',' read -ra TAG_ARRAY <<< "$DOCKER_TAGS"
+          TAG_FLAGS=()
+          for tag in "${TAG_ARRAY[@]}"; do
+            TAG_FLAGS+=("-t" "$tag")
+          done
+          docker build "${TAG_FLAGS[@]}" .
+
+      - name: Push Docker image
+        env:
+          DOCKER_TAGS: ${{ steps.tags.outputs.tags }}
+        run: |
+          IFS=',' read -ra TAG_ARRAY <<< "$DOCKER_TAGS"
+          for tag in "${TAG_ARRAY[@]}"; do
+            docker push "$tag"
+          done

.github/workflows/secret-scanning.yml

@@ -122,9 +122,14 @@ jobs:
               run: pip install uv==0.11.16
 
             - name: Verify uv lockfiles (drift + exclude-newer window)
+              # Keep this list in step with the other two copies of it: UV_PROJECTS
+              # in the root Makefile and the per-project uv-lock hooks in
+              # .pre-commit-config.yaml. A project missing here still gets the
+              # commit-time hook, but loses the CI backstop that catches a
+              # widened exclude-newer pushed with --no-verify.
               run: |
                   set -euo pipefail
-                  for dir in . flip-api docs trust/trust-api trust/imaging-api trust/data-access-api trust/xnat/tests deploy/providers/AWS; do
+                  for dir in . flip-api docs trust/trust-api trust/imaging-api trust/data-access-api trust/omop-db trust/xnat/tests deploy/providers/AWS; do
                     echo "::group::uv lock --check ($dir)"
                     uv lock --check --project "$dir"
                     echo "::endgroup::"

.github/workflows/test_helm_chart.yml

@@ -101,6 +101,112 @@ jobs:
             --set omopDb.external.host=test.example.com \
             > /dev/null
 
+      # omopDb.vocabLoad.s3Bucket defaults to "" (the licensed bundle has no public
+      # mirror — FLIP#842/843), so every other render in this job skips the
+      # vocab-load Job. Without this step its ~110-line body is never rendered in
+      # CI and a broken .Values path there would merge green, only to fail the
+      # WHOLE Helm release of the first operator who configures a bucket.
+      - name: Render template (vocab-load enabled)
+        run: |
+          helm template trust-release deploy/providers/kubernetes/ \
+            --set omopDb.vocabLoad.s3Bucket=test-bucket > /tmp/vocab.yaml
+          if ! grep -q "omop-vocab-load" /tmp/vocab.yaml; then
+            echo "::error::the vocab-load Job did not render with s3Bucket set — its body is now untested in CI"
+            exit 1
+          fi
+
+      # The probe stage is what keeps a routine `helm upgrade` from re-downloading
+      # the multi-GB bundle into a fresh emptyDir. Drop it and nothing breaks
+      # loudly — the Job still loads correctly, just slowly, on the critical path
+      # of a release hook. Scoped with --show-only so these strings have to appear
+      # in THIS Job and not merely somewhere in the release.
+      - name: Vocab-load probes before it fetches
+        run: |
+          helm template trust-release deploy/providers/kubernetes/ \
+            --set omopDb.vocabLoad.s3Bucket=test-bucket \
+            --show-only templates/omop-db-vocab-load-job.yaml > /tmp/vocabjob.yaml
+          # Every pattern is passed with -e: three of these begin with "-" and
+          # would otherwise be parsed as grep options rather than as patterns.
+          # fetch-bundle is asserted here as well as below, so that the ordering
+          # check cannot pass vacuously on an empty line number if it is renamed.
+          # load-vocab and its loader invocation are asserted because none of the
+          # probe/fetch strings would notice a Job that probes, fetches, and then
+          # never loads — which is green everywhere else and leaves empty cohorts.
+          #
+          # The marker patterns deliberately carry their whole `if` line rather
+          # than just the path. Asserting the bare token `-f /vocab-work/...`
+          # matches `! -f /vocab-work/...` too, so inverting the fetch guard —
+          # which makes every upgrade re-download the bundle, silently, the exact
+          # regression this step exists to catch — passed the earlier form. Same
+          # for the touch: asserted alone it survives `--check || true; touch`,
+          # which writes the marker unconditionally and only surfaces on the next
+          # fresh install, as a failed release.
+          for want in \
+            '- name: probe-vocab' \
+            '- name: fetch-bundle' \
+            '- name: load-vocab' \
+            'if /flip/omop/load_core_vocab.sh --check; then' \
+            '/flip/omop/load_core_vocab.sh "/vocab-work/' \
+            'touch /vocab-work/.vocab-present' \
+            'if [ -f /vocab-work/.vocab-present ]; then'
+          do
+            if ! grep -qF -e "${want}" /tmp/vocabjob.yaml; then
+              echo "::error::vocab-load Job no longer probes before fetching (missing: ${want})"
+              exit 1
+            fi
+          done
+          # The probe is only worth anything if it runs BEFORE the fetch.
+          probe_at=$(grep -nF -m1 -e '- name: probe-vocab' /tmp/vocabjob.yaml | cut -d: -f1)
+          fetch_at=$(grep -nF -m1 -e '- name: fetch-bundle' /tmp/vocabjob.yaml | cut -d: -f1)
+          if [ "${probe_at}" -gt "${fetch_at}" ]; then
+            echo "::error::probe-vocab must be ordered before fetch-bundle in initContainers"
+            exit 1
+          fi
+          # unzip is baked into the omop-db image precisely so this Job never
+          # installs anything at run time: S3 must be the only host it needs to
+          # reach, or a trust behind an egress allowlist cannot load a vocabulary.
+          # The other half of that contract — that the image really does ship
+          # unzip — is asserted in test_trust_omop_db.yml, which is where it can
+          # actually fire: THIS workflow's pull_request trigger is path-filtered
+          # to deploy/providers/kubernetes/**, so it does not run for a change to
+          # the Dockerfile it would be guarding.
+          if grep -qE 'microdnf|dnf |yum |apt-get |apt |apk add|pip install' /tmp/vocabjob.yaml; then
+            echo "::error::vocab-load Job installs packages at run time again — that needs egress to a package mirror"
+            exit 1
+          fi
+          # All three containers share the work dir. Drop the mount from
+          # fetch-bundle and its skip test reads a path that cannot exist, so
+          # every upgrade silently re-downloads the bundle it just skipped.
+          mounts=$(grep -cF -e 'mountPath: /vocab-work' /tmp/vocabjob.yaml)
+          if [ "${mounts}" -ne 3 ]; then
+            echo "::error::expected /vocab-work mounted in all 3 containers, found ${mounts}"
+            exit 1
+          fi
+          # A deleted resources value renders as `resources:` + `null`, which
+          # drops the ephemeral-storage bound the work dir relies on.
+          if grep -A1 -E '^\s*resources:' /tmp/vocabjob.yaml | grep -qE '^\s*null\s*$'; then
+            echo "::error::a vocabLoad resources block rendered null — the emptyDir would be bounded only by the node"
+            exit 1
+          fi
+
+      # Asserted with an explicit `if`, not `! grep -q`: `set -e` exempts a
+      # negated command, so that form only fails the step by virtue of being the
+      # LAST line — appending so much as a debug echo would silently disarm it.
+      # The positive control matters too: `! grep -q` also passes on an empty
+      # file, so without it "the Job is absent" and "nothing rendered at all"
+      # would be indistinguishable.
+      - name: Render template (vocab-load skipped without a bucket)
+        run: |
+          helm template trust-release deploy/providers/kubernetes/ > /tmp/novocab.yaml
+          if ! grep -q "omop-db" /tmp/novocab.yaml; then
+            echo "::error::the default render produced no omop-db resources — the absence check below would pass vacuously"
+            exit 1
+          fi
+          if grep -q "omop-vocab-load" /tmp/novocab.yaml; then
+            echo "::error::the vocab-load Job rendered with no s3Bucket set — a failed hook fails the WHOLE Helm release"
+            exit 1
+          fi
+
       # FLIP-PT-091: the orthanc user map must stay a required secret reference.
       # With optional: true, an install whose Secret lacks the key boots a PACS
       # with no registered users instead of failing the pod.

.github/workflows/test_trust_omop_db.yml

@@ -0,0 +1,84 @@
+# Copyright (c) Guy's and St Thomas' NHS Foundation Trust & King's College London
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Trust - OMOP DB CI
+
+on:
+  push:
+    branches: [main, develop]
+    paths:
+      - "trust/omop-db/**"
+      - ".github/workflows/test_trust_omop_db.yml"
+  pull_request:
+    branches: [main, develop]
+
+permissions:
+  contents: read
+
+jobs:
+  omop-db:
+    # Lint + type-check + unit tests for the omop-db populate tooling. No
+    # backing services needed — the DB-touching pipeline is manual by design
+    # (the licensed core vocabulary cannot be fetched in CI; see
+    # trust/omop-db/README.md).
+    runs-on: ubuntu-latest
+    # Matches every sibling test workflow that uploads to Codecov — CODECOV_TOKEN
+    # is scoped to this environment.
+    environment: flip
+    defaults:
+      run:
+        working-directory: ./trust/omop-db
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        run: pip install uv
+
+      - name: Sync omop-db-tools venv
+        run: uv sync --locked
+
+      - name: Lint, type-check and unit tests
+        run: make local_test
+
+      # The Kubernetes vocab-load Job unpacks the bundle with the image's own
+      # unzip so that it installs nothing at run time and needs no egress to a
+      # package mirror. That contract spans two repos' worth of distance — the
+      # Helm template assumes what this Dockerfile provides — so assert the
+      # artifact itself rather than the Dockerfile text. This gates the GHCR
+      # publish (docker_build_omop_db.yml runs on this workflow's success), so a
+      # broken image cannot reach a cluster, where the symptom would be
+      # `unzip: command not found` inside a failing release hook.
+      - name: Image carries the seed-time loader contract
+        run: |
+          docker build -t omop-db-ci-check .
+          docker run --rm omop-db-ci-check sh -c '
+            command -v unzip > /dev/null || { echo "unzip missing from the image"; exit 1; }
+            test -x /flip/omop/load_core_vocab.sh || { echo "load_core_vocab.sh missing or not executable"; exit 1; }
+            test -f /flip/omop/constraints.sql || { echo "constraints.sql missing"; exit 1; }
+            /flip/omop/load_core_vocab.sh 2>&1 | grep -q "usage:" || { echo "loader did not run"; exit 1; }
+          '
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          files: trust/omop-db/coverage-unit.xml
+          flags: omop-db
+          fail_ci_if_error: false
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.pre-commit-config.yaml

@@ -91,6 +91,10 @@ repos:
         name: uv-lock (data-access-api)
         args: ['--check', '--project', 'trust/data-access-api']
         files: '^trust/data-access-api/(pyproject\.toml|uv\.lock|uv\.toml)$'
+      - id: uv-lock
+        name: uv-lock (omop-db)
+        args: ['--check', '--project', 'trust/omop-db']
+        files: '^trust/omop-db/(pyproject\.toml|uv\.lock|uv\.toml)$'
       - id: uv-lock
         name: uv-lock (xnat-tests)
         args: ['--check', '--project', 'trust/xnat/tests']