[Feature]: Merge flip-omop-db into the monorepo as the trust/omop-db build source

.github/workflows/docker_build_omop_db.yml

@@ -0,0 +1,146 @@
+# Copyright (c) Guy's and St Thomas' NHS Foundation Trust & King's College London
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Build and Push Docker Image for OMOP DB
+
+on:
+  workflow_dispatch:
+  # Publish only after the omop-db test suite ("Trust - OMOP DB CI") completes
+  # on main/develop; the job-level `if` below gates on its success. The image
+  # is vocab-free (FLIP#842) — nothing licensed enters the build context, which
+  # is what makes this CI build possible at all.
+  workflow_run:
+    workflows: ["Trust - OMOP DB CI"]
+    types: [completed]
+    branches: [main, develop]
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-push:
+    # Only publish when the upstream test run passed. workflow_dispatch (manual)
+    # has no workflow_run context, so allow it through unconditionally.
+    # The repository guard skips publishing on forks, which cannot push to
+    # ghcr.io/londonaicentre and would otherwise show a spurious CI failure.
+    if: >-
+      (github.event_name == 'workflow_dispatch' ||
+      github.event.workflow_run.conclusion == 'success') &&
+      github.repository == 'londonaicentre/FLIP'
+    runs-on: ubuntu-latest
+    permissions: # override top-level read-only default to allow GHCR push
+      contents: read
+      packages: write
+    defaults:
+      run:
+        working-directory: ./trust/omop-db
+    env:
+      REGISTRY: ghcr.io
+      IMAGE_NAME: londonaicentre/omop-db
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          # workflow_run.* is empty on manual workflow_dispatch runs, so fall
+          # back to the dispatched ref so manual builds check out correctly.
+          repository: ${{ github.event.workflow_run.head_repository.full_name || github.repository }}
+          ref: ${{ github.event.workflow_run.head_sha || github.sha }}
+
+      - name: Determine tags
+        id: tags
+        env:
+          GH_REF_NAME: ${{ github.ref_name }}
+          GH_EVENT_NAME: ${{ github.event_name }}
+          GH_REF: ${{ github.ref }}
+          GH_SHA: ${{ github.sha }}
+          GH_WR_BRANCH: ${{ github.event.workflow_run.head_branch }}
+          GH_WR_EVENT: ${{ github.event.workflow_run.event }}
+          GH_WR_SHA: ${{ github.event.workflow_run.head_sha }}
+        run: |
+          # On a workflow_run trigger, github.sha / github.ref_name point at the
+          # default branch (where this workflow definition lives), NOT the commit
+          # that was tested and checked out. Use the triggering run's head_sha /
+          # head_branch so the :<sha> and :<branch> tags match the built code.
+          if [[ "$GH_EVENT_NAME" == "workflow_run" ]]; then
+            SHA="$GH_WR_SHA"
+            REF_NAME="$GH_WR_BRANCH"
+          else
+            SHA="$GH_SHA"
+            REF_NAME="$GH_REF_NAME"
+          fi
+
+          TAGS="${REGISTRY}/${IMAGE_NAME}:${SHA}"
+
+          # An empty sha would silently publish a mutable literal `sha-` tag — refuse.
+          [[ -n "$SHA" ]] || { echo "::error::empty commit SHA — cannot compute the sha-<short7> tag"; exit 1; }
+
+          # Immutable short-SHA tag (FLIP#751), pushed uniformly on every publish.
+          # Length 7 must match the tag resolution in deploy/providers/AWS/Makefile.
+          TAGS="${TAGS},${REGISTRY}/${IMAGE_NAME}:sha-${SHA:0:7}"
+
+          # Branch name sanitization
+          SAFE_REF_NAME=$(echo "$REF_NAME" | sed 's/[^a-zA-Z0-9]/-/g')
+          TAGS="${TAGS},${REGISTRY}/${IMAGE_NAME}:${SAFE_REF_NAME}"
+
+          # Branch number (if starts with number)
+          if [[ "$REF_NAME" =~ ^[0-9]+ ]]; then
+              BRANCH_NUM=$(echo "$REF_NAME" | grep -oE '^[0-9]+')
+              TAGS="${TAGS},${REGISTRY}/${IMAGE_NAME}:${BRANCH_NUM}"
+          fi
+
+          # PR Number (if PR)
+          if [[ "$GH_EVENT_NAME" == "pull_request" ]]; then
+             PR_NUMBER=$(echo "$GH_REF" | awk -F / '{print $3}')
+             TAGS="${TAGS},${REGISTRY}/${IMAGE_NAME}:pr-${PR_NUMBER}"
+          fi
+
+          # Determine if this is a merge/push to main or develop
+          BRANCH_NAME=""
+          if [[ "$GH_EVENT_NAME" == "workflow_run" && "$GH_WR_EVENT" == "push" ]]; then
+            BRANCH_NAME="$GH_WR_BRANCH"
+          elif [[ "$GH_EVENT_NAME" == "push" ]]; then
+            BRANCH_NAME="$GH_REF_NAME"
+          fi
+
+          if [[ "$BRANCH_NAME" == "main" ]]; then
+            TAGS="${TAGS},${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:prod,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest"
+          elif [[ "$BRANCH_NAME" == "develop" ]]; then
+            TAGS="${TAGS},${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:stag"
+          fi
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+          echo "Generated tags: ${TAGS}"
+
+      - name: Log in to GitHub Container Registry
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_ACTOR: ${{ github.actor }}
+        run: echo "$GH_TOKEN" | docker login $REGISTRY -u "$GH_ACTOR" --password-stdin
+
+      - name: Build Docker image
+        env:
+          DOCKER_TAGS: ${{ steps.tags.outputs.tags }}
+        run: |
+          IFS=',' read -ra TAG_ARRAY <<< "$DOCKER_TAGS"
+          TAG_FLAGS=()
+          for tag in "${TAG_ARRAY[@]}"; do
+            TAG_FLAGS+=("-t" "$tag")
+          done
+          docker build "${TAG_FLAGS[@]}" .
+
+      - name: Push Docker image
+        env:
+          DOCKER_TAGS: ${{ steps.tags.outputs.tags }}
+        run: |
+          IFS=',' read -ra TAG_ARRAY <<< "$DOCKER_TAGS"
+          for tag in "${TAG_ARRAY[@]}"; do
+            docker push "$tag"
+          done

.github/workflows/test_trust_omop_db.yml

@@ -0,0 +1,62 @@
+# Copyright (c) Guy's and St Thomas' NHS Foundation Trust & King's College London
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Trust - OMOP DB CI
+
+on:
+  push:
+    branches: [main, develop]
+    paths:
+      - "trust/omop-db/**"
+      - ".github/workflows/test_trust_omop_db.yml"
+  pull_request:
+    branches: [main, develop]
+
+permissions:
+  contents: read
+
+jobs:
+  omop-db:
+    # Lint + type-check + unit tests for the omop-db populate tooling. No
+    # backing services needed — the DB-touching pipeline is manual by design
+    # (the baked vocabulary cannot be fetched in CI; see trust/omop-db/README.md).
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./trust/omop-db
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        run: pip install uv
+
+      - name: Sync omop-db-tools venv
+        run: uv sync --locked
+
+      - name: Lint, type-check and unit tests
+        run: make local_test
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          files: trust/omop-db/coverage-unit.xml
+          flags: omop-db
+          fail_ci_if_error: false
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.pre-commit-config.yaml

@@ -91,6 +91,10 @@ repos:
         name: uv-lock (data-access-api)
         args: ['--check', '--project', 'trust/data-access-api']
         files: '^trust/data-access-api/(pyproject\.toml|uv\.lock|uv\.toml)$'
+      - id: uv-lock
+        name: uv-lock (omop-db)
+        args: ['--check', '--project', 'trust/omop-db']
+        files: '^trust/omop-db/(pyproject\.toml|uv\.lock|uv\.toml)$'
       - id: uv-lock
         name: uv-lock (xnat-tests)
         args: ['--check', '--project', 'trust/xnat/tests']

AGENTS.md

@@ -21,7 +21,7 @@ FLIP/
 │   ├── trust-api/      # Trust API gateway (Python/FastAPI)
 │   ├── data-access-api/# OMOP database queries (Python/FastAPI)
 │   ├── imaging-api/    # DICOM image retrieval (Python/FastAPI)
-│   ├── omop-db/        # Mocked OMOP database (PostgreSQL)
+│   ├── omop-db/        # Mocked OMOP database (PostgreSQL) + omop-db image build source & populate tooling (#834)
 │   ├── orthanc/        # Mocked PACS server
 │   └── xnat/           # Mocked XNAT neuroimaging service
 ├── deploy/             # Docker Compose files (dev/prod, flower/nvflare); FL network provisioning now lives under fl-services/<backend>/, not here
@@ -373,7 +373,7 @@ GitHub Actions: `test_flip_api.yml`, `test_flip_ui.yml`, `test_trust_*.yml`, `do
 
 ### Docker image builds: gated on tests, manual trigger for branches
 
-**The application `docker_build_*.yml` workflows (`flip_api`, `trust_trust_api`, `trust_imaging_api`, `trust_data_access_api`) auto-publish to GHCR only after their service's test workflow passes on `develop` or `main`.** They trigger via `workflow_run` on the matching test workflow (`FLIP API CI`, `Trust - Trust API CI`, etc.) and a job-level `if` gates on `workflow_run.conclusion == 'success'` — a red test suite never publishes. Path filtering is inherited from the test workflow, so a build still only fires when that service changed. (`orthanc`, `xnat_*` keep their direct push trigger — they have no test suite to gate on; `flip-ui` is a CI smoke test that never publishes.)
+**The application `docker_build_*.yml` workflows (`flip_api`, `trust_trust_api`, `trust_imaging_api`, `trust_data_access_api`, `omop_db`) auto-publish to GHCR only after their service's test workflow passes on `develop` or `main`.** They trigger via `workflow_run` on the matching test workflow (`FLIP API CI`, `Trust - Trust API CI`, etc.) and a job-level `if` gates on `workflow_run.conclusion == 'success'` — a red test suite never publishes. Path filtering is inherited from the test workflow, so a build still only fires when that service changed. (`orthanc`, `xnat_*` keep their direct push trigger — they have no test suite to gate on; `flip-ui` is a CI smoke test that never publishes.)
 
 Every publish also pushes an immutable **`sha-<short7>`** tag (first 7 chars of the built commit) alongside the mutable `:stag`/`:prod` tags. Hub ECS deploys pin these sha tags via task-definition revisions — `make deploy-centralhub` resolves the env branch tip's tag, `make rollback-centralhub` repoints at the previous revision (FLIP#751; see `deploy/providers/AWS/README.md` "Central Hub deploys and rollback"). `deploy-centralhub` also prints an **FL quiesce reminder** (FLIP#770; on `PROD=true` it adds an interactive are-you-sure confirmation, stag stays non-interactive): replacing `fl-server-net-1` kills any in-flight training run, so enable deployment mode first — it pauses FL job pickup (queued jobs hold; the running job finishes and frees its net) — and wait until the hub's `GET /fl/quiesce` reports deployment mode ON and no BUSY net, making "enable mode → wait → deploy → disable" the standard redeploy workflow.
 

CLAUDE.md

@@ -21,7 +21,7 @@ FLIP/
 │   ├── trust-api/      # Trust API gateway (Python/FastAPI)
 │   ├── data-access-api/# OMOP database queries (Python/FastAPI)
 │   ├── imaging-api/    # DICOM image retrieval (Python/FastAPI)
-│   ├── omop-db/        # Mocked OMOP database (PostgreSQL)
+│   ├── omop-db/        # Mocked OMOP database (PostgreSQL) + omop-db image build source & populate tooling (#834)
 │   ├── orthanc/        # Mocked PACS server
 │   └── xnat/           # Mocked XNAT neuroimaging service
 ├── deploy/             # Docker Compose files (dev/prod, flower/nvflare); FL network provisioning now lives under fl-services/<backend>/, not here
@@ -373,7 +373,7 @@ GitHub Actions: `test_flip_api.yml`, `test_flip_ui.yml`, `test_trust_*.yml`, `do
 
 ### Docker image builds: gated on tests, manual trigger for branches
 
-**The application `docker_build_*.yml` workflows (`flip_api`, `trust_trust_api`, `trust_imaging_api`, `trust_data_access_api`) auto-publish to GHCR only after their service's test workflow passes on `develop` or `main`.** They trigger via `workflow_run` on the matching test workflow (`FLIP API CI`, `Trust - Trust API CI`, etc.) and a job-level `if` gates on `workflow_run.conclusion == 'success'` — a red test suite never publishes. Path filtering is inherited from the test workflow, so a build still only fires when that service changed. (`orthanc`, `xnat_*` keep their direct push trigger — they have no test suite to gate on; `flip-ui` is a CI smoke test that never publishes.)
+**The application `docker_build_*.yml` workflows (`flip_api`, `trust_trust_api`, `trust_imaging_api`, `trust_data_access_api`, `omop_db`) auto-publish to GHCR only after their service's test workflow passes on `develop` or `main`.** They trigger via `workflow_run` on the matching test workflow (`FLIP API CI`, `Trust - Trust API CI`, etc.) and a job-level `if` gates on `workflow_run.conclusion == 'success'` — a red test suite never publishes. Path filtering is inherited from the test workflow, so a build still only fires when that service changed. (`orthanc`, `xnat_*` keep their direct push trigger — they have no test suite to gate on; `flip-ui` is a CI smoke test that never publishes.)
 
 Every publish also pushes an immutable **`sha-<short7>`** tag (first 7 chars of the built commit) alongside the mutable `:stag`/`:prod` tags. Hub ECS deploys pin these sha tags via task-definition revisions — `make deploy-centralhub` resolves the env branch tip's tag, `make rollback-centralhub` repoints at the previous revision (FLIP#751; see `deploy/providers/AWS/README.md` "Central Hub deploys and rollback"). `deploy-centralhub` also prints an **FL quiesce reminder** (FLIP#770; on `PROD=true` it adds an interactive are-you-sure confirmation, stag stays non-interactive): replacing `fl-server-net-1` kills any in-flight training run, so enable deployment mode first — it pauses FL job pickup (queued jobs hold; the running job finishes and frees its net) — and wait until the hub's `GET /fl/quiesce` reports deployment mode ON and no BUSY net, making "enable mode → wait → deploy → disable" the standard redeploy workflow.
 

Makefile

@@ -375,6 +375,7 @@ unit_test:
 	$(MAKE) -C flip-ui unit_test
 	$(MAKE) -C trust/data-access-api unit_test
 	$(MAKE) -C trust/imaging-api unit_test
+	$(MAKE) -C trust/omop-db unit_test
 	$(MAKE) -C trust/trust-api unit_test
 	$(MAKE) -C trust/xnat unit_test
 
@@ -383,7 +384,7 @@ integration_test:
 	$(MAKE) -C trust integration_test
 
 # Python projects managed by uv; each has its own pyproject.toml + uv.lock.
-UV_PROJECTS := . flip-api docs trust/trust-api trust/imaging-api trust/data-access-api trust/xnat/tests deploy/providers/AWS
+UV_PROJECTS := . flip-api docs trust/trust-api trust/imaging-api trust/data-access-api trust/omop-db trust/xnat/tests deploy/providers/AWS
 
 # Regenerate every uv.lock so it matches its pyproject.toml. Run after changing
 # dependencies in any service, or to refresh all lockfiles in one pass.

THIRD_PARTY_NOTICES.md

@@ -8,3 +8,30 @@ Copyright (c) 2020, Washington University School of Medicine
 
 Licensed under the BSD 2-Clause License.
 See trust/xnat/LICENSE.txt.
+
+## OHDSI CommonDataModel
+
+The OMOP CDM 5.4 PostgreSQL DDL files under trust/omop-db/files/
+(OMOPCDM_postgresql_5.4_*.sql) are derived from the OHDSI CommonDataModel
+project (https://github.com/OHDSI/CommonDataModel, inst/ddl/5.4/postgresql),
+extended with the MI-CDM medical-imaging tables (image_occurrence,
+image_feature).
+
+Copyright (c) Observational Health Data Sciences and Informatics (OHDSI)
+
+Licensed under the Apache License, Version 2.0.
+
+## DICOM2OMOP
+
+The DICOM vocabulary loader (trust/omop-db/src/omop_db_tools/load_dicom_vocab.py)
+is adapted from the DICOM2OMOP project
+(https://github.com/paulnagy/DICOM2OMOP), and the DICOM vocabulary bundle it
+loads (vocab_dicom_paulnagy_20260109) is a byte-for-byte copy of four files
+from that project's "files/OMOP CDM Staging/" directory at commit
+1ef3354c0a1b335dcc7934717b1946e76dda94b8, generated by that project from the
+NEMA DICOM standard (PS3); the relationship frame is converted from the
+upstream pickle to CSV in the published copy.
+
+Copyright (c) Paul Nagy
+
+Licensed under the Apache License, Version 2.0.

deploy/providers/AWS/Makefile

@@ -475,14 +475,18 @@ seed-trust-data: check-trust-ec2-enabled ssh-config ## Load trust-specific OMOP
 	@[ -n "$(FL_KIT_SLOT_NUMBER)" ] || (echo "❌  FL_KIT_SLOT_NUMBER not found in trust/.env.$(KIT).$(KIT_ENV_SUFFIX) — run register-trusts first"; exit 1)
 	@[ -n "$(OMOP_DATA_VERSION)" ] || (echo "❌  trust/omop-db/.data_version not found"; exit 1)
 	@[ -n "$(ORTHANC_DATA_VERSION)" ] || (echo "❌  trust/orthanc/.data_version not found"; exit 1)
+	@[ -n "$(OMOP_POSTGRES_PASSWORD_KIT)" ] || (echo "❌  OMOP_POSTGRES_PASSWORD not found in trust/.env.$(KIT).$(KIT_ENV_SUFFIX) — the vocab-load play needs the cluster credentials"; exit 1)
 	@echo "🗄️  Seeding Trust EC2 with trust$(FL_KIT_SLOT_NUMBER) data (OMOP v$(OMOP_DATA_VERSION), Orthanc v$(ORTHANC_DATA_VERSION))..."
 	@uv run ansible-playbook site.yml \
 		--tags data \
 		-e flip_aicentre_bucket=$(AICENTRE_BUCKET_NAME) \
 		-e fl_kit_date=$(FL_KIT_DATE) \
 		-e trust_num=$(FL_KIT_SLOT_NUMBER) \
 		-e omop_data_version=$(OMOP_DATA_VERSION) \
-		-e orthanc_data_version=$(ORTHANC_DATA_VERSION)
+		-e orthanc_data_version=$(ORTHANC_DATA_VERSION) \
+		-e omop_postgres_user=$(OMOP_POSTGRES_USER_KIT) \
+		-e omop_postgres_password=$(OMOP_POSTGRES_PASSWORD_KIT) \
+		-e omop_postgres_db=$(OMOP_POSTGRES_DB_KIT)
 	@echo "✅ Trust data seeding complete."
 
 .PHONY: deploy-centralhub
@@ -752,6 +756,12 @@ deploy-ui: ## Build flip-ui from the working tree and deploy it to S3 + invalida
 FL_KIT_SLOT := $(shell sed -n 's/^FL_KIT_SLOT=//p' ../../../trust/.env.$(KIT).$(KIT_ENV_SUFFIX) 2>/dev/null)
 FL_KIT_SLOT_NUMBER := $(shell sed -n 's/^FL_KIT_SLOT_NUMBER=//p' ../../../trust/.env.$(KIT).$(KIT_ENV_SUFFIX) 2>/dev/null)
 
+# Trust-local OMOP DB credentials from the kit — passed to the vocab-load play
+# (they must match the values the restored pgdata cluster was built with).
+OMOP_POSTGRES_USER_KIT := $(shell sed -n 's/^OMOP_POSTGRES_USER=//p' ../../../trust/.env.$(KIT).$(KIT_ENV_SUFFIX) 2>/dev/null)
+OMOP_POSTGRES_PASSWORD_KIT := $(shell sed -n 's/^OMOP_POSTGRES_PASSWORD=//p' ../../../trust/.env.$(KIT).$(KIT_ENV_SUFFIX) 2>/dev/null)
+OMOP_POSTGRES_DB_KIT := $(shell sed -n 's/^OMOP_POSTGRES_DB=//p' ../../../trust/.env.$(KIT).$(KIT_ENV_SUFFIX) 2>/dev/null)
+
 # Data versions read from the canonical .data_version files — these govern both
 # the local update-omop-data / update-orthanc-data scripts and the Ansible plays.
 OMOP_DATA_VERSION := $(shell cat ../../../trust/omop-db/.data_version 2>/dev/null | tr -d ' \n\r\t')