diff --git a/.github/workflows/docker_build_monailabel.yml b/.github/workflows/docker_build_monailabel.yml new file mode 100644 index 000000000..60b9d38e1 --- /dev/null +++ b/.github/workflows/docker_build_monailabel.yml @@ -0,0 +1,168 @@ +# Copyright (c) Guy's and St Thomas' NHS Foundation Trust & King's College London +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Build and Push Docker Image for MONAI Label + +# Direct push trigger like orthanc / xnat_* — monailabel has no test suite to +# gate on. workflow_dispatch covers branch-pinned testing (see CLAUDE.md +# "Docker image builds"). +on: + workflow_dispatch: + push: + branches: [main, develop] + paths: + - "trust/monailabel/**" + - ".github/workflows/docker_build_monailabel.yml" + # PR runs build + the import smoke only — the push steps are gated below, + # so nothing is published from a pull request. + pull_request: + paths: + - "trust/monailabel/**" + - ".github/workflows/docker_build_monailabel.yml" + +permissions: + contents: read + +jobs: + build-and-push: + # Skip on forks: they cannot push to ghcr.io/londonaicentre. + if: github.repository == 'londonaicentre/FLIP' + runs-on: ubuntu-latest + permissions: # override top-level read-only default to allow GHCR push + contents: read + packages: write + defaults: + run: + working-directory: ./trust/monailabel + env: + REGISTRY: ghcr.io + IMAGE_NAME: londonaicentre/monailabel + BUILD_ARGS: "" + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + repository: ${{ github.event.workflow_run.head_repository.full_name }} + ref: ${{ github.event.workflow_run.head_sha }} + + # The image is ~10.4 GB (torch + CUDA runtime wheels + SAM2); a stock + # ubuntu-latest runner has ~14 GB free, which the build blows through in + # intermediate layers. Reclaim the ~25 GB of preinstalled toolchains this + # job never uses. Plain rm rather than a third-party action so the + # workflow keeps the repo's actions/* -only supply-chain posture. + - name: Reclaim runner disk space + working-directory: / + run: | + df -h / | tail -1 + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/.ghcup /usr/local/share/boost + sudo docker system prune -af --volumes >/dev/null 2>&1 || true + df -h / | tail -1 + + - name: Determine tags + id: tags + env: + GH_REF_NAME: ${{ github.ref_name }} + GH_EVENT_NAME: ${{ github.event_name }} + GH_REF: ${{ github.ref }} + GH_SHA: ${{ github.sha }} + GH_WR_BRANCH: ${{ github.event.workflow_run.head_branch }} + GH_WR_EVENT: ${{ github.event.workflow_run.event }} + run: | + TAGS="${REGISTRY}/${IMAGE_NAME}:${{ github.sha }}" + + # An empty sha would silently publish a mutable literal `sha-` tag — refuse. + [[ -n "$GH_SHA" ]] || { echo "::error::empty commit SHA — cannot compute the sha- tag"; exit 1; } + + # Immutable short-SHA tag (FLIP#751), pushed uniformly on every publish. + # Length 7 must match the tag resolution in deploy/providers/AWS/Makefile. + TAGS="${TAGS},${REGISTRY}/${IMAGE_NAME}:sha-${GH_SHA:0:7}" + + # Branch name sanitization + SAFE_REF_NAME=$(echo "$GH_REF_NAME" | sed 's/[^a-zA-Z0-9]/-/g') + TAGS="${TAGS},${REGISTRY}/${IMAGE_NAME}:${SAFE_REF_NAME}" + + # Branch number (if starts with number) + if [[ "$GH_REF_NAME" =~ ^[0-9]+ ]]; then + BRANCH_NUM=$(echo "$GH_REF_NAME" | grep -oE '^[0-9]+') + TAGS="${TAGS},${REGISTRY}/${IMAGE_NAME}:${BRANCH_NUM}" + fi + + # PR Number (if PR) + if [[ "$GH_EVENT_NAME" == "pull_request" ]]; then + PR_NUMBER=$(echo "$GH_REF" | awk -F / '{print $3}') + TAGS="${TAGS},${REGISTRY}/${IMAGE_NAME}:pr-${PR_NUMBER}" + fi + + # Determine if this is a merge/push to main or develop + BRANCH_NAME="" + if [[ "$GH_EVENT_NAME" == "workflow_run" && "$GH_WR_EVENT" == "push" ]]; then + BRANCH_NAME="$GH_WR_BRANCH" + elif [[ "$GH_EVENT_NAME" == "push" ]]; then + BRANCH_NAME="$GH_REF_NAME" + fi + + if [[ "$BRANCH_NAME" == "main" ]]; then + TAGS="${TAGS},${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:prod" + elif [[ "$BRANCH_NAME" == "develop" ]]; then + TAGS="${TAGS},${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:stag" + fi + echo "tags=${TAGS}" >> $GITHUB_OUTPUT + echo "Generated tags: ${TAGS}" + + - name: Log in to GitHub Container Registry + if: github.event_name != 'pull_request' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_ACTOR: ${{ github.actor }} + run: echo "$GH_TOKEN" | docker login $REGISTRY -u "$GH_ACTOR" --password-stdin + + # The Dockerfile itself asserts the two known-bad states at build time + # (unpatched XNAT datastore auth, non-CUDA-12 torch), so a green build + # already guarantees those. + - name: Build Docker image + env: + DOCKER_TAGS: ${{ steps.tags.outputs.tags }} + run: | + IFS=',' read -ra TAG_ARRAY <<< "$DOCKER_TAGS" + TAG_FLAGS=() + for tag in "${TAG_ARRAY[@]}"; do + TAG_FLAGS+=("-t" "$tag") + done + # shellcheck disable=SC2086 + docker build $BUILD_ARGS "${TAG_FLAGS[@]}" . + + # No GPU on the runner, so this stays at import level: the stack that + # must coexist (monailabel @ pinned commit, torch cu12, sam2, curl for + # the entrypoint's probes) actually loads in the built image. + - name: Smoke test — runtime stack imports + env: + DOCKER_TAGS: ${{ steps.tags.outputs.tags }} + run: | + TAG="${DOCKER_TAGS%%,*}" + docker run --rm --entrypoint sh "$TAG" -c ' + curl --version >/dev/null && + python -c " + import monailabel, torch, sam2 + from monailabel.datastore.xnat import XNATDatastore + print(\"monailabel\", monailabel.__version__, \"torch\", torch.__version__) + "' + + - name: Push Docker image + if: github.event_name != 'pull_request' + env: + DOCKER_TAGS: ${{ steps.tags.outputs.tags }} + run: | + IFS=',' read -ra TAG_ARRAY <<< "$DOCKER_TAGS" + for tag in "${TAG_ARRAY[@]}"; do + docker push "$tag" + done diff --git a/.github/workflows/test_helm_chart.yml b/.github/workflows/test_helm_chart.yml index fe2759399..b1fcc3925 100644 --- a/.github/workflows/test_helm_chart.yml +++ b/.github/workflows/test_helm_chart.yml @@ -101,6 +101,26 @@ jobs: --set omopDb.external.host=test.example.com \ > /dev/null + # monailabel.enabled defaults false, so every other render skips its whole body — + # render coverage only: the kind install job must NOT enable it (the image is + # ~10.4 GB and needs a GPU, neither of which kind has). Also asserts the + # publicUrl `required` guard actually fires, since it is the one setting that + # cannot be defaulted (the clinician's browser calls it — see trust/README.md). + - name: Render template (monailabel enabled) + run: | + helm template trust-release deploy/providers/kubernetes/ \ + --set monailabel.enabled=true \ + --set monailabel.publicUrl=http://node.example.com:30030 > /tmp/monailabel.yaml + if ! grep -q "flip-trust.*-monailabel" /tmp/monailabel.yaml; then + echo "::error::monailabel resources did not render with monailabel.enabled=true" + exit 1 + fi + if helm template trust-release deploy/providers/kubernetes/ \ + --set monailabel.enabled=true > /dev/null 2>&1; then + echo "::error::monailabel rendered without publicUrl — the required guard is gone" + exit 1 + fi + # omopDb.vocabLoad.s3Bucket defaults to "" (the licensed bundle has no public # mirror — FLIP#842/843), so every other render in this job skips the # vocab-load Job. Without this step its ~110-line body is never rendered in diff --git a/AGENTS.md b/AGENTS.md index 7f46c6fdd..44c2f317d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -403,7 +403,7 @@ GitHub Actions: `test_flip_api.yml`, `test_flip_ui.yml`, `test_trust_*.yml`, `do ### Docker image builds: gated on tests, manual trigger for branches -**The application `docker_build_*.yml` workflows (`flip_api`, `trust_trust_api`, `trust_imaging_api`, `trust_data_access_api`, `omop_db`) auto-publish to GHCR only after their service's test workflow passes on `develop` or `main`.** They trigger via `workflow_run` on the matching test workflow (`FLIP API CI`, `Trust - Trust API CI`, etc.) and a job-level `if` gates on `workflow_run.conclusion == 'success'` — a red test suite never publishes. Path filtering is inherited from the test workflow, so a build still only fires when that service changed. (`orthanc`, `xnat_*` keep their direct push trigger — they have no separate test workflow to gate on; `orthanc` instead runs an in-job auth smoke test between build and push, and also on PRs touching `trust/orthanc/**`, so a red smoke never publishes — FLIP-PT-091; `flip-ui` is a CI smoke test that never publishes.) +**The application `docker_build_*.yml` workflows (`flip_api`, `trust_trust_api`, `trust_imaging_api`, `trust_data_access_api`, `omop_db`) auto-publish to GHCR only after their service's test workflow passes on `develop` or `main`.** They trigger via `workflow_run` on the matching test workflow (`FLIP API CI`, `Trust - Trust API CI`, etc.) and a job-level `if` gates on `workflow_run.conclusion == 'success'` — a red test suite never publishes. Path filtering is inherited from the test workflow, so a build still only fires when that service changed. (`orthanc`, `xnat_*`, `monailabel` keep their direct push trigger — they have no separate test workflow to gate on; `orthanc` instead runs an in-job auth smoke test between build and push, and also on PRs touching `trust/orthanc/**`, so a red smoke never publishes — FLIP-PT-091; `monailabel` likewise runs an in-job import smoke and reclaims runner disk first, its image being ~10.4 GB against ubuntu-latest's ~14 GB free; `flip-ui` is a CI smoke test that never publishes.) Every publish also pushes an immutable **`sha-`** tag (first 7 chars of the built commit) alongside the mutable `:stag`/`:prod` tags. Hub ECS deploys pin these sha tags via task-definition revisions — `make deploy-centralhub` resolves the env branch tip's tag, `make rollback-centralhub` repoints at the previous revision (FLIP#751; see `deploy/providers/AWS/README.md` "Central Hub deploys and rollback"). `deploy-centralhub` also prints an **FL quiesce reminder** (FLIP#770; on `PROD=true` it adds an interactive are-you-sure confirmation, stag stays non-interactive): replacing `fl-server-net-1` kills any in-flight training run, so enable deployment mode first — it pauses FL job pickup (queued jobs hold; the running job finishes and frees its net) — and wait until the hub's `GET /fl/quiesce` reports deployment mode ON and no BUSY net, making "enable mode → wait → deploy → disable" the standard redeploy workflow. diff --git a/CLAUDE.md b/CLAUDE.md index 53460ee33..0ed1d4d41 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -403,7 +403,7 @@ GitHub Actions: `test_flip_api.yml`, `test_flip_ui.yml`, `test_trust_*.yml`, `do ### Docker image builds: gated on tests, manual trigger for branches -**The application `docker_build_*.yml` workflows (`flip_api`, `trust_trust_api`, `trust_imaging_api`, `trust_data_access_api`, `omop_db`) auto-publish to GHCR only after their service's test workflow passes on `develop` or `main`.** They trigger via `workflow_run` on the matching test workflow (`FLIP API CI`, `Trust - Trust API CI`, etc.) and a job-level `if` gates on `workflow_run.conclusion == 'success'` — a red test suite never publishes. Path filtering is inherited from the test workflow, so a build still only fires when that service changed. (`orthanc`, `xnat_*` keep their direct push trigger — they have no separate test workflow to gate on; `orthanc` instead runs an in-job auth smoke test between build and push, and also on PRs touching `trust/orthanc/**`, so a red smoke never publishes — FLIP-PT-091; `flip-ui` is a CI smoke test that never publishes.) +**The application `docker_build_*.yml` workflows (`flip_api`, `trust_trust_api`, `trust_imaging_api`, `trust_data_access_api`, `omop_db`) auto-publish to GHCR only after their service's test workflow passes on `develop` or `main`.** They trigger via `workflow_run` on the matching test workflow (`FLIP API CI`, `Trust - Trust API CI`, etc.) and a job-level `if` gates on `workflow_run.conclusion == 'success'` — a red test suite never publishes. Path filtering is inherited from the test workflow, so a build still only fires when that service changed. (`orthanc`, `xnat_*`, `monailabel` keep their direct push trigger — they have no separate test workflow to gate on; `orthanc` instead runs an in-job auth smoke test between build and push, and also on PRs touching `trust/orthanc/**`, so a red smoke never publishes — FLIP-PT-091; `monailabel` likewise runs an in-job import smoke and reclaims runner disk first, its image being ~10.4 GB against ubuntu-latest's ~14 GB free; `flip-ui` is a CI smoke test that never publishes.) Every publish also pushes an immutable **`sha-`** tag (first 7 chars of the built commit) alongside the mutable `:stag`/`:prod` tags. Hub ECS deploys pin these sha tags via task-definition revisions — `make deploy-centralhub` resolves the env branch tip's tag, `make rollback-centralhub` repoints at the previous revision (FLIP#751; see `deploy/providers/AWS/README.md` "Central Hub deploys and rollback"). `deploy-centralhub` also prints an **FL quiesce reminder** (FLIP#770; on `PROD=true` it adds an interactive are-you-sure confirmation, stag stays non-interactive): replacing `fl-server-net-1` kills any in-flight training run, so enable deployment mode first — it pauses FL job pickup (queued jobs hold; the running job finishes and frees its net) — and wait until the hub's `GET /fl/quiesce` reports deployment mode ON and no BUSY net, making "enable mode → wait → deploy → disable" the standard redeploy workflow. diff --git a/deploy/providers/kubernetes/README.md b/deploy/providers/kubernetes/README.md index 307def5e8..a7c44bb33 100644 --- a/deploy/providers/kubernetes/README.md +++ b/deploy/providers/kubernetes/README.md @@ -237,6 +237,34 @@ Available services: | `observability.loki` | Log aggregation | Yes | | `observability.alloy` | Log collection agent | No (DaemonSet) | | `observability.grafana` | Metrics dashboard | Yes | +| `monailabel` | Optional AI-assisted annotation in the XNAT OHIF viewer (off by default) | Yes | + +### MONAI Label (optional) + +Off by default — needs an NVIDIA GPU node and a ~10.4 GB image. Enable with: + +```yaml +monailabel: + enabled: true + publicUrl: "http://:30030" # REQUIRED: the clinician's BROWSER calls this — + # XNAT stores it and never proxies it +``` + +Chart-specific notes (the full operational guide, including the stock-viewer quirks, is +[trust/README.md#monai-label-optional](../../../trust/README.md#monai-label-optional)): + +- Reads DICOM straight off xnat-web's archive PVC (read-only `archive` subPath). With the + default `ReadWriteOnce` storage the pod is pinned to xnat-web's node + (`coScheduleWithXnat: true`); only set it `false` on an RWX storage class. +- Exposed as a `NodePort` (default `30030`) so the browser can reach it; a scoped + NetworkPolicy opens exactly that port through the namespace's default-deny ingress + (`service.allowExternalIngress`). The API is unauthenticated — restrict node-port reach + at the network layer. +- Pretrained weights (incl. the ~900 MB SAM checkpoint) persist in the + `monailabel-models` PVC; first start on a cold volume takes minutes (the startup probe + allows 30). +- The chart's XNAT already ships the `ohif-viewer` plugin in its roster (`xnat.web.plugins`), + so no extra plugin step — unlike the compose trust, which deliberately excludes it (FLIP#662). ### External Service Override diff --git a/deploy/providers/kubernetes/templates/monailabel.yaml b/deploy/providers/kubernetes/templates/monailabel.yaml new file mode 100644 index 000000000..8b49b4416 --- /dev/null +++ b/deploy/providers/kubernetes/templates/monailabel.yaml @@ -0,0 +1,223 @@ +# Copyright (c) 2026 Guy's and St Thomas' NHS Foundation Trust & King's College London +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.monailabel.enabled }} +# Optional MONAI Label server: AI-assisted annotation (SAM click-to-segment, DeepEdit +# multi-organ) in the XNAT OHIF viewer. Mirrors the compose overlay +# (trust/deploy/compose_trust.*.monailabel.yml) — see trust/README.md#monai-label-optional +# for the operational caveats (browser-reachable URL, unauthenticated API, stock-viewer +# quirks). Unlike compose, the chart's XNAT already ships the ohif-viewer plugin in its +# roster, so there is no separate plugin step here. +# +# The entrypoint handles XNAT registration itself: it polls XNAT and the service account, +# starts the server, then PUTs monailabel.publicUrl to /xapi/ohifaiaa/servers and fails +# loudly if the registration is rejected. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "flip-trust.fullname" . }}-monailabel + namespace: {{ include "flip-trust.namespace" . }} + labels: + {{- include "flip-trust.labels" . | nindent 4 }} + app.kubernetes.io/component: monailabel +spec: + replicas: 1 + # Recreate: the weights PVC is ReadWriteOnce and the server holds GPU memory — a rolling + # update would deadlock on both. + strategy: + type: Recreate + selector: + matchLabels: + {{- include "flip-trust.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: monailabel + template: + metadata: + labels: + {{- include "flip-trust.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: monailabel + spec: + enableServiceLinks: false + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ include "flip-trust.imagePullSecrets" . | nindent 8 }} + {{- end }} + {{- if .Values.monailabel.gpu.enabled }} + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + {{- end }} + {{- if .Values.monailabel.coScheduleWithXnat }} + # The XNAT archive lives on xnat-web's data PVC. With the default ReadWriteOnce + # access mode that volume can only be mounted on the node where xnat-web runs, so + # this pod must land there too. Set coScheduleWithXnat: false only on an RWX + # storage class. + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + {{- include "flip-trust.selectorLabels" . | nindent 18 }} + app.kubernetes.io/component: xnat-web + topologyKey: kubernetes.io/hostname + {{- end }} + containers: + - name: monailabel + image: "{{ .Values.monailabel.image.repository }}:{{ .Values.monailabel.image.tag }}" + imagePullPolicy: {{ .Values.monailabel.image.pullPolicy }} + env: + - name: MONAI_LABEL_PORT + value: {{ .Values.monailabel.port | quote }} + - name: MONAI_LABEL_DATASTORE + value: "xnat" + - name: MONAI_LABEL_DATASTORE_URL + value: "http://xnat-web:8080" + - name: MONAI_LABEL_DATASTORE_USERNAME + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.create }}{{ include "flip-trust.fullname" . }}-secrets{{ else }}{{ .Values.secrets.existingName }}{{ end }} + key: xnat-service-user + - name: MONAI_LABEL_DATASTORE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ if .Values.secrets.create }}{{ include "flip-trust.fullname" . }}-secrets{{ else }}{{ .Values.secrets.existingName }}{{ end }} + key: xnat-service-password + # Mounted at the archive itself, not its parent: the datastore resolves files + # as + , so a parent + # path silently misses and falls back to per-scan HTTP downloads. + - name: MONAI_LABEL_DATASTORE_ASSET_PATH + value: "/workspace/xnat-data/archive" + - name: MONAI_LABEL_DATASTORE_PROJECT + value: {{ .Values.monailabel.projects | quote }} + - name: MONAI_LABEL_MODELS + value: {{ .Values.monailabel.models | quote }} + # Handed to the OHIF viewer, which calls it from the clinician's BROWSER — + # XNAT never proxies it. Must resolve outside the cluster, which the chart + # cannot derive, so it is required rather than defaulted. + - name: MONAI_LABEL_PUBLIC_URL + value: {{ required "monailabel.publicUrl is required when monailabel.enabled — the URL the clinician's browser uses to reach the MONAI Label service (e.g. http://:)" .Values.monailabel.publicUrl | quote }} + ports: + - name: http + containerPort: {{ .Values.monailabel.port }} + # Model load downloads pretrained weights on a cold volume (minutes, incl. the + # ~900MB SAM checkpoint); the startup probe gives it half an hour before the + # readiness probe takes over. + startupProbe: + httpGet: + path: /info/ + port: http + periodSeconds: 15 + failureThreshold: 120 + readinessProbe: + httpGet: + path: /info/ + port: http + periodSeconds: 30 + resources: + {{- if .Values.monailabel.gpu.enabled }} + limits: + nvidia.com/gpu: {{ .Values.monailabel.gpu.count | quote }} + {{- end }} + {{- with .Values.monailabel.resources }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: xnat-data + mountPath: /workspace/xnat-data/archive + subPath: archive + # Labels are written back through the XNAT REST API, never the filesystem. + readOnly: true + - name: models + mountPath: /workspace/app/radiology/model + - name: shm + mountPath: /dev/shm + volumes: + - name: xnat-data + persistentVolumeClaim: + claimName: {{ include "flip-trust.fullname" . }}-xnat-web + - name: models + persistentVolumeClaim: + claimName: {{ include "flip-trust.fullname" . }}-monailabel-models + - name: shm + emptyDir: + medium: Memory + sizeLimit: {{ .Values.monailabel.shmSize }} +--- +apiVersion: v1 +kind: Service +metadata: + name: monailabel + namespace: {{ include "flip-trust.namespace" . }} + labels: + {{- include "flip-trust.labels" . | nindent 4 }} + app.kubernetes.io/component: monailabel +spec: + # The clinician's browser must reach this directly (see MONAI_LABEL_PUBLIC_URL above), + # so NodePort/LoadBalancer is the working default posture; ClusterIP only makes sense + # behind an ingress that fronts it. + type: {{ .Values.monailabel.service.type }} + selector: + {{- include "flip-trust.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: monailabel + ports: + - name: http + port: {{ .Values.monailabel.port }} + targetPort: http + {{- if and (eq .Values.monailabel.service.type "NodePort") .Values.monailabel.service.nodePort }} + nodePort: {{ .Values.monailabel.service.nodePort }} + {{- end }} +--- +# Pretrained model weights (incl. the SAM checkpoint) — fetched on first start, persisted +# so a pod restart does not re-download them. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "flip-trust.fullname" . }}-monailabel-models + namespace: {{ include "flip-trust.namespace" . }} + labels: + {{- include "flip-trust.labels" . | nindent 4 }} + app.kubernetes.io/component: monailabel +spec: + accessModes: + - ReadWriteOnce + {{- if .Values.monailabel.persistence.storageClassName }} + storageClassName: {{ .Values.monailabel.persistence.storageClassName }} + {{- end }} + resources: + requests: + storage: {{ .Values.monailabel.persistence.size }} +{{- if and .Values.networkPolicies.enabled .Values.monailabel.service.allowExternalIngress }} +--- +# The namespace default-denies ingress; the MONAI Label API must be reachable from the +# clinician's browser (outside the cluster), so open exactly its port on exactly its pod. +# The API is unauthenticated and holds the XNAT service-account credentials — matching the +# compose deployment's posture, where the port is host-published; restrict who can reach +# the node port at the network layer. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "flip-trust.fullname" . }}-monailabel-ingress + namespace: {{ include "flip-trust.namespace" . }} + labels: + {{- include "flip-trust.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "flip-trust.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: monailabel + ingress: + - ports: + - port: {{ .Values.monailabel.port }} + protocol: TCP + policyTypes: + - Ingress +{{- end }} +{{- end }} diff --git a/deploy/providers/kubernetes/values.schema.json b/deploy/providers/kubernetes/values.schema.json index 9ce746c52..080d29f3d 100644 --- a/deploy/providers/kubernetes/values.schema.json +++ b/deploy/providers/kubernetes/values.schema.json @@ -4,7 +4,10 @@ "description": "Schema for values.yaml of the flip-trust Helm chart", "type": "object", "additionalProperties": true, - "required": ["trustName", "trustNumber"], + "required": [ + "trustName", + "trustNumber" + ], "properties": { "trustName": { "type": "string", @@ -17,7 +20,11 @@ }, "environment": { "type": "string", - "enum": ["production", "staging", "development"], + "enum": [ + "production", + "staging", + "development" + ], "description": "Deployment environment" }, "logLevel": { @@ -30,23 +37,34 @@ }, "flBackend": { "type": "string", - "enum": ["nvflare", "flower"], + "enum": [ + "nvflare", + "flower" + ], "description": "FL backend selection" }, "namespace": { "type": "object", "additionalProperties": true, "properties": { - "create": { "type": "boolean" }, - "name": { "type": "string" } + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } } }, "secrets": { "type": "object", "additionalProperties": true, "properties": { - "create": { "type": "boolean" }, - "existingName": { "type": "string" }, + "create": { + "type": "boolean" + }, + "existingName": { + "type": "string" + }, "data": { "type": "object", "additionalProperties": true @@ -57,8 +75,13 @@ "type": "object", "additionalProperties": true, "properties": { - "enabled": { "type": "boolean" }, - "replicas": { "type": "integer", "minimum": 1 }, + "enabled": { + "type": "boolean" + }, + "replicas": { + "type": "integer", + "minimum": 1 + }, "resources": { "type": "object", "additionalProperties": true, @@ -66,15 +89,23 @@ "requests": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } }, "limits": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } } } @@ -86,17 +117,29 @@ "liveness": { "type": "object", "properties": { - "initialDelaySeconds": { "type": "integer" }, - "periodSeconds": { "type": "integer" }, - "timeoutSeconds": { "type": "integer" } + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + } } }, "readiness": { "type": "object", "properties": { - "initialDelaySeconds": { "type": "integer" }, - "periodSeconds": { "type": "integer" }, - "timeoutSeconds": { "type": "integer" } + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + } } } } @@ -104,16 +147,26 @@ "image": { "type": "object", "properties": { - "repository": { "type": "string" }, - "tag": { "type": "string" }, - "pullPolicy": { "type": "string" } + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "pullPolicy": { + "type": "string" + } } }, "service": { "type": "object", "properties": { - "port": { "type": "integer" }, - "type": { "type": "string" } + "port": { + "type": "integer" + }, + "type": { + "type": "string" + } } } } @@ -122,8 +175,13 @@ "type": "object", "additionalProperties": true, "properties": { - "enabled": { "type": "boolean" }, - "replicas": { "type": "integer", "minimum": 1 }, + "enabled": { + "type": "boolean" + }, + "replicas": { + "type": "integer", + "minimum": 1 + }, "resources": { "type": "object", "additionalProperties": true, @@ -131,15 +189,23 @@ "requests": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } }, "limits": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } } } @@ -151,17 +217,29 @@ "liveness": { "type": "object", "properties": { - "initialDelaySeconds": { "type": "integer" }, - "periodSeconds": { "type": "integer" }, - "timeoutSeconds": { "type": "integer" } + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + } } }, "readiness": { "type": "object", "properties": { - "initialDelaySeconds": { "type": "integer" }, - "periodSeconds": { "type": "integer" }, - "timeoutSeconds": { "type": "integer" } + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + } } } } @@ -169,16 +247,26 @@ "image": { "type": "object", "properties": { - "repository": { "type": "string" }, - "tag": { "type": "string" }, - "pullPolicy": { "type": "string" } + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "pullPolicy": { + "type": "string" + } } }, "service": { "type": "object", "properties": { - "port": { "type": "integer" }, - "type": { "type": "string" } + "port": { + "type": "integer" + }, + "type": { + "type": "string" + } } } } @@ -187,8 +275,13 @@ "type": "object", "additionalProperties": true, "properties": { - "enabled": { "type": "boolean" }, - "replicas": { "type": "integer", "minimum": 1 }, + "enabled": { + "type": "boolean" + }, + "replicas": { + "type": "integer", + "minimum": 1 + }, "resources": { "type": "object", "additionalProperties": true, @@ -196,15 +289,23 @@ "requests": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } }, "limits": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } } } @@ -216,17 +317,29 @@ "liveness": { "type": "object", "properties": { - "initialDelaySeconds": { "type": "integer" }, - "periodSeconds": { "type": "integer" }, - "timeoutSeconds": { "type": "integer" } + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + } } }, "readiness": { "type": "object", "properties": { - "initialDelaySeconds": { "type": "integer" }, - "periodSeconds": { "type": "integer" }, - "timeoutSeconds": { "type": "integer" } + "initialDelaySeconds": { + "type": "integer" + }, + "periodSeconds": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + } } } } @@ -234,16 +347,26 @@ "image": { "type": "object", "properties": { - "repository": { "type": "string" }, - "tag": { "type": "string" }, - "pullPolicy": { "type": "string" } + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "pullPolicy": { + "type": "string" + } } }, "service": { "type": "object", "properties": { - "port": { "type": "integer" }, - "type": { "type": "string" } + "port": { + "type": "integer" + }, + "type": { + "type": "string" + } } } } @@ -252,22 +375,32 @@ "type": "object", "additionalProperties": true, "properties": { - "enabled": { "type": "boolean" }, + "enabled": { + "type": "boolean" + }, "resources": { "type": "object", "properties": { "requests": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } }, "limits": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } } } @@ -275,8 +408,12 @@ "gpu": { "type": "object", "properties": { - "enabled": { "type": "boolean" }, - "count": { "type": "integer" } + "enabled": { + "type": "boolean" + }, + "count": { + "type": "integer" + } } } } @@ -285,16 +422,33 @@ "type": "object", "additionalProperties": true, "properties": { - "enabled": { "type": "boolean" }, + "enabled": { + "type": "boolean" + }, "vocabLoad": { "type": "object", "properties": { - "enabled": { "type": "boolean" }, - "s3Bucket": { "type": "string" }, - "bundleName": { "type": "string" }, - "workDirSize": { "type": "string", "minLength": 1 }, - "fetchResources": { "type": "object", "minProperties": 1 }, - "loadResources": { "type": "object", "minProperties": 1 } + "enabled": { + "type": "boolean" + }, + "s3Bucket": { + "type": "string" + }, + "bundleName": { + "type": "string" + }, + "workDirSize": { + "type": "string", + "minLength": 1 + }, + "fetchResources": { + "type": "object", + "minProperties": 1 + }, + "loadResources": { + "type": "object", + "minProperties": 1 + } } }, "resources": { @@ -303,15 +457,23 @@ "requests": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } }, "limits": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } } } @@ -322,23 +484,36 @@ "type": "object", "additionalProperties": true, "properties": { - "enabled": { "type": "boolean" }, - "replicas": { "type": "integer", "minimum": 1 }, + "enabled": { + "type": "boolean" + }, + "replicas": { + "type": "integer", + "minimum": 1 + }, "resources": { "type": "object", "properties": { "requests": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } }, "limits": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } } } @@ -349,26 +524,38 @@ "type": "object", "additionalProperties": true, "properties": { - "enabled": { "type": "boolean" }, + "enabled": { + "type": "boolean" + }, "web": { "type": "object", "properties": { - "enabled": { "type": "boolean" }, + "enabled": { + "type": "boolean" + }, "resources": { "type": "object", "properties": { "requests": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } }, "limits": { "type": "object", "properties": { - "memory": { "type": "string" }, - "cpu": { "type": "string" } + "memory": { + "type": "string" + }, + "cpu": { + "type": "string" + } } } } @@ -384,19 +571,25 @@ "loki": { "type": "object", "properties": { - "enabled": { "type": "boolean" } + "enabled": { + "type": "boolean" + } } }, "alloy": { "type": "object", "properties": { - "enabled": { "type": "boolean" } + "enabled": { + "type": "boolean" + } } }, "grafana": { "type": "object", "properties": { - "enabled": { "type": "boolean" } + "enabled": { + "type": "boolean" + } } } } @@ -405,28 +598,158 @@ "type": "object", "additionalProperties": true, "properties": { - "enabled": { "type": "boolean" }, - "minReplicas": { "type": "integer" }, - "maxReplicas": { "type": "integer" } + "enabled": { + "type": "boolean" + }, + "minReplicas": { + "type": "integer" + }, + "maxReplicas": { + "type": "integer" + } } }, "networkPolicies": { "type": "object", "additionalProperties": true, "properties": { - "enabled": { "type": "boolean" }, + "enabled": { + "type": "boolean" + }, "allowedEgressCIDRsWithPorts": { "type": "array", "description": "Egress rules combining CIDRs with a specific port (e.g. fl-server gRPC on 8002)", "items": { "type": "object", - "required": ["cidrs", "port"], + "required": [ + "cidrs", + "port" + ], "properties": { - "cidrs": { "type": "array", "items": { "type": "string" } }, - "port": { "type": "integer" }, - "protocol": { "type": "string", "enum": ["TCP", "UDP", "SCTP"] } + "cidrs": { + "type": "array", + "items": { + "type": "string" + } + }, + "port": { + "type": "integer" + }, + "protocol": { + "type": "string", + "enum": [ + "TCP", + "UDP", + "SCTP" + ] + } + } + } + } + } + }, + "monailabel": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "image": { + "type": "object", + "properties": { + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "pullPolicy": { + "type": "string", + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + } + } + }, + "port": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "models": { + "type": "string" + }, + "projects": { + "type": "string" + }, + "publicUrl": { + "type": "string" + }, + "shmSize": { + "type": "string" + }, + "gpu": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "count": { + "type": "integer", + "minimum": 0 + } + } + }, + "coScheduleWithXnat": { + "type": "boolean" + }, + "service": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ClusterIP", + "NodePort", + "LoadBalancer" + ] + }, + "nodePort": { + "anyOf": [ + { + "type": "integer", + "minimum": 30000, + "maximum": 32767 + }, + { + "type": "null" + }, + { + "type": "string", + "maxLength": 0 + } + ] + }, + "allowExternalIngress": { + "type": "boolean" } } + }, + "persistence": { + "type": "object", + "properties": { + "size": { + "type": "string" + }, + "storageClassName": { + "type": "string" + } + } + }, + "resources": { + "type": "object" } } } diff --git a/deploy/providers/kubernetes/values.yaml b/deploy/providers/kubernetes/values.yaml index 03bc23ccb..1bedc8680 100644 --- a/deploy/providers/kubernetes/values.yaml +++ b/deploy/providers/kubernetes/values.yaml @@ -648,6 +648,51 @@ xnat: memory: "256Mi" cpu: "200m" +# --------------------------------------------------------------------------- +# MONAI Label — optional AI-assisted annotation in the XNAT OHIF viewer +# --------------------------------------------------------------------------- +# Off by default: needs an NVIDIA GPU node and pulls a ~10.4GB image. Mirrors the compose +# overlay (trust/deploy/compose_trust.*.monailabel.yml); operational caveats — the +# browser-reachable URL, the unauthenticated API, the stock-viewer quirks — are documented +# in trust/README.md#monai-label-optional. +monailabel: + enabled: false + image: + repository: ghcr.io/londonaicentre/monailabel + tag: stag + pullPolicy: IfNotPresent + port: 8030 + # Radiology models to load. 'all' loads nine, each downloading its own weights; the SAM + # tasks (sam_2d/sam_3d) register whenever the sam2 package is importable, independent of + # this list. + models: "deepedit" + # Comma-separated XNAT project allowlist; empty means every project on this trust. + projects: "" + # REQUIRED when enabled: the URL the clinician's BROWSER uses to reach the service + # (e.g. http://:30030). XNAT stores it and hands it to the OHIF viewer — it is + # never proxied, so a cluster-internal name will not work. + publicUrl: "" + # /dev/shm for dataloader workers (emptyDir, medium: Memory). + shmSize: 8Gi + gpu: + enabled: true + count: 1 + # Pin the pod to xnat-web's node so the archive PVC (ReadWriteOnce by default) can be + # mounted read-only here too. Set false only on an RWX storage class. + coScheduleWithXnat: true + service: + type: NodePort + # Fixed nodePort (30000-32767) so publicUrl stays stable; empty lets the cluster pick. + nodePort: 30030 + # Adds a NetworkPolicy opening the service port on this pod through the namespace's + # default-deny ingress — required for browser access when networkPolicies.enabled. + allowExternalIngress: true + persistence: + # Pretrained weights (incl. the ~900MB SAM checkpoint). + size: 20Gi + storageClassName: "" + resources: {} + # --------------------------------------------------------------------------- # Observability Stack — Loki, Alloy, Grafana # --------------------------------------------------------------------------- diff --git a/trust/AGENTS.md b/trust/AGENTS.md index b54a2c472..32180a585 100644 --- a/trust/AGENTS.md +++ b/trust/AGENTS.md @@ -99,7 +99,7 @@ GHCR login from `~/.docker/config.json`. | `deploy/compose_trust.production.yml` | Prod Docker Compose (GHCR images; declares the `trust-local-{loki,grafana}-data` named volumes as defaults) | | `deploy/compose_trust.{env}.{flower\|nvflare}.yml` | FL backend variants | | `deploy/compose_trust.{env}.gpu.yml` | GPU passthrough overlay — added by `up-trust` / `up-fl-clients-kit` via `GPU_OVERRIDE` only when the kit's `NUM_AVAILABLE_GPUS > 0`; reserves host NVIDIA GPU(s) for the fl-client. `up-trust-ec2` never applies it (the EC2 t3.xlarge is GPU-less, so the fl-client is CPU-only there regardless of the kit) | -| `deploy/compose_trust.development.monailabel.yml` | Optional MONAI Label server (AI-assisted annotation in the XNAT OHIF viewer) — added by `up-trust` / `build` via `MONAILABEL_OVERRIDE` only when the kit (or CLI) sets `MONAI_LABEL=true`. Requires `NUM_AVAILABLE_GPUS > 0`; `up-trust` refuses otherwise, and refuses under `PROD=true\|stag` because no `monailabel` image is published yet (FLIP#55). Bind-mounts the trust's XNAT archive read-only at the path `MONAI_LABEL_XNAT_ARCHIVE_DIR` resolves to — anchored at `xnat/` because the kit's `XNAT_DATA_DIR` is written relative to `trust/xnat/`. See [README.md](README.md#monai-label-optional) | +| `deploy/compose_trust.{env}.monailabel.yml` | Optional MONAI Label server (AI-assisted annotation in the XNAT OHIF viewer) — added by `up-trust` / `build` via `MONAILABEL_OVERRIDE` only when the kit (or CLI) sets `MONAI_LABEL=true`. Requires `NUM_AVAILABLE_GPUS > 0`; `up-trust` refuses otherwise. Dev builds from `trust/monailabel/`; production pulls `ghcr.io/londonaicentre/monailabel:${DOCKER_TAG}` (published by `docker_build_monailabel.yml`) and `up-trust` prints a FLIP#662 warning — enabling it re-introduces the OHIF viewer plugin the trust XNAT deliberately excludes. Bind-mounts the trust's XNAT archive read-only at the path `MONAI_LABEL_XNAT_ARCHIVE_DIR` resolves to — anchored at `xnat/` because the kit's `XNAT_DATA_DIR` is written relative to `trust/xnat/`. See [README.md](README.md#monai-label-optional) | | `deploy/compose_trust-1_override.yml` | Dev trust-1 host-port bindings | | `.env..` | Per-trust kit file, e.g. `.env.GSTT.development`, `.env..production` (TRUST_API_KEY, TRUST_INTERNAL_SERVICE_KEY, FL_KIT_SLOT, FL_KIT_SLOT_NUMBER, EXPECTED_TRUST_ID, host-local ports/dirs, **FL_KIT_DIR** — root of the FL participant kit, default `/opt/flip/fl-kit` matching the Ansible-staged EC2 path); gitignored. Templates: per-trust dev examples `.env.GSTT.development.example` / `.env.KCH.development.example`; the generic scaffold base `.env.example`, consumed by `make new-trust`. Same kit-file schema everywhere — `make -C trust up-trust KIT= PROD=` is the only dispatch | diff --git a/trust/CLAUDE.md b/trust/CLAUDE.md index bc00f6349..aa0a0797b 100644 --- a/trust/CLAUDE.md +++ b/trust/CLAUDE.md @@ -99,7 +99,7 @@ GHCR login from `~/.docker/config.json`. | `deploy/compose_trust.production.yml` | Prod Docker Compose (GHCR images; declares the `trust-local-{loki,grafana}-data` named volumes as defaults) | | `deploy/compose_trust.{env}.{flower\|nvflare}.yml` | FL backend variants | | `deploy/compose_trust.{env}.gpu.yml` | GPU passthrough overlay — added by `up-trust` / `up-fl-clients-kit` via `GPU_OVERRIDE` only when the kit's `NUM_AVAILABLE_GPUS > 0`; reserves host NVIDIA GPU(s) for the fl-client. `up-trust-ec2` never applies it (the EC2 t3.xlarge is GPU-less, so the fl-client is CPU-only there regardless of the kit) | -| `deploy/compose_trust.development.monailabel.yml` | Optional MONAI Label server (AI-assisted annotation in the XNAT OHIF viewer) — added by `up-trust` / `build` via `MONAILABEL_OVERRIDE` only when the kit (or CLI) sets `MONAI_LABEL=true`. Requires `NUM_AVAILABLE_GPUS > 0`; `up-trust` refuses otherwise, and refuses under `PROD=true\|stag` because no `monailabel` image is published yet (FLIP#55). Bind-mounts the trust's XNAT archive read-only at the path `MONAI_LABEL_XNAT_ARCHIVE_DIR` resolves to — anchored at `xnat/` because the kit's `XNAT_DATA_DIR` is written relative to `trust/xnat/`. See [README.md](README.md#monai-label-optional) | +| `deploy/compose_trust.{env}.monailabel.yml` | Optional MONAI Label server (AI-assisted annotation in the XNAT OHIF viewer) — added by `up-trust` / `build` via `MONAILABEL_OVERRIDE` only when the kit (or CLI) sets `MONAI_LABEL=true`. Requires `NUM_AVAILABLE_GPUS > 0`; `up-trust` refuses otherwise. Dev builds from `trust/monailabel/`; production pulls `ghcr.io/londonaicentre/monailabel:${DOCKER_TAG}` (published by `docker_build_monailabel.yml`) and `up-trust` prints a FLIP#662 warning — enabling it re-introduces the OHIF viewer plugin the trust XNAT deliberately excludes. Bind-mounts the trust's XNAT archive read-only at the path `MONAI_LABEL_XNAT_ARCHIVE_DIR` resolves to — anchored at `xnat/` because the kit's `XNAT_DATA_DIR` is written relative to `trust/xnat/`. See [README.md](README.md#monai-label-optional) | | `deploy/compose_trust-1_override.yml` | Dev trust-1 host-port bindings | | `.env..` | Per-trust kit file, e.g. `.env.GSTT.development`, `.env..production` (TRUST_API_KEY, TRUST_INTERNAL_SERVICE_KEY, FL_KIT_SLOT, FL_KIT_SLOT_NUMBER, EXPECTED_TRUST_ID, host-local ports/dirs, **FL_KIT_DIR** — root of the FL participant kit, default `/opt/flip/fl-kit` matching the Ansible-staged EC2 path); gitignored. Templates: per-trust dev examples `.env.GSTT.development.example` / `.env.KCH.development.example`; the generic scaffold base `.env.example`, consumed by `make new-trust`. Same kit-file schema everywhere — `make -C trust up-trust KIT= PROD=` is the only dispatch | diff --git a/trust/Makefile b/trust/Makefile index f0372a10f..0f757d7cf 100644 --- a/trust/Makefile +++ b/trust/Makefile @@ -176,22 +176,24 @@ __MONAI_LABEL_XNAT_DIR := $(or $(strip $(XNAT_DATA_DIR)),$(if $(filter true stag MONAI_LABEL_XNAT_ARCHIVE_DIR := $(abspath $(if $(filter /%,$(__MONAI_LABEL_XNAT_DIR)),$(__MONAI_LABEL_XNAT_DIR),xnat/$(__MONAI_LABEL_XNAT_DIR)))/xnat-data/archive export MONAI_LABEL_XNAT_ARCHIVE_DIR -# MONAI Label is dev/hybrid only for now: the prod compose runs published images and no -# monailabel image is published yet (the NVIDIA PyTorch base is too large to build on the -# ubuntu-latest runners the docker_build_* workflows use). Fail here rather than let -# compose fail on a missing overlay file. +# A GPU-less host cannot run MONAI Label at all, so that stays a hard refusal. Under PROD +# it runs from the published ghcr.io/londonaicentre/monailabel image +# (docker_build_monailabel.yml) — allowed, but with an explicit warning: it re-introduces +# the XNAT OHIF viewer plugin that the trust XNAT deliberately excludes (FLIP#662 +# bulk-import livelock). Intended for hybrid/on-prem annotation trusts; think twice on a +# trust that pulls large cohorts. define require_monailabel_supported - @if [ "$(strip $(MONAI_LABEL))" = "true" ] && [ -n "$(filter true stag,$(PROD))" ]; then \ - echo "❌ MONAI_LABEL=true is not supported with PROD=$(PROD) yet — no published"; \ - echo " monailabel image (see FLIP#55). Use the development stack, or build and"; \ - echo " push the image to GHCR first and add a production overlay."; \ - exit 1; \ - fi @if [ "$(strip $(MONAI_LABEL))" = "true" ] && [ -z "$(GPU_OVERRIDE)" ]; then \ echo "❌ MONAI_LABEL=true needs a GPU, but NUM_AVAILABLE_GPUS=$(or $(strip $(NUM_AVAILABLE_GPUS)),unset)"; \ echo " for KIT=$(KIT). Set NUM_AVAILABLE_GPUS>0 in trust/$(KIT_FILE) on a GPU host."; \ exit 1; \ fi + @if [ "$(strip $(MONAI_LABEL))" = "true" ] && [ -n "$(filter true stag,$(PROD))" ]; then \ + echo "⚠️ MONAI_LABEL=true with PROD=$(PROD): AI-assisted annotation needs the XNAT OHIF"; \ + echo " viewer plugin, which trust XNAT deliberately excludes because of the bulk-import"; \ + echo " livelock it drives (FLIP#662). Enable only on trusts that accept that trade-off"; \ + echo " (hybrid/on-prem annotation trusts, not high-volume cohort pullers)."; \ + fi endef MONAI_LABEL_STATUS_MSG := $(if $(MONAILABEL_OVERRIDE),🧠 MONAI Label on — serving $(or $(strip $(MONAI_LABEL_MODELS)),deepedit) at $(or $(strip $(MONAI_LABEL_PUBLIC_URL)),http://localhost:$(or $(strip $(MONAI_LABEL_PORT)),8030)),) diff --git a/trust/README.md b/trust/README.md index 0169fd60c..39ac0dcbe 100644 --- a/trust/README.md +++ b/trust/README.md @@ -181,9 +181,11 @@ Notes: back to HTTP downloads only for scans it cannot resolve there. - Pretrained weights are fetched on first start and persisted in the `monailabel-models` volume, so a container recreate does not re-download them. -- Currently **development/hybrid only**: the production stack runs published images and no - `monailabel` image is published yet. `MONAI_LABEL=true` with `PROD=true|stag` fails with a - message pointing here (see FLIP#55). +- Production (`PROD=true|stag`) runs the published `ghcr.io/londonaicentre/monailabel` image + (built by `docker_build_monailabel.yml`; dev builds locally from `trust/monailabel/`). + Enabling it under `PROD` prints a warning: it re-introduces the OHIF viewer plugin that trust + XNAT deliberately excludes (FLIP#662) — intended for hybrid/on-prem annotation trusts, not + high-volume cohort pullers. ## Running standalone (remote trust operator) diff --git a/trust/deploy/compose_trust.production.monailabel.yml b/trust/deploy/compose_trust.production.monailabel.yml new file mode 100644 index 000000000..44724a3ca --- /dev/null +++ b/trust/deploy/compose_trust.production.monailabel.yml @@ -0,0 +1,62 @@ +# Copyright (c) Guy's and St Thomas' NHS Foundation Trust & King's College London +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# MONAI Label overlay (production). Applied by trust/Makefile only when the kit sets +# MONAI_LABEL=true AND declares NUM_AVAILABLE_GPUS > 0 (see MONAILABEL_OVERRIDE). +# Image-only like the rest of the production compose — published by +# .github/workflows/docker_build_monailabel.yml. +# +# Enabling this re-introduces the XNAT OHIF viewer plugin that the trust XNAT +# deliberately excludes (FLIP#662 bulk-import livelock): intended for hybrid/on-prem +# annotation trusts, weigh it before enabling on a trust that pulls large cohorts. +# up-trust prints this warning when the flag is on. + +services: + + monailabel: + image: ghcr.io/londonaicentre/monailabel:${DOCKER_TAG} + restart: always + volumes: + # Read-only mount of the XNAT archive itself (not its parent — the datastore + # resolves files as + , so a parent + # path silently misses and falls back to per-scan HTTP downloads). Labels are + # written back through the XNAT REST API, never the filesystem. + - ${MONAI_LABEL_XNAT_ARCHIVE_DIR}:/workspace/xnat-data/archive:ro + # Pretrained weights are fetched on first start; persist them so a container + # recreate does not re-download every model (~GBs incl. the SAM checkpoint). + - monailabel-models:/workspace/app/radiology/model + environment: + - MONAI_LABEL_PORT=${MONAI_LABEL_PORT:-8030} + - MONAI_LABEL_DATASTORE=xnat + - MONAI_LABEL_DATASTORE_URL=${XNAT_URL:-http://xnat-web:8080} + - MONAI_LABEL_DATASTORE_USERNAME=${XNAT_SERVICE_USER} + - MONAI_LABEL_DATASTORE_PASSWORD=${XNAT_SERVICE_PASSWORD} + - MONAI_LABEL_DATASTORE_ASSET_PATH=/workspace/xnat-data/archive + # Comma-separated XNAT project allowlist; empty means every project. + - MONAI_LABEL_DATASTORE_PROJECT=${MONAI_LABEL_PROJECTS:-} + - MONAI_LABEL_MODELS=${MONAI_LABEL_MODELS:-deepedit} + # Reached from the clinician's BROWSER, never proxied by XNAT — must be + # browser-resolvable, and same-origin/TLS-fronted when XNAT is HTTPS. + - MONAI_LABEL_PUBLIC_URL=${MONAI_LABEL_PUBLIC_URL:-http://localhost:${MONAI_LABEL_PORT:-8030}} + ports: + - "${MONAI_LABEL_PORT:-8030}:${MONAI_LABEL_PORT:-8030}" + shm_size: ${MONAI_LABEL_SHM_SIZE:-8gb} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: ${NUM_AVAILABLE_GPUS} + capabilities: [gpu] + +volumes: + monailabel-models: