diff --git a/.github/workflows/terraform_apply.yml b/.github/workflows/terraform_apply.yml index b34ada653..70e0d4e13 100644 --- a/.github/workflows/terraform_apply.yml +++ b/.github/workflows/terraform_apply.yml @@ -186,6 +186,15 @@ jobs: K8S_TRUST_PUBLIC_IPS: ${{ vars.K8S_TRUST_PUBLIC_IPS }} JOB_RESOURCE_SPEC_NUM_GPUS: ${{ vars.JOB_RESOURCE_SPEC_NUM_GPUS }} JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB: ${{ vars.JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB }} + # Optional LZA keys (FLIP#749). Expected to be unset in the aws-stag / + # aws-prod environments — a platform-managed estate is applied from a + # laptop, and unset is the legacy value for all five. Passed through so + # compose-ci-env.sh's manifest stays the one place the mapping lives. + ACCESS_LOGS_BUCKET_NAME: ${{ vars.ACCESS_LOGS_BUCKET_NAME }} + CF_LOGS_BUCKET_NAME: ${{ vars.CF_LOGS_BUCKET_NAME }} + EFS_PROVISION_IMAGE: ${{ vars.EFS_PROVISION_IMAGE }} + LZA_VPC_NAME: ${{ vars.LZA_VPC_NAME }} + MANAGE_DNS: ${{ vars.MANAGE_DNS }} ADMIN_USER_PASSWORD: ${{ secrets.ADMIN_USER_PASSWORD }} AES_KEY_BASE64: ${{ secrets.AES_KEY_BASE64 }} INTERNAL_SERVICE_KEY: ${{ secrets.INTERNAL_SERVICE_KEY }} diff --git a/.github/workflows/terraform_drift.yml b/.github/workflows/terraform_drift.yml index b79bef854..24866b767 100644 --- a/.github/workflows/terraform_drift.yml +++ b/.github/workflows/terraform_drift.yml @@ -156,6 +156,15 @@ jobs: K8S_TRUST_PUBLIC_IPS: ${{ vars.K8S_TRUST_PUBLIC_IPS }} JOB_RESOURCE_SPEC_NUM_GPUS: ${{ vars.JOB_RESOURCE_SPEC_NUM_GPUS }} JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB: ${{ vars.JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB }} + # Optional LZA keys (FLIP#749). Expected to be unset in the aws-stag / + # aws-prod environments — a platform-managed estate is applied from a + # laptop, and unset is the legacy value for all five. Passed through so + # compose-ci-env.sh's manifest stays the one place the mapping lives. + ACCESS_LOGS_BUCKET_NAME: ${{ vars.ACCESS_LOGS_BUCKET_NAME }} + CF_LOGS_BUCKET_NAME: ${{ vars.CF_LOGS_BUCKET_NAME }} + EFS_PROVISION_IMAGE: ${{ vars.EFS_PROVISION_IMAGE }} + LZA_VPC_NAME: ${{ vars.LZA_VPC_NAME }} + MANAGE_DNS: ${{ vars.MANAGE_DNS }} ADMIN_USER_PASSWORD: ${{ secrets.ADMIN_USER_PASSWORD }} AES_KEY_BASE64: ${{ secrets.AES_KEY_BASE64 }} INTERNAL_SERVICE_KEY: ${{ secrets.INTERNAL_SERVICE_KEY }} diff --git a/.github/workflows/terraform_plan.yml b/.github/workflows/terraform_plan.yml index 6e093c183..391ab3af3 100644 --- a/.github/workflows/terraform_plan.yml +++ b/.github/workflows/terraform_plan.yml @@ -142,6 +142,15 @@ jobs: K8S_TRUST_PUBLIC_IPS: ${{ vars.K8S_TRUST_PUBLIC_IPS }} JOB_RESOURCE_SPEC_NUM_GPUS: ${{ vars.JOB_RESOURCE_SPEC_NUM_GPUS }} JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB: ${{ vars.JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB }} + # Optional LZA keys (FLIP#749). Expected to be unset in the aws-stag / + # aws-prod environments — a platform-managed estate is applied from a + # laptop, and unset is the legacy value for all five. Passed through so + # compose-ci-env.sh's manifest stays the one place the mapping lives. + ACCESS_LOGS_BUCKET_NAME: ${{ vars.ACCESS_LOGS_BUCKET_NAME }} + CF_LOGS_BUCKET_NAME: ${{ vars.CF_LOGS_BUCKET_NAME }} + EFS_PROVISION_IMAGE: ${{ vars.EFS_PROVISION_IMAGE }} + LZA_VPC_NAME: ${{ vars.LZA_VPC_NAME }} + MANAGE_DNS: ${{ vars.MANAGE_DNS }} ADMIN_USER_PASSWORD: ${{ secrets.ADMIN_USER_PASSWORD }} AES_KEY_BASE64: ${{ secrets.AES_KEY_BASE64 }} INTERNAL_SERVICE_KEY: ${{ secrets.INTERNAL_SERVICE_KEY }} diff --git a/.gitignore b/.gitignore index 216b8b4e9..a69b96df9 100644 --- a/.gitignore +++ b/.gitignore @@ -337,6 +337,8 @@ flip-fl-api/transfer/*/* .env.development .env.stag .env.production +.env.lza-prod +.env.lza-stag # Per-trust operator kits — one file per trust, named by trust CODE and # environment (e.g. trust/.env.KCH.development, trust/.env..production). # Minted by the hub via register-trust / POST /admin/trusts; contain plaintext diff --git a/AGENTS.md b/AGENTS.md index 8b2d44903..3568af266 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -412,9 +412,9 @@ After changes, evaluate if docs need updating: - `FL_PROVISIONED_DIR` — path to the in-tree provisioned FL artifacts, derived per-backend by `deploy/fl_backend.mk` from `FL_BACKEND`: `fl-services/nvflare/provision/workspace-dev` (nvflare startup kits) or `fl-services/flower/provision/creds` (flower per-net TLS certs + SuperNode keys). Both gitignored. Read only by the dev compose overlays for the cert/workspace volume mounts; override at the CLI for a one-off (`make up FL_PROVISIONED_DIR=...`). FL Makefiles are **per-backend** — each `fl-services//Makefile` owns that backend's `build`/`provision`/`up`/`down`/`submit` (flower also `up-secure`); the root Makefile forwards only `build-fl` by `FL_BACKEND`. Each backend's `fl-services//Makefile` also owns its network provisioning (NVFLARE adds `provision`/`provision-2-nets`/`provision-stag`/`provision-prod`/`upload-kits-to-s3`; the project YAMLs, `scripts/`, and gitignored `workspace-{dev,stag,prod}/` output live under `provision/`). Provision with `make -C fl-services/nvflare provision-2-nets` (nvflare) or `make -C fl-services/flower provision NET_NUMBER=` (flower). To run a backend standalone + submit without the full stack: `make -C fl-services/ up` (or `up-secure`) then `make -C fl-services/ submit APP=`. - `FL_APP_BASE_DIR` — Local directory holding the base FL application templates (the repo's `fl-apps/` tree), baked into the flip-api image and bind-mounted in dev. flip-api walks `///` to bundle an application (uploading into `FL_APP_DESTINATION_BUCKET/`) and reads each backend's manifest from `//required_files.json`. The walk is an **allowlist, not a mirror** (FLIP#1008): the bundle is the backend's app folder(s) — `app/`, plus NVFLARE's per-site `app_*/` — together with its single root file (`meta.json` for NVFLARE, `pyproject.toml` for Flower), and everything else in the template directory is excluded and listed at debug level. Inside an app folder, dot-prefixed entries, `__pycache__`, `*.pyc`/`*.pyo`, `uv.lock` and symlinks are dropped too. This matters most for an operator-provided tree: the walk fails **closed**, so an unrecognised file is silently omitted rather than shipped to a trust, and the omissions appear only in the debug log. Default `/app/fl-apps`; override to mount operator-provided templates. Replaces the removed `FL_APP_BASE_BUCKET` S3 dependency (FLIP#724): base templates are no longer published to S3 (the `fl-apps-push-s3-*` sync workflows are gone), so a template hotfix now ships by rebuilding + redeploying the flip-api image rather than syncing S3. `fl-apps/` is baked into the image via a BuildKit named build context (`fl_apps=../fl-apps`) since it sits outside flip-api's build context. For the Flower backend, the template pyprojects also steer Flower's **per-run dependency install** (`uv sync` on every app launch; SuperNodes opt in via `--allow-runtime-dependency-installation` in the composes): `[tool.uv.sources]` pins `flip-utils` to the source kept at `/opt/flip-utils` inside the FL images (never PyPI — FLIP#767; a flip-utils change ships by rebuilding the FL images, `make build-fl FL_BACKEND=flower`) and torch/torchvision to the cu128 index (PyPI's default cu130 wheels need driver >=580). - `FL_KIT_SLOT_NAMES` — JSON list (e.g. `["Trust_1", "Trust_2"]`) of FL kit-slot names for the hub's `fl_kit_slot` pool that `register_trust` claims from; each name must match a provisioned participant kit (in-tree workspace for dev, `s3:///fl-flare-participant-kits//net-/services//` for stag/prod; slot names are global across nets — every net carries a kit per name). The pool is seeded at flip-api boot and **reconciled on demand** when a registration finds it exhausted (`resolve_fl_kit_slot_names`, additive — never deletes or re-assigns rows); only then does `NoFreeKitSlotError` surface. Single source per env: dev = this env var (a `DevSettings`-only field; restart to change, settings load once); stag/prod = the `/flip/fl_kit_slot_names` SSM parameter (Terraform-rendered from this var — the list is plain config, not a secret; deliberately **no env fallback**, so a broken/missing parameter means the pool can't grow, loudly, never masked by stale task-def env). Growing the pool is an env-file edit + `make -C deploy/providers/AWS apply-fl-kit-slots` (targeted plan/apply of just the parameter, plain-text diff) — **no restart, no task-definition change**. One-command workflow: `make -C deploy/providers/AWS add-fl-kits N= PROD=stag|true` (N = "ensure N more live slots": activate spares toward N first, mint only the shortfall on every net → additive S3 upload → env edit → parameter apply); full runbook in `fl-services/nvflare/README.md` ("Onboarding a new client onto an existing network"). NVFLARE-only dynamics — Flower's SuperNode key labelling reads the list at net startup. -- `PROD` — `true` (production), `stag` (staging), unset (development) +- `PROD` — `true` (production), `stag` (staging), `lza` / `lza-stag` (production / staging on an AWS Landing Zone Accelerator estate, FLIP#749 — meaningful for `deploy/providers/AWS` targets, the FL-kit upload targets under `fl-services//`, and the kit-file targets here (`new-trust`, `sync-trust-kit[s]`), which must name `trust/.env..lza-prod` / `.lza-stag` the way the AWS side reads them back; select the root `.env.lza-prod` / `.env.lza-stag` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying onto an LZA estate"), unset (development) - `FLIP_INSTANCE` — names a **second dev hub** so two stacks can run on one host (FLIP#957). Unset (the norm) every derived name is exactly what it was before the knob existed. When set it prefixes four things, all of them names that are global to the docker daemon and so cannot be scoped by `-p`: the hub compose project name (`COMPOSE_PROJECT`), all six hub-side Docker networks, each trust's compose project (`TRUST_PROJECT`, `-trust`) and each trust's XNAT swarm stack (`XNAT_STACK`, `-xnat`). The last two carry it for the same reason the trust overlays do — FL kit slots are handed out per hub, so a second hub restarts its numbering at 1 and its first trust would otherwise adopt or deploy over the default stack's `trust1`/`xnat1`. Those six follow **one** name rule — `${FLIP_INSTANCE:+$FLIP_INSTANCE-}deploy_`, i.e. `deploy_central-hub-network`, `deploy_central-hub-trust-apis-network`, `deploy_fl-net-{1,2}` **and** `deploy_trust-network-{1,2}` — where `deploy_` names the hub compose project that owns the network, which is also exactly what compose would generate (`_`) if it still created them. It stopped generating anything once they became `external: true` ("the `name` field is used as is and is not scoped with the project name"), which is why the prefix is written out by hand in the composes, both Makefiles and `scripts/check_local_status.py`. FLIP#957 renamed **four** of them onto that rule: `central-hub-network` and `central-hub-trust-apis-network` had been left bare, and `shared-net-{1,2}` became `fl-net-{1,2}` — "shared" described who happened to be attached rather than what the network is for, and stopped being true the moment flip-api came off it. The two numbering axes are **not** parallel, which is the trap the new name defuses: `fl-net-` is numbered by **FL net** (the `fl_nets` table, `NET_NUMBER`, `fl-server-net-`) while `trust-network-` is numbered by **trust slot**, so one trust routinely sits on `trust-network-2` and `fl-net-1` at once. `fl-net-` is the FL **data plane**, and its membership is exactly two services — the hub's `fl-server-net-` and every trust's `fl-client-net-`, making it the FL twin of `central-hub-trust-apis-network` (flip-api ↔ trust-api). Everything else hub-side stays on `default` and reaches the FL server there over its control ports: **`flip-api` is deliberately not on it**, and neither are `fl-api-net-` or flower's `register-supernode-keys-net-`: it fronts the database, nothing on the FL data plane calls it (an fl-client carries no hub URL and no hub credential, only `TRUST_INTERNAL_SERVICE_KEY`), and the one callback that exists — fl-server's `FLIP_API_INTERNAL_URL` — goes over `default`, the hub-internal network, which both fl-servers now join. Because these networks are external and pre-created, all four renames are **not** transparent — an existing dev host must re-run `make create-networks`, recreate whatever was attached to the old ones, and `docker network rm` the four leftovers. The project name matters because compose derives it from the directory of the first `-f` file, always `deploy/`, so without it both stacks land in project `deploy` and `up` on one tears down the other. The networks are prefixed *separately* rather than left to `-p` because they are a cross-project contract: the hub project creates them and each trust — its own compose project — joins them `external: true` by literal name, which `-p` cannot scope. The trust overlays carry the prefix like everything else because a trust number alone does not isolate them: slots are handed out per-hub, so a second hub restarts its numbering at 1 and its first trust would otherwise land on the default stack's `deploy_trust-network-1`. Since every network belongs to exactly one instance, `make remove-networks FLIP_INSTANCE=` is instance-scoped and removes that instance's networks *including* its trust overlays. Both values are derived once in [`deploy/instance.mk`](deploy/instance.mk) (`INSTANCE_PREFIX` and `COMPOSE_PROJECT`), included by the root, trust, `trust/xnat`, flip-api and flip-ui Makefiles, which also exports `FLIP_INSTANCE` so the compose files see it; the compose files interpolate `${FLIP_INSTANCE}` rather than the make variable because they are also invoked directly, without make. **Container names are not prefixed and are not set at all** — no hub service declares `container_name`, in the development composes or the production ones, so compose names them from the project. On the default stack that is `deploy-flip-api-1`, `deploy-flip-db-1`, `deploy-flip-ui-1`, `deploy-pgadmin-1`, `deploy-fl-api-net-1-1`, `deploy-fl-server-net-1-1`; on a second stack `-deploy-…`. Since FLIP normally runs a single hub those default-stack names are deterministic, so **docs spell them out literally** — a concrete name is easier to read, copy and grep than a `docker compose -p deploy exec ` form — and only a doc explicitly about a second stack needs the `-deploy-…` variant. **Tooling must not**: a Makefile or script has to work on either instance, so it addresses the service through compose (`$(DOCKER_COMMAND) logs flip-api`) or resolves the container from the `com.docker.compose.project` / `.service` labels, as [`scripts/check_local_status.py`](scripts/check_local_status.py) does. (None of this reaches AWS: the production composes are a local prod-image harness, never a deployment target — on ECS the container names come from the task definitions in `deploy/providers/AWS/ecs_tasks.tf` and discovery from Cloud Map.) The compose **service keys** are the stable identity instead, and are load-bearing: docker registers each as a network alias on every network the container joins, which is how the FL kits keep resolving `fl-server-net-1` (`fed_client.json`'s `target`, and the SuperLink certificate's SAN) and how `NET_ENDPOINTS` reaches `fl-api-net-1`. Renaming a service key breaks TLS and the provisioned kits; renaming a container name breaks nothing. A second stack also needs its own value for every host port (`UI_PORT`, `API_PORT`, `DB_PORT`, `PGADMIN_PORT`, `FL_API_PORT`, `API_DEBUG_PORT`, `FL_API_DEBUG_PORT`, and on flower `FLOWER_SUPERLINK_NET_{1,2}_PORT`), its own `CENTRAL_HUB_API_URL` (it embeds `API_PORT`, so a copied env file silently points the second stack's UI and trust-api at the *first* stack's API — the one misconfiguration here that still starts cleanly and looks right), its own `XNAT_PORT` per trust kit, and its own trust numbers — all host-global and not covered by the prefix. Two traps when running one: `NET_ENDPOINTS` (which the hub seeds into `fl_nets` at boot) must carry the service name `fl-api-net-1`, never the pre-FLIP#957 `flip-fl-api-net-1`, which is now a DNS name in no stack at all; and `FL_PROVISIONED_DIR` may be given as either a relative or an absolute path (relative resolves against the repo root). -- `MAIN_ENV_FILE` — which repo-root env file the Makefiles load, as a bare **filename** (never a path). Defaults by `PROD` to `.env.production` / `.env.stag` / `.env.development`; override it to run a second stack from the *same checkout* — `make up MAIN_ENV_FILE=.env.b.development FLIP_INSTANCE=b` — instead of needing a second clone. The root Makefile `export`s it, and the flip-api, flip-ui and trust Makefiles each `include ../$(MAIN_ENV_FILE)`, which is why the value must stay a bare filename: each one prepends its own `../`. Every include is wildcard-guarded, so a value that resolves to nothing is **skipped silently** and the service builds with no environment at all rather than failing — check the `Using MAIN_ENV_FILE:` line each Makefile prints if a second stack comes up with empty config. `scripts/check_local_status.py` honours it too (via the environment), so a status check reports on the stack you are actually running. +- `MAIN_ENV_FILE` — which repo-root env file the Makefiles load, as a bare **filename** (never a path). Defaults by `PROD` to `.env.production` / `.env.stag` / `.env.lza-prod` / `.env.lza-stag` / `.env.development`; override it to run a second stack from the *same checkout* — `make up MAIN_ENV_FILE=.env.b.development FLIP_INSTANCE=b` — instead of needing a second clone. The root Makefile `export`s it, and the flip-api, flip-ui and trust Makefiles each `include ../$(MAIN_ENV_FILE)`, which is why the value must stay a bare filename: each one prepends its own `../`. Every include is wildcard-guarded, so a value that resolves to nothing is **skipped silently** and the service builds with no environment at all rather than failing — check the `Using MAIN_ENV_FILE:` line each Makefile prints if a second stack comes up with empty config. `scripts/check_local_status.py` honours it too (via the environment), so a status check reports on the stack you are actually running. - `DB_PORT` — in **dev** this is the *host* port flip-db is published on (the container side is pinned to 5432, which is what flip-api dials over the docker network); in `.env.stag` / `.env.production` it is the port the RDS Proxy listens on, since nothing is published there. Both default to 5432, so the difference only shows up when a second dev stack moves it. - `AES_KEY_BASE64` — encryption key for trust communication - A remote trust operator only needs their kit file (`trust/.env.`) — no hub `.env.` needed on trust hosts. diff --git a/CLAUDE.md b/CLAUDE.md index d527dabab..7de31c6d9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -412,9 +412,9 @@ After changes, evaluate if docs need updating: - `FL_PROVISIONED_DIR` — path to the in-tree provisioned FL artifacts, derived per-backend by `deploy/fl_backend.mk` from `FL_BACKEND`: `fl-services/nvflare/provision/workspace-dev` (nvflare startup kits) or `fl-services/flower/provision/creds` (flower per-net TLS certs + SuperNode keys). Both gitignored. Read only by the dev compose overlays for the cert/workspace volume mounts; override at the CLI for a one-off (`make up FL_PROVISIONED_DIR=...`). FL Makefiles are **per-backend** — each `fl-services//Makefile` owns that backend's `build`/`provision`/`up`/`down`/`submit` (flower also `up-secure`); the root Makefile forwards only `build-fl` by `FL_BACKEND`. Each backend's `fl-services//Makefile` also owns its network provisioning (NVFLARE adds `provision`/`provision-2-nets`/`provision-stag`/`provision-prod`/`upload-kits-to-s3`; the project YAMLs, `scripts/`, and gitignored `workspace-{dev,stag,prod}/` output live under `provision/`). Provision with `make -C fl-services/nvflare provision-2-nets` (nvflare) or `make -C fl-services/flower provision NET_NUMBER=` (flower). To run a backend standalone + submit without the full stack: `make -C fl-services/ up` (or `up-secure`) then `make -C fl-services/ submit APP=`. - `FL_APP_BASE_DIR` — Local directory holding the base FL application templates (the repo's `fl-apps/` tree), baked into the flip-api image and bind-mounted in dev. flip-api walks `///` to bundle an application (uploading into `FL_APP_DESTINATION_BUCKET/`) and reads each backend's manifest from `//required_files.json`. The walk is an **allowlist, not a mirror** (FLIP#1008): the bundle is the backend's app folder(s) — `app/`, plus NVFLARE's per-site `app_*/` — together with its single root file (`meta.json` for NVFLARE, `pyproject.toml` for Flower), and everything else in the template directory is excluded and listed at debug level. Inside an app folder, dot-prefixed entries, `__pycache__`, `*.pyc`/`*.pyo`, `uv.lock` and symlinks are dropped too. This matters most for an operator-provided tree: the walk fails **closed**, so an unrecognised file is silently omitted rather than shipped to a trust, and the omissions appear only in the debug log. Default `/app/fl-apps`; override to mount operator-provided templates. Replaces the removed `FL_APP_BASE_BUCKET` S3 dependency (FLIP#724): base templates are no longer published to S3 (the `fl-apps-push-s3-*` sync workflows are gone), so a template hotfix now ships by rebuilding + redeploying the flip-api image rather than syncing S3. `fl-apps/` is baked into the image via a BuildKit named build context (`fl_apps=../fl-apps`) since it sits outside flip-api's build context. For the Flower backend, the template pyprojects also steer Flower's **per-run dependency install** (`uv sync` on every app launch; SuperNodes opt in via `--allow-runtime-dependency-installation` in the composes): `[tool.uv.sources]` pins `flip-utils` to the source kept at `/opt/flip-utils` inside the FL images (never PyPI — FLIP#767; a flip-utils change ships by rebuilding the FL images, `make build-fl FL_BACKEND=flower`) and torch/torchvision to the cu128 index (PyPI's default cu130 wheels need driver >=580). - `FL_KIT_SLOT_NAMES` — JSON list (e.g. `["Trust_1", "Trust_2"]`) of FL kit-slot names for the hub's `fl_kit_slot` pool that `register_trust` claims from; each name must match a provisioned participant kit (in-tree workspace for dev, `s3:///fl-flare-participant-kits//net-/services//` for stag/prod; slot names are global across nets — every net carries a kit per name). The pool is seeded at flip-api boot and **reconciled on demand** when a registration finds it exhausted (`resolve_fl_kit_slot_names`, additive — never deletes or re-assigns rows); only then does `NoFreeKitSlotError` surface. Single source per env: dev = this env var (a `DevSettings`-only field; restart to change, settings load once); stag/prod = the `/flip/fl_kit_slot_names` SSM parameter (Terraform-rendered from this var — the list is plain config, not a secret; deliberately **no env fallback**, so a broken/missing parameter means the pool can't grow, loudly, never masked by stale task-def env). Growing the pool is an env-file edit + `make -C deploy/providers/AWS apply-fl-kit-slots` (targeted plan/apply of just the parameter, plain-text diff) — **no restart, no task-definition change**. One-command workflow: `make -C deploy/providers/AWS add-fl-kits N= PROD=stag|true` (N = "ensure N more live slots": activate spares toward N first, mint only the shortfall on every net → additive S3 upload → env edit → parameter apply); full runbook in `fl-services/nvflare/README.md` ("Onboarding a new client onto an existing network"). NVFLARE-only dynamics — Flower's SuperNode key labelling reads the list at net startup. -- `PROD` — `true` (production), `stag` (staging), unset (development) +- `PROD` — `true` (production), `stag` (staging), `lza` / `lza-stag` (production / staging on an AWS Landing Zone Accelerator estate, FLIP#749 — meaningful for `deploy/providers/AWS` targets, the FL-kit upload targets under `fl-services//`, and the kit-file targets here (`new-trust`, `sync-trust-kit[s]`), which must name `trust/.env..lza-prod` / `.lza-stag` the way the AWS side reads them back; select the root `.env.lza-prod` / `.env.lza-stag` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying onto an LZA estate"), unset (development) - `FLIP_INSTANCE` — names a **second dev hub** so two stacks can run on one host (FLIP#957). Unset (the norm) every derived name is exactly what it was before the knob existed. When set it prefixes four things, all of them names that are global to the docker daemon and so cannot be scoped by `-p`: the hub compose project name (`COMPOSE_PROJECT`), all six hub-side Docker networks, each trust's compose project (`TRUST_PROJECT`, `-trust`) and each trust's XNAT swarm stack (`XNAT_STACK`, `-xnat`). The last two carry it for the same reason the trust overlays do — FL kit slots are handed out per hub, so a second hub restarts its numbering at 1 and its first trust would otherwise adopt or deploy over the default stack's `trust1`/`xnat1`. Those six follow **one** name rule — `${FLIP_INSTANCE:+$FLIP_INSTANCE-}deploy_`, i.e. `deploy_central-hub-network`, `deploy_central-hub-trust-apis-network`, `deploy_fl-net-{1,2}` **and** `deploy_trust-network-{1,2}` — where `deploy_` names the hub compose project that owns the network, which is also exactly what compose would generate (`_`) if it still created them. It stopped generating anything once they became `external: true` ("the `name` field is used as is and is not scoped with the project name"), which is why the prefix is written out by hand in the composes, both Makefiles and `scripts/check_local_status.py`. FLIP#957 renamed **four** of them onto that rule: `central-hub-network` and `central-hub-trust-apis-network` had been left bare, and `shared-net-{1,2}` became `fl-net-{1,2}` — "shared" described who happened to be attached rather than what the network is for, and stopped being true the moment flip-api came off it. The two numbering axes are **not** parallel, which is the trap the new name defuses: `fl-net-` is numbered by **FL net** (the `fl_nets` table, `NET_NUMBER`, `fl-server-net-`) while `trust-network-` is numbered by **trust slot**, so one trust routinely sits on `trust-network-2` and `fl-net-1` at once. `fl-net-` is the FL **data plane**, and its membership is exactly two services — the hub's `fl-server-net-` and every trust's `fl-client-net-`, making it the FL twin of `central-hub-trust-apis-network` (flip-api ↔ trust-api). Everything else hub-side stays on `default` and reaches the FL server there over its control ports: **`flip-api` is deliberately not on it**, and neither are `fl-api-net-` or flower's `register-supernode-keys-net-`: it fronts the database, nothing on the FL data plane calls it (an fl-client carries no hub URL and no hub credential, only `TRUST_INTERNAL_SERVICE_KEY`), and the one callback that exists — fl-server's `FLIP_API_INTERNAL_URL` — goes over `default`, the hub-internal network, which both fl-servers now join. Because these networks are external and pre-created, all four renames are **not** transparent — an existing dev host must re-run `make create-networks`, recreate whatever was attached to the old ones, and `docker network rm` the four leftovers. The project name matters because compose derives it from the directory of the first `-f` file, always `deploy/`, so without it both stacks land in project `deploy` and `up` on one tears down the other. The networks are prefixed *separately* rather than left to `-p` because they are a cross-project contract: the hub project creates them and each trust — its own compose project — joins them `external: true` by literal name, which `-p` cannot scope. The trust overlays carry the prefix like everything else because a trust number alone does not isolate them: slots are handed out per-hub, so a second hub restarts its numbering at 1 and its first trust would otherwise land on the default stack's `deploy_trust-network-1`. Since every network belongs to exactly one instance, `make remove-networks FLIP_INSTANCE=` is instance-scoped and removes that instance's networks *including* its trust overlays. Both values are derived once in [`deploy/instance.mk`](deploy/instance.mk) (`INSTANCE_PREFIX` and `COMPOSE_PROJECT`), included by the root, trust, `trust/xnat`, flip-api and flip-ui Makefiles, which also exports `FLIP_INSTANCE` so the compose files see it; the compose files interpolate `${FLIP_INSTANCE}` rather than the make variable because they are also invoked directly, without make. **Container names are not prefixed and are not set at all** — no hub service declares `container_name`, in the development composes or the production ones, so compose names them from the project. On the default stack that is `deploy-flip-api-1`, `deploy-flip-db-1`, `deploy-flip-ui-1`, `deploy-pgadmin-1`, `deploy-fl-api-net-1-1`, `deploy-fl-server-net-1-1`; on a second stack `-deploy-…`. Since FLIP normally runs a single hub those default-stack names are deterministic, so **docs spell them out literally** — a concrete name is easier to read, copy and grep than a `docker compose -p deploy exec ` form — and only a doc explicitly about a second stack needs the `-deploy-…` variant. **Tooling must not**: a Makefile or script has to work on either instance, so it addresses the service through compose (`$(DOCKER_COMMAND) logs flip-api`) or resolves the container from the `com.docker.compose.project` / `.service` labels, as [`scripts/check_local_status.py`](scripts/check_local_status.py) does. (None of this reaches AWS: the production composes are a local prod-image harness, never a deployment target — on ECS the container names come from the task definitions in `deploy/providers/AWS/ecs_tasks.tf` and discovery from Cloud Map.) The compose **service keys** are the stable identity instead, and are load-bearing: docker registers each as a network alias on every network the container joins, which is how the FL kits keep resolving `fl-server-net-1` (`fed_client.json`'s `target`, and the SuperLink certificate's SAN) and how `NET_ENDPOINTS` reaches `fl-api-net-1`. Renaming a service key breaks TLS and the provisioned kits; renaming a container name breaks nothing. A second stack also needs its own value for every host port (`UI_PORT`, `API_PORT`, `DB_PORT`, `PGADMIN_PORT`, `FL_API_PORT`, `API_DEBUG_PORT`, `FL_API_DEBUG_PORT`, and on flower `FLOWER_SUPERLINK_NET_{1,2}_PORT`), its own `CENTRAL_HUB_API_URL` (it embeds `API_PORT`, so a copied env file silently points the second stack's UI and trust-api at the *first* stack's API — the one misconfiguration here that still starts cleanly and looks right), its own `XNAT_PORT` per trust kit, and its own trust numbers — all host-global and not covered by the prefix. Two traps when running one: `NET_ENDPOINTS` (which the hub seeds into `fl_nets` at boot) must carry the service name `fl-api-net-1`, never the pre-FLIP#957 `flip-fl-api-net-1`, which is now a DNS name in no stack at all; and `FL_PROVISIONED_DIR` may be given as either a relative or an absolute path (relative resolves against the repo root). -- `MAIN_ENV_FILE` — which repo-root env file the Makefiles load, as a bare **filename** (never a path). Defaults by `PROD` to `.env.production` / `.env.stag` / `.env.development`; override it to run a second stack from the *same checkout* — `make up MAIN_ENV_FILE=.env.b.development FLIP_INSTANCE=b` — instead of needing a second clone. The root Makefile `export`s it, and the flip-api, flip-ui and trust Makefiles each `include ../$(MAIN_ENV_FILE)`, which is why the value must stay a bare filename: each one prepends its own `../`. Every include is wildcard-guarded, so a value that resolves to nothing is **skipped silently** and the service builds with no environment at all rather than failing — check the `Using MAIN_ENV_FILE:` line each Makefile prints if a second stack comes up with empty config. `scripts/check_local_status.py` honours it too (via the environment), so a status check reports on the stack you are actually running. +- `MAIN_ENV_FILE` — which repo-root env file the Makefiles load, as a bare **filename** (never a path). Defaults by `PROD` to `.env.production` / `.env.stag` / `.env.lza-prod` / `.env.lza-stag` / `.env.development`; override it to run a second stack from the *same checkout* — `make up MAIN_ENV_FILE=.env.b.development FLIP_INSTANCE=b` — instead of needing a second clone. The root Makefile `export`s it, and the flip-api, flip-ui and trust Makefiles each `include ../$(MAIN_ENV_FILE)`, which is why the value must stay a bare filename: each one prepends its own `../`. Every include is wildcard-guarded, so a value that resolves to nothing is **skipped silently** and the service builds with no environment at all rather than failing — check the `Using MAIN_ENV_FILE:` line each Makefile prints if a second stack comes up with empty config. `scripts/check_local_status.py` honours it too (via the environment), so a status check reports on the stack you are actually running. - `DB_PORT` — in **dev** this is the *host* port flip-db is published on (the container side is pinned to 5432, which is what flip-api dials over the docker network); in `.env.stag` / `.env.production` it is the port the RDS Proxy listens on, since nothing is published there. Both default to 5432, so the difference only shows up when a second dev stack moves it. - `AES_KEY_BASE64` — encryption key for trust communication - A remote trust operator only needs their kit file (`trust/.env.`) — no hub `.env.` needed on trust hosts. diff --git a/Makefile b/Makefile index e73313994..1a23a6d15 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,18 @@ else ifeq ($(PROD),stag) MAIN_ENV_FILE=.env.stag __DCKR_SUFFIX=production ENV=stag +# LZA estate (FLIP#749): the hub itself deploys via deploy/providers/AWS, but the +# kit-file targets here (new-trust, sync-trust-kit[s]) must name +# trust/.env..$(ENV) the way that Makefile's KIT_ENV_SUFFIX and +# register-trusts.sh read it back — .lza-prod / .lza-stag, never .stag. +else ifeq ($(PROD),lza) +MAIN_ENV_FILE=.env.lza-prod +__DCKR_SUFFIX=production +ENV=lza-prod +else ifeq ($(PROD),lza-stag) +MAIN_ENV_FILE=.env.lza-stag +__DCKR_SUFFIX=production +ENV=lza-stag else MAIN_ENV_FILE=.env.development __DCKR_SUFFIX=development diff --git a/deploy/README.md b/deploy/README.md index fee5d341a..a6a17ccd8 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -271,8 +271,12 @@ This runbook is for the case where **you** have lost access to your TOTP device The Central Hub has **one supported production deployment**: ECS Fargate via the Terraform root in [`deploy/providers/AWS/`](providers/AWS/README.md). The task definitions in `ecs_tasks.tf` (env maps in -`locals.tf`) are the **canonical definition of production container config**. Deploying into an AWS -LZA-governed account is an env-gated **mode** of that same root, not a separate path +`locals.tf`) are the **canonical definition of production container config**. That root offers **two +deployment modes**, both permanently supported: the default **self-contained single-account** shape +(`PROD=stag`/`PROD=true`), where FLIP creates its own VPC and edge; and the **platform-managed** shape +(`PROD=lza`, or `PROD=lza-stag` for a staging estate) for an AWS Landing Zone Accelerator estate, where the +network and edge are owned by the accelerator. The second is an env-gated mode of the same root, not a separate path — see +[Deploying onto an LZA estate](providers/AWS/README.md#deploying-onto-an-lza-estate-prodlza) ([FLIP#749](https://github.com/londonaicentre/FLIP/issues/749)). The ECS FL task definitions serve **both FL backends** ([FLIP#566](https://github.com/londonaicentre/FLIP/issues/566)): `FL_BACKEND` in the env file switches the same task families between NVFLARE and Flower (SuperLink ports/command/creds — Flower @@ -286,7 +290,8 @@ Cloud Map + public FL hostnames). > `compose.production*.yml` files remain maintained **only** as the local prod-image harness > (`make up PROD=stag|true` — the baked images, no dev mounts). When changing production config, change > Terraform first and update the compose files only as far as the local harness needs. Remaining hub-EC2 -> material is removed once the LZA migration's legacy decommission lands (FLIP#749 WP6). +> material is removed once the AI Centre's own legacy account is decommissioned (FLIP#749 WP6) — that +> retires one *account*, not the self-contained deployment mode, which stays supported. ### Trusts diff --git a/deploy/providers/AWS/AGENTS.md b/deploy/providers/AWS/AGENTS.md index b75582ec8..32b42b510 100644 --- a/deploy/providers/AWS/AGENTS.md +++ b/deploy/providers/AWS/AGENTS.md @@ -5,6 +5,8 @@ | File | Resources | |------|-----------| | `main.tf` | Provider config, VPC, subnets, IGW, NAT, route tables, RDS instance, Secrets Manager, SES | +| `network_lza.tf` | LZA platform-managed network (FLIP#749): VPC/subnet data lookups + the `local.vpc_id` / `local.app_subnet_ids` / `local.data_subnet_ids` locals both paths consume | +| `fl_ingress_lza.tf` | LZA-only ingress (FLIP#749 WP3, ported from the #829 e2e harness): internal FL NLB with static per-subnet IPs + its TG/SG, the ALB ingress rule for the networking-account relay path, and the `/flip/networking/*` edge-handoff SSM params | | `services.tf` | S3 buckets, Cognito | | `rds_proxy.tf` | RDS Proxy + IAM DB auth (proxy, IAM role/policy, SG, `rds-db:connect`) — see FLIP#556 | | `ecs.tf` | ECS cluster, capacity providers, ECS CloudWatch log groups (ALB / NLB / target groups / listener rules live in `main.tf`) | @@ -28,13 +30,16 @@ | ------- | ------------- | --------- | | `stag` | Staging | `flipstag` | | `prod` | Production | `flipprod` | -| `FlipDeveloperAccess-080369786334` | Developer access | — | +| `lza-prod` | LZA estate, production (`PROD=lza`, FLIP#749; `FLIPAdminAccess` permission set) | `FLIPProduction` | +| `lza-stag` | LZA estate, staging (`PROD=lza-stag`; `FLIPAdminAccess` permission set) | staging workload account (provisioning tracked on FLIP#749) | +| `dev` | Development (the `dev/` root: Cognito + SES; `FlipDeveloperAccess` permission set) | `flipdev` | ## Key Deploy Commands ```bash make full-deploy PROD=stag # Full staging deploy make full-deploy PROD=true # Full prod deploy +make init/plan/apply PROD=lza # LZA estate, platform-managed network (env-gated; full-deploy chains untested there — see README "Deploying onto an LZA estate"). PROD=lza-stag = staging semantics on the same mode (requires LZA_VPC_NAME in .env.lza-stag) make full-deploy-hybrid PROD= [LOCAL_TRUST_IP=] # Hybrid with on-prem trust make full-deploy-hub-only PROD= # Hub only, NO cloud Trust EC2 (all trusts on-prem, e.g. GPU hosts) — see README "Hub-only Deployment" make init/plan/apply # Terraform workflow diff --git a/deploy/providers/AWS/CLAUDE.md b/deploy/providers/AWS/CLAUDE.md index 5ca5d01ad..f98fadeb6 100644 --- a/deploy/providers/AWS/CLAUDE.md +++ b/deploy/providers/AWS/CLAUDE.md @@ -5,6 +5,8 @@ | File | Resources | |------|-----------| | `main.tf` | Provider config, VPC, subnets, IGW, NAT, route tables, RDS instance, Secrets Manager, SES | +| `network_lza.tf` | LZA platform-managed network (FLIP#749): VPC/subnet data lookups + the `local.vpc_id` / `local.app_subnet_ids` / `local.data_subnet_ids` locals both paths consume | +| `fl_ingress_lza.tf` | LZA-only ingress (FLIP#749 WP3, ported from the #829 e2e harness): internal FL NLB with static per-subnet IPs + its TG/SG, the ALB ingress rule for the networking-account relay path, and the `/flip/networking/*` edge-handoff SSM params | | `services.tf` | S3 buckets, Cognito | | `rds_proxy.tf` | RDS Proxy + IAM DB auth (proxy, IAM role/policy, SG, `rds-db:connect`) — see FLIP#556 | | `ecs.tf` | ECS cluster, capacity providers, ECS CloudWatch log groups (ALB / NLB / target groups / listener rules live in `main.tf`) | @@ -28,13 +30,16 @@ | ------- | ------------- | --------- | | `stag` | Staging | `flipstag` | | `prod` | Production | `flipprod` | -| `FlipDeveloperAccess-080369786334` | Developer access | — | +| `lza-prod` | LZA estate, production (`PROD=lza`, FLIP#749; `FLIPAdminAccess` permission set) | `FLIPProduction` | +| `lza-stag` | LZA estate, staging (`PROD=lza-stag`; `FLIPAdminAccess` permission set) | staging workload account (provisioning tracked on FLIP#749) | +| `dev` | Development (the `dev/` root: Cognito + SES; `FlipDeveloperAccess` permission set) | `flipdev` | ## Key Deploy Commands ```bash make full-deploy PROD=stag # Full staging deploy make full-deploy PROD=true # Full prod deploy +make init/plan/apply PROD=lza # LZA estate, platform-managed network (env-gated; full-deploy chains untested there — see README "Deploying onto an LZA estate"). PROD=lza-stag = staging semantics on the same mode (requires LZA_VPC_NAME in .env.lza-stag) make full-deploy-hybrid PROD= [LOCAL_TRUST_IP=] # Hybrid with on-prem trust make full-deploy-hub-only PROD= # Hub only, NO cloud Trust EC2 (all trusts on-prem, e.g. GPU hosts) — see README "Hub-only Deployment" make init/plan/apply # Terraform workflow diff --git a/deploy/providers/AWS/Makefile b/deploy/providers/AWS/Makefile index 78fa807d8..b6fab3507 100644 --- a/deploy/providers/AWS/Makefile +++ b/deploy/providers/AWS/Makefile @@ -12,6 +12,18 @@ ifeq ($(PROD),true) MAIN_ENV_FILE=../../../.env.production +else ifeq ($(PROD),lza) +# LZA estate (FLIP#749): a prod-grade environment on a platform-managed (AWS +# Landing Zone Accelerator) network, the second of the two supported deployment +# modes. Gitignored like the other env files — see the "Deploying onto an LZA +# estate" section in README.md for the required values. +MAIN_ENV_FILE=../../../.env.lza-prod +else ifeq ($(PROD),lza-stag) +# Staging on an LZA estate: same platform-managed mode as PROD=lza, but in the +# staging workload account with staging semantics (TF_VAR_environment=stag, so +# the prod-only RDS hardening stays off, and deploys track origin/develop like +# legacy stag). Requires LZA_VPC_NAME in the env file — see the guard below. +MAIN_ENV_FILE=../../../.env.lza-stag else # Sometimes PROD may not be set, default to staging environment in that case PROD=stag @@ -54,6 +66,11 @@ $(if $(FL_KIT_DATE),,$(error 🧠 $(if $(filter nvflare,$(FL_BACKEND)),FLARE_KIT # the env file or on the command line if your local profile names differ. PROD_AWS_PROFILE ?= prod STAG_AWS_PROFILE ?= stag +# The LZA workload accounts are reached via the Identity Center +# `FLIPAdminAccess` permission set; like prod/stag above, the defaults are +# short local aliases for it in ~/.aws/config (FLIP#749). +LZA_AWS_PROFILE ?= lza-prod +LZA_STAG_AWS_PROFILE ?= lza-stag # Refuse to operate against an unintended AWS account. Profile name is the safety # guard because Make has no easy way to read the account ID without an `aws` call. @@ -61,6 +78,20 @@ ifeq ($(PROD),true) ifneq ($(AWS_PROFILE),$(PROD_AWS_PROFILE)) $(error "For production deployments, AWS_PROFILE must be set to '$(PROD_AWS_PROFILE)' (override via PROD_AWS_PROFILE)") endif +else ifeq ($(PROD),lza) +ifneq ($(AWS_PROFILE),$(LZA_AWS_PROFILE)) +$(error "For LZA deployments, AWS_PROFILE must be set to '$(LZA_AWS_PROFILE)' (override via LZA_AWS_PROFILE)") +endif +else ifeq ($(PROD),lza-stag) +ifneq ($(AWS_PROFILE),$(LZA_STAG_AWS_PROFILE)) +$(error "For LZA staging deployments, AWS_PROFILE must be set to '$(LZA_STAG_AWS_PROFILE)' (override via LZA_STAG_AWS_PROFILE)") +endif +# The variables.tf default VPC name is the PROD template (AWSAccelerator-eu-west-2-prod); +# there is no defaultable name for the staging VPC, so require it explicitly rather +# than letting the data lookup fail with an opaque "no results" in the wrong account. +ifeq ($(LZA_VPC_NAME),) +$(error "For LZA staging deployments, LZA_VPC_NAME must be set in .env.lza-stag (the Name tag of the staging platform VPC)") +endif else ifneq ($(AWS_PROFILE),$(STAG_AWS_PROFILE)) $(error "For staging deployments, AWS_PROFILE must be set to '$(STAG_AWS_PROFILE)' (override via STAG_AWS_PROFILE)") @@ -68,13 +99,31 @@ endif endif ## SETTING VARIABLES FOR TERRAFORM -# Map the Make-level PROD flag (true|stag) onto a Terraform-friendly environment name. -# Used in main.tf to gate prod-only RDS hardening (deletion protection, final snapshot). -export TF_VAR_environment=$(if $(filter true,$(PROD)),prod,stag) +# Map the Make-level PROD flag (true|lza|lza-stag|stag) onto a Terraform-friendly +# environment name. Used in main.tf to gate prod-only RDS hardening (deletion +# protection, final snapshot). PROD=lza is a production estate, so it keeps the prod +# hardening; PROD=lza-stag is staging semantics on the platform-managed network — +# the two axes are orthogonal, toggled by the flag below (FLIP#749). +export TF_VAR_environment=$(if $(filter true lza,$(PROD)),prod,stag) + +# Platform-managed network toggle (FLIP#749): on an LZA workload account the VPC +# layer is provisioned by the AWS Accelerator pipeline and SCP-denied to this +# stack, so Terraform switches from creating the network to data lookups +# (see network_lza.tf). Off for every legacy environment. +export TF_VAR_lza_managed_network=$(if $(filter lza lza-stag,$(PROD)),true,false) +# Name tag of the platform-managed VPC; only consulted on the LZA modes. Set +# LZA_VPC_NAME in the env file to override the variables.tf default +# (AWSAccelerator-eu-west-2-prod) — optional on PROD=lza if the platform template +# keeps that name, mandatory on PROD=lza-stag (guard above). +ifneq ($(LZA_VPC_NAME),) +export TF_VAR_lza_vpc_name=$(LZA_VPC_NAME) +endif -# Kit-file env suffix: PROD=true → .production kits, otherwise → .stag. Selects -# the trust kit file trust/.env.. for the trust-deploy targets. -KIT_ENV_SUFFIX := $(if $(filter true,$(PROD)),production,stag) +# Kit-file env suffix: PROD=true → .production kits, PROD=lza → .lza-prod kits (a +# separate namespace so the parallel-running legacy prod kits are never touched +# during the FLIP#749 migration), PROD=lza-stag → .lza-stag kits, otherwise → .stag. +# Selects the trust kit file trust/.env.. for the trust-deploy targets. +KIT_ENV_SUFFIX := $(if $(filter true,$(PROD)),production,$(if $(filter lza,$(PROD)),lza-prod,$(if $(filter lza-stag,$(PROD)),lza-stag,stag))) export TF_VAR_max_azs=2 export TF_VAR_vpc_cidr=10.0.0.0/16 export TF_VAR_public_subnets=["10.0.1.0/24", "10.0.2.0/24"] @@ -87,6 +136,9 @@ export TF_VAR_FLIP_MODEL_FILES_UPLOADS_BUCKET_NAME=${FLIP_MODEL_FILES_UPLOADS_BU export TF_VAR_FLIP_FL_RESULTS_BUCKET_NAME=${FLIP_FL_RESULTS_BUCKET_NAME} export TF_VAR_FLIP_APP_BUNDLES_BUCKET_NAME=${FLIP_APP_BUNDLES_BUCKET_NAME} export TF_VAR_AICENTRE_BUCKET_NAME=${AICENTRE_BUCKET_NAME} +# Optional log-bucket name overrides (empty = derived from flip_alb_subdomain; see variables.tf) +export TF_VAR_ACCESS_LOGS_BUCKET_NAME=${ACCESS_LOGS_BUCKET_NAME} +export TF_VAR_CF_LOGS_BUCKET_NAME=${CF_LOGS_BUCKET_NAME} # A copied-but-unedited .env leaves `` placeholders in place. Those are non-empty, so an # emptiness check passes them straight through to Terraform, which fails much later with an opaque # AWS bucket-name error. Treat a placeholder as unset — same idea as XNAT_CREDENTIAL_PLACEHOLDER in @@ -136,6 +188,14 @@ endif export TF_VAR_docker_image_tag=${DOCKER_TAG} export TF_VAR_flip_fl_image_tag=${DOCKER_FL_TAG} export TF_VAR_docker_registry=${DOCKER_REGISTRY} +# One-shot EFS-provision utility image (default amazon/aws-cli from Docker Hub +# in variables.tf). The LZA account has no internet egress, so .env.lza-prod +# points this at the in-account ECR Public pull-through cache mirror +# (…/ecr-public/aws-cli/aws-cli:, FLIP#749). Export only when set so +# legacy envs keep the variables.tf default rather than an empty string. +ifneq ($(EFS_PROVISION_IMAGE),) +export TF_VAR_efs_provision_image=${EFS_PROVISION_IMAGE} +endif export TF_VAR_fl_api_name=${DOCKER_FL_API_NAME} export TF_VAR_fl_server_name=${DOCKER_FL_SERVER_NAME} export TF_VAR_fl_client_name=${DOCKER_FL_CLIENT_NAME} @@ -165,6 +225,13 @@ DEPLOY_TRUST_EC2 ?= true export TF_VAR_deploy_trust_ec2=$(DEPLOY_TRUST_EC2) export TF_VAR_AWS_REGION=${AWS_REGION} export TF_VAR_VPC_NAME=${VPC_NAME} +# DNS availability (FLIP#749): the LZA account has no Route53 hosted zone until +# the platform-side DNS migration lands, so .env.lza-prod sets MANAGE_DNS=false +# for the first bring-up (default CloudFront domain, no custom certs/records) and +# flips it to true once the zone exists there. Unset (legacy prod/stag) ⇒ true — +# behaviour unchanged. +MANAGE_DNS ?= true +export TF_VAR_manage_dns=$(MANAGE_DNS) export TF_VAR_ADMIN_USER_PASSWORD=${ADMIN_USER_PASSWORD} export TF_VAR_SES_VERIFIED_EMAIL=${SES_VERIFIED_EMAIL} export TF_VAR_flip_alb_subdomain=${ALB_SUBDOMAIN} @@ -193,7 +260,9 @@ CENTRALHUB_ECS_SERVICES = flip-api fl-api-net-1 fl-server-net-1 # recorded commit and rollback-centralhub can repoint at the previous revision. The # env-file tags (DOCKER_TAG / DOCKER_FL_TAG) remain the bootstrap-only defaults Terraform # bakes into a (re)created task-definition family — day-to-day deploys never touch them. -DEPLOY_GIT_REF := $(if $(filter true,$(PROD)),origin/main,origin/develop) +# PROD=lza tracks origin/main like legacy prod (both are production estates, FLIP#749); +# PROD=lza-stag falls through to origin/develop like legacy stag. +DEPLOY_GIT_REF := $(if $(filter true lza,$(PROD)),origin/main,origin/develop) TRUST_IMAGES = \ $(DOCKER_REGISTRY)trust-api:$(DOCKER_TAG) \ @@ -817,12 +886,21 @@ deploy-ui: ## Build flip-ui from the working tree and deploy it to S3 + invalida $(_AWS_ENV) aws s3 cp ../../../flip-ui/dist/js/window.js s3://$$UI_BUCKET/js/window.js \ --cache-control "no-cache, no-store, must-revalidate" \ --content-type "application/javascript; charset=utf-8" + @# On LZA there is no workload distribution to invalidate (cloudfront.tf gates + @# it off; the networking account's edge distribution serves the bucket) and no + @# cross-account invalidation permission -- BY DESIGN (FLIP#749): the hashed + @# bundles are immutable and index.html/window.js carry no-cache metadata, so a + @# deploy is visible on the next index.html fetch with no invalidation at all. +ifneq ($(filter lza lza-stag,$(PROD)),) + @echo "🧹 Skipping CloudFront invalidation on LZA (edge distribution honours the Cache-Control metadata)." +else @echo "🧹 Invalidating CloudFront..." @DIST_ID=$$($(_AWS_ENV) terraform output -raw CloudfrontDistributionId); \ [ -n "$$DIST_ID" ] || { echo "❌ Terraform output CloudfrontDistributionId is empty"; exit 1; }; \ $(_AWS_ENV) aws cloudfront create-invalidation \ --distribution-id "$$DIST_ID" \ --paths "/*" >/dev/null +endif @echo "✅ deploy-ui complete." .PHONY: deploy-ark-demo diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index e907438c2..bc86fbe77 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -38,7 +38,7 @@ In both models, trusts poll the Central Hub for tasks over HTTPS — all communi 5. **GitHub CLI** installed via [GitHub CLI installation guide](https://cli.github.com/) 6. **SSH key pair** created at `~/.ssh/host-aws` (see [deploy README](../../README.md)) 7. **Environment files** configured: (see [deploy README](../../README.md)) - - `.env.stag` (staging) or `.env.production` (production) in project root + - `.env.stag` (staging), `.env.production` (production), or `.env.lza-prod` (LZA FLIPProduction) in project root - Service-specific `.env` files (see Environment Configuration section) ### Required AWS Permissions @@ -536,6 +536,13 @@ not empty the list** — `modules/cognito` defaults it to `["https://localhost:4 puts a localhost origin into the production allowlist. `tests/test_cognito_callback_urls.py` guards both directions, and runs in CI as the `AWS deploy tests` job of `validate_terraform.yml`. +The stag/prod list holds one entry, and it is the expression `local.ui_origin` rather than a literal +URL: the canonical UI origin differs between a DNS-managed environment (`https://` +— every legacy environment) and a zone-less LZA bring-up, where it is the reachable edge/CloudFront +domain. `cloudfront.tf` resolves it once, and the same local feeds the S3 bucket CORS rules, so the +two allowlists cannot drift apart. Adding a *second* origin means adding a literal entry alongside it; +the guard follows the reference into `cloudfront.tf`, so a localhost origin is caught either way. + To change it: ```bash @@ -644,10 +651,13 @@ The `PROD` variable determines which environment files are loaded: - `PROD=stag` → Uses the root `.env.stag` - `PROD=true` → Uses the root `.env.production` +- `PROD=lza` → Uses the root `.env.lza-prod` (production on an LZA-governed estate — see + [Deploying onto an LZA estate](#deploying-onto-an-lza-estate-prodlza)) +- `PROD=lza-stag` → Uses the root `.env.lza-stag` (staging on an LZA-governed estate) If `PROD` is omitted when running the AWS provider Makefile, it defaults to staging. -The Makefile maps `PROD` onto `TF_VAR_environment` (`prod` when `PROD=true`, otherwise `stag`). Terraform branches on this variable to gate prod-only RDS hardening — see [RDS lifecycle](#rds-lifecycle-stag-vs-prod). +The Makefile maps `PROD` onto `TF_VAR_environment` (`prod` when `PROD=true` or `PROD=lza`, otherwise `stag`). Terraform branches on this variable to gate prod-only RDS hardening — see [RDS lifecycle](#rds-lifecycle-stag-vs-prod). `PROD=lza` additionally sets the orthogonal `TF_VAR_lza_managed_network=true` platform-managed-network toggle. #### AWS profile aliases @@ -698,6 +708,247 @@ make apply See [`dev/README.md`](./dev/README.md) for the first-time setup workflow (the dev resources are Terraform-managed from day one; there is no import step). +### Deploying onto an LZA estate (PROD=lza) + +This Terraform root supports **two deployment modes**, both permanently: + +| Mode | Selected by | Network | Ingress | +| --- | --- | --- | --- | +| **Self-contained** (default) | `PROD=stag` / `PROD=true` | FLIP creates its own VPC, subnets, IGW, NAT | In-account CloudFront + public FL NLB | +| **Platform-managed (LZA)** | `PROD=lza` / `PROD=lza-stag` | Discovered from the accelerator-provisioned VPC; FLIP creates none of it | Shared networking account's two-tier edge, over the Transit Gateway | + +Self-contained single-account is the supported open-source deployment shape and is not going away. The LZA mode +([FLIP#749](https://github.com/londonaicentre/FLIP/issues/749)) is the multi-account shape for estates running AWS's +[Landing Zone Accelerator](https://aws.amazon.com/solutions/implementations/landing-zone-accelerator-on-aws/), where +the network, guardrails and edge are owned by the accelerator pipeline rather than by FLIP. + +The two are one root module, not a fork: every LZA adaptation is gated behind `var.lza_managed_network` (set from +the LZA `PROD` values), so with `PROD=true`/`PROD=stag` the resolved configuration is identical to before — the +self-contained environments are never touched by LZA work. The AI Centre's own LZA targets are the +**FLIPProduction** workload account in `eu-west-2` (the `lza-prod` profile alias, running alongside legacy prod) +and a staging workload account reached as `lza-stag`. + +Environment (prod vs stag semantics) and network mode are **orthogonal axes**, and the two `PROD` values set them +independently: + +| | `PROD=lza` | `PROD=lza-stag` | +| --- | --- | --- | +| Env file / kit suffix | `.env.lza-prod` / `trust/.env..lza-prod` | `.env.lza-stag` / `trust/.env..lza-stag` | +| Profile guard | `lza-prod` (`LZA_AWS_PROFILE`) | `lza-stag` (`LZA_STAG_AWS_PROFILE`) | +| `TF_VAR_environment` | `prod` — RDS deletion protection + final snapshot on | `stag` — disposable, like legacy stag | +| `deploy-centralhub` git ref | `origin/main` | `origin/develop` | +| `LZA_VPC_NAME` | optional (defaults to the prod template name) | **required** in the env file — there is no defaultable staging VPC name, so the Makefile refuses to run without it rather than letting the lookup fail opaquely | + +Everything below reads naturally for either value; where it says `PROD=lza`, staging substitutes `lza-stag` and +its own account-scoped values (state bucket, `flip-lza-stag-*` bucket namespace, pull-through registry URL); the raw +`TF_VAR_` exports at the end of the env block below are estate-wide values shared by both (one ingress VPC, one +LogArchive bucket), except the two edge ones, which stay empty until a staging edge exists. Each +LZA environment is its **own workload account** — never co-tenant two environments in one account: the stack's +resource names (`flip-cluster`, `flip-api`, `flip-database-proxy`, the `/flip/networking/*` handoff params) are +fixed per account by design. + +**What `PROD=lza` selects:** + +| Concern | Value | +| --- | --- | +| Env file | root `.env.lza-prod` (gitignored, like the other env files) | +| Profile guard | `AWS_PROFILE=lza-prod` — a short local alias for the workload account's `FLIPAdminAccess` permission set, per the same convention as `prod`/`stag` (override via `LZA_AWS_PROFILE`) | +| `TF_VAR_environment` | `prod` — LZA is a production estate, so all prod-only hardening (RDS deletion protection, final snapshot) stays on | +| `TF_VAR_lza_managed_network` | `true` — the platform-managed-network toggle, orthogonal to `environment` (see below) | +| Trust kit suffix | `trust/.env..lza-prod` — a separate namespace so legacy prod kits are never overwritten | +| `deploy-centralhub` git ref | `origin/main` (same as legacy prod) | + +**Platform-managed vs FLIP-managed.** The LZA account's network is owned by the accelerator pipeline +([londonaicentre/lza](https://github.com/londonaicentre/lza)) and VPC-layer creation is SCP-denied in-account, so with +`TF_VAR_lza_managed_network=true` Terraform: + +- **skips creating**: the VPC module (VPC/subnets/NAT/IGW/EIPs), the in-account VPC endpoints (interface endpoints are + centralised in the Network account; S3+DynamoDB gateway endpoints are platform-provided), the DHCP options, and the + `/flip/networking/*` SSM params (legacy TGW coupling — the LZA TGW attachment is platform-managed); +- **discovers instead**: the `AWSAccelerator-eu-west-2-prod` VPC and its subnets by Name tag (`network_lza.tf`; + override the name via `LZA_VPC_NAME`). Subnet lookups match ALL `-app-*` / `-data-*` hits, so subnets the platform + team adds later — as the second AZ's were — appear on the next plan with no code change; +- **places by connectivity need**: RDS instances go to the isolated **data** subnets (local routes only — nothing + there can reach TGW/endpoints, and RDS doesn't need to); ECS tasks, the internal ALB, the RDS Proxy, EFS mount + targets and the EC2 hosts go to the TGW-routed **app** subnets (they need the central endpoints / image pulls); +- **gates off**: the SG-drift CloudTrail→EventBridge→Lambda stack (`security.tf` — the org baseline of Control Tower + org trail, GuardDuty, Security Hub and Config covers it), the public FL-server NLB + target group + DNS record + + SG rules (no IGW and VPC Block Public Access make an internet-facing NLB impossible), and the workload CloudFront + distribution + its VPC origin (the `GRCLOUDFRONTVPCORIGIN` SCP denies VPC origins by design — a VPC origin dials + the ALB inside the VPC, bypassing the TGW + central firewall). Ingress instead rides the networking account's + two-tier edge (proven end-to-end in FLIP#829/PR#830 and now serving the real stack): the edge CloudFront serves + the UI bucket via cross-account OAC and relays `/api/*` to the internal ALB, and the edge NLB forwards FL traffic + over TGW to the internal FL NLB in `fl_ingress_lza.tf` (static per-subnet IPs the edge registers once as targets), + which fronts `fl-server-net-1`. + +Everything else (ECS Fargate, RDS + Proxy, Cognito, S3 + CMK, Secrets Manager, SES, EFS, Cloud Map, internal ALB) +remains FLIP-managed exactly as on legacy prod; the legacy WAF/OAC/CloudFront-function components stay standing +unused on LZA to keep legacy churn minimal. + +**Edge wiring is two-phase — by construction, not configuration.** The networking account's edge stack +([aicentre-lza-iac](https://github.com/londonaicentre/aicentre-lza-iac)) is built *from* this stack's outputs: the +first workload `apply` publishes the `/flip/networking/*` SSM handoff params (FL NLB private IPs + port, ALB DNS +name, web port) that the edge NLB and relay consume, so the workload account necessarily applies before the edge +distribution exists. On that first apply `TF_VAR_lza_web_edge_domain` and `TF_VAR_lza_web_edge_distribution_arn` +are still empty: the UI-bucket policy then grants no principal (fail-closed — the edge simply cannot read the +bucket yet) and `local.ui_origin` is a placeholder. Once the edge stack is up, set both values in `.env.lza-prod` +(the edge distribution's default domain and its ARN) and re-apply to grant the cross-account OAC read and point +bucket CORS + Cognito URLs at the edge domain. This ordering is why the two variables deliberately carry no +"required-when-LZA" validation — it would hard-fail the legitimate first apply. + +**Prerequisites (provisioned out-of-band in each LZA account, not Terraform-managed here).** Every +`PROD=lza*` account needs these three before its first `plan`; the commands below are the ones the +FLIPStaging bring-up used (2026-09-01), with `PROD`/profile swapped per environment. + +- TF state bucket (`flip-terraform-state-lza`, or `-lza-stag`; versioned, SSE-KMS, public access blocked): + `make create-backend PROD=lza` — idempotent, reads the bucket name from the env file. +- ECR **pull-through cache rules** — the account has no internet egress, so images come from in-account mirrors over + the central `ecr.api`/`ecr.dkr` endpoints: prefix `ghcr/` mirroring `ghcr.io` (upstream auth via a read-only GHCR + PAT in the `ecr-pullthroughcache/ghcr` Secrets Manager secret) and the credential-less `ecr-public/` prefix + mirroring `public.ecr.aws` (used for the EFS-provision utility image). The execution role's + `ecr:BatchImportUpstreamImage`/`ecr:CreateRepository` grant for first-pull imports IS Terraform-managed + (`iam_ecs.tf`, LZA-gated). **Create the secret before the rule that references it, and both before the first + `plan`** (see the KMS-alias trap above): + + ```bash + # The GHCR PAT. Copy the existing read-only one from another LZA account (shown), + # or mint a fresh token per account for tighter isolation. The value must never + # be echoed — write it via a 0600 temp file. + TMP=$(mktemp) && chmod 600 "$TMP" + aws secretsmanager get-secret-value --profile \ + --secret-id ecr-pullthroughcache/ghcr --query SecretString --output text > "$TMP" + aws secretsmanager create-secret --profile \ + --name ecr-pullthroughcache/ghcr --secret-string "file://$TMP" --query ARN --output text + shred -u "$TMP" + + aws ecr create-pull-through-cache-rule --profile \ + --ecr-repository-prefix ghcr --upstream-registry-url ghcr.io --credential-arn + aws ecr create-pull-through-cache-rule --profile \ + --ecr-repository-prefix ecr-public --upstream-registry-url public.ecr.aws + ``` + + Nothing is pre-populated: the first pull of each image pays the upstream fetch. Cached **mutable** tags can + serve up to ~24h stale, which is why deploys pin the immutable `sha-` tags. +- An `lza-prod` profile in `~/.aws/config` for the account's Identity Center `FLIPAdminAccess` permission set (an + `aws configure sso` against the account, then rename the generated profile — same short-alias convention as + `prod`/`stag`; override the expected name via `LZA_AWS_PROFILE`). + +**`.env.lza-prod`.** Carries the same keys as `.env.production` (start from that shape); the values that MUST differ, +plus the LZA-only keys: + +```bash +# Terraform backend + account +FLIP_TFSTATE_BUCKET_NAME=flip-terraform-state-lza +AWS_REGION=eu-west-2 + +# Registry: the ghcr/ pull-through cache, NOT ghcr.io (no internet egress). +# Composes with the image names exactly like the GHCR prefix does: +# : → .../ghcr/londonaicentre/flip-api: +DOCKER_REGISTRY=.dkr.ecr.eu-west-2.amazonaws.com/ghcr/londonaicentre/ +# EFS-provision one-shot utility image via the credential-less ecr-public/ cache +EFS_PROVISION_IMAGE=.dkr.ecr.eu-west-2.amazonaws.com/ecr-public/aws-cli/aws-cli:2.22.35 + +# Bucket names are globally unique and the legacy flipprod-* names stay taken +# while the old account lives — the LZA env uses its own flip-lza-* namespace. +FLIP_MODEL_FILES_UPLOADS_BUCKET_NAME=flip-lza-model-files-uploads +FLIP_FL_RESULTS_BUCKET_NAME=flip-lza-fl-results +FLIP_APP_BUNDLES_BUCKET_NAME=flip-lza-app-bundles +AICENTRE_BUCKET_NAME=flip-lza-aicentre +FLIP_UI_BUCKET_NAME=flip-lza-ui +# The two log buckets default to subdomain-derived names +# (flip-access-logs-/flip-cf-logs-) — but ALB_SUBDOMAIN keeps its +# post-cutover value here, so those derived names are still owned by legacy +# prod. Bucket names are global: override them into the flip-lza-* namespace. +ACCESS_LOGS_BUCKET_NAME=flip-lza-access-logs +CF_LOGS_BUCKET_NAME=flip-lza-cf-logs + +# No Route53 hosted zone in the account yet (its move is a platform-side DNS +# line item) — first bring-up runs on the default CloudFront domain. Flip to +# true (and re-apply) once the zone lands. ALB_SUBDOMAIN/NLB_SUBDOMAIN keep +# their eventual post-cutover values meanwhile (used for resource naming). +MANAGE_DNS=false +ALB_SUBDOMAIN=app.flip.aicentre.co.uk +NLB_SUBDOMAIN=fl.app.flip.aicentre.co.uk + +# LZA-only Terraform inputs with no Makefile mapping of their own — exported +# straight from the env file (the Makefile includes it as make syntax, so an +# `export TF_VAR_…=` line reaches Terraform unchanged). Values come from the +# networking account. +# ingress-VPC subnet CIDRs (edge NLB + CloudFront relay path over the TGW) +export TF_VAR_networking_ingress_cidrs=[""] +# the LogArchive account's central ELB access-logs bucket (accelerator guardrail) +export TF_VAR_lza_elb_access_logs_bucket= +# the edge distribution — EMPTY on the first apply (the edge is built from this +# stack's outputs, see "Edge wiring is two-phase"), then filled in + re-applied +export TF_VAR_lza_web_edge_domain= +export TF_VAR_lza_web_edge_distribution_arn= + +# Optional: only needed if the platform VPC template is renamed. +# LZA_VPC_NAME=AWSAccelerator-eu-west-2-prod +``` + +Secrets (`AES_KEY_BASE64`, `INTERNAL_SERVICE_KEY*`, `ADMIN_USER_PASSWORD`, …) are minted fresh for the account during +the WP3 bring-up and later replaced by the carried-over legacy **values** in the WP4 data migration (so existing trust +kits and encrypted data stay valid) — never reuse the legacy Secrets Manager secret itself. + +**First bring-up without DNS (`MANAGE_DNS=false`).** The zone lookup would hard-fail in a zone-less account, so +`MANAGE_DNS=false` skips it plus every Route53 record and both DNS-validated ACM certs. Consequences, all of which +revert by flipping `MANAGE_DNS=true` + `make plan`/`apply` once the zone lands: + +- The user-facing URL is the networking account's edge distribution on its default `*.cloudfront.net` domain + (`TF_VAR_lza_web_edge_domain`) — the workload distribution is gated off on LZA, so `terraform output + CloudfrontDistributionDomain` is null there by design. +- The edge→ALB `/api/*` relay leg terminates on a **plain-HTTP** ALB listener (an ALB HTTPS listener needs an + ISSUED cert, and issuance needs DNS validation). Viewer traffic stays HTTPS at the edge, and the relay leg never + leaves the TGW + central-firewall path. Accepted as a bring-up-only limitation. +- Bucket CORS and the Cognito sign-in hostname/callback URLs follow the CloudFront default domain automatically + (`local.ui_origin`), so uploads, downloads and sign-in work; `make deploy-ui` must generate `window.js` with + `CENTRAL_HUB_API_URL` pointing at the CloudFront domain. +- Trusts polling the hub would need the CloudFront domain as `CENTRAL_HUB_API_URL` — fine for WP3 smoke trusts; + the real cutover is DNS-only and happens after the zone migrates. + +**Fresh-account trap: create a Secrets Manager secret before the first `plan`.** On a +brand-new workload account `make plan` fails at the very end with + +``` +Error: reading KMS Alias (alias/aws/secretsmanager): empty result + with data.aws_kms_alias.secretsmanager, on rds_proxy.tf line 37 +``` + +even though the plan itself computed cleanly. `alias/aws/secretsmanager` is an +**AWS-managed** key: AWS creates it lazily, the first time the account uses Secrets +Manager, so in an account that has never held a secret the alias genuinely does not +exist yet and the data source has nothing to read. Nothing is wrong with the +configuration — the account is simply too new. Creating *any* secret mints the key, +and the ECR pull-through credential below is the one every LZA account needs anyway, +so do that step first and the plan completes. (Verified on the FLIPStaging bring-up, +2026-09-01.) + +**Attaching the real domain to the edge.** CloudFront is never addressed by IP — distribution IPs are anycast +and rotate — so the domain is wired up DNS-side, in the networking account where the edge lives: a public +Route 53 hosted zone for the FLIP subdomain (e.g. `flip.example.org`) is created there; a +DNS-validated ACM certificate for the canonical names is minted **in us-east-1** (a CloudFront requirement) +and attached to the distribution as its alternate domain names; the zone then carries an **A/AAAA alias +record** — e.g. `app.flip.example.org` → `d111111abcdef8.cloudfront.net` (an alias, never a literal +IP) — and the parent domain's DNS host delegates the subdomain to the zone's four `ns-*.awsdns-*` name +servers. The FL name (`fl.app.flip.example.org`) is the same shape aliasing the edge NLB. The +concrete cutover runbook stays in the private platform repos. + +**State of the LZA path** (tracked on #749): + +- **Multi-AZ has landed platform-side** (verified against the live account 2026-08-24): both AZs carry an + `-app-*`, a `-data-*` and a `-tgw-*` subnet, which clears the earlier single-AZ constraints — the live + `flip-db-subnet-group` spans both AZs, and the TGW attachment now rides `tgw-a` **and** `tgw-b`, so traffic + originating in the `-b` subnets no longer blackholes (the condition reported on PR#830). No FLIP-side change + was needed: the subnet lookups glob every matching Name tag, so the second AZ arrived on an ordinary plan. +- **FL ingress is wired.** `fl_ingress_lza.tf` creates the internal NLB (static per-subnet IPs, so the edge can + register them once as targets), its target group fronting `fl-server-net-1`, and the `/flip/networking/*` SSM + handoff the networking account consumes. The full chain — internet → edge NLB → TGW → central firewall → + internal NLB → ECS — was proven with dummy services first (FLIP#829 / PR#830). +- **Still untested against `PROD=lza`:** the `full-deploy*` chains, `make status`/`check_status.py`, `update_env.py` + and `make destroy`. WP3 exercises the `init`/`plan`/`apply` (+ `deploy-centralhub`/`deploy-ui`) loop first and + fixes up the auxiliary tooling as findings come in. + ### Terraform module layout ``` diff --git a/deploy/providers/AWS/certificate.tf b/deploy/providers/AWS/certificate.tf index 53b3ac419..7f25b8289 100644 --- a/deploy/providers/AWS/certificate.tf +++ b/deploy/providers/AWS/certificate.tf @@ -26,9 +26,16 @@ # This is because the domain_validation_options are only known after # the certificate is created, so Terraform cannot plan the DNS records # in a single pass. +# +# The whole chain is skipped when var.manage_dns is false (the zone-less +# first LZA bring-up, FLIP#749): DNS validation is impossible without the +# hosted zone, and an unvalidated cert cannot be attached to the ALB — which +# serves plain HTTP on the private CloudFront-VPC-origin leg instead (see the +# listeners comment in main.tf). ############################ resource "aws_acm_certificate" "flip" { + count = var.manage_dns ? 1 : 0 domain_name = var.flip_alb_subdomain validation_method = "DNS" @@ -44,7 +51,7 @@ resource "aws_acm_certificate" "flip" { # DNS validation record resource "aws_route53_record" "cert_validation" { for_each = { - for dvo in tolist(aws_acm_certificate.flip.domain_validation_options) : dvo.domain_name => { + for dvo in var.manage_dns ? tolist(aws_acm_certificate.flip[0].domain_validation_options) : [] : dvo.domain_name => { name = dvo.resource_record_name record = dvo.resource_record_value type = dvo.resource_record_type @@ -56,16 +63,30 @@ resource "aws_route53_record" "cert_validation" { records = [each.value.record] ttl = 60 type = each.value.type - zone_id = data.aws_route53_zone.subdomain.zone_id + zone_id = data.aws_route53_zone.subdomain[0].zone_id } # Certificate validation resource "aws_acm_certificate_validation" "flip" { - certificate_arn = aws_acm_certificate.flip.arn + count = var.manage_dns ? 1 : 0 + certificate_arn = aws_acm_certificate.flip[0].arn validation_record_fqdns = [for record in aws_route53_record.cert_validation : record.fqdn] } output "CertificateArn" { - description = "ACM Certificate ARN (validated)" - value = aws_acm_certificate_validation.flip.certificate_arn + description = "ACM Certificate ARN (validated; null when manage_dns is false — FLIP#749)" + value = var.manage_dns ? aws_acm_certificate_validation.flip[0].certificate_arn : null +} + +# State migration for the counts added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_acm_certificate.flip + to = aws_acm_certificate.flip[0] +} + +moved { + from = aws_acm_certificate_validation.flip + to = aws_acm_certificate_validation.flip[0] } diff --git a/deploy/providers/AWS/cloudfront.tf b/deploy/providers/AWS/cloudfront.tf index 54f746019..d4de13ea3 100644 --- a/deploy/providers/AWS/cloudfront.tf +++ b/deploy/providers/AWS/cloudfront.tf @@ -45,8 +45,14 @@ provider "aws" { ############################ # CloudFront viewer cert (us-east-1) ############################ +# +# Skipped when var.manage_dns is false (the zone-less first LZA bring-up, +# FLIP#749): the viewer cert can't be DNS-validated without the zone, and a +# custom viewer cert is pointless anyway while the distribution has no aliases +# — it serves the default *.cloudfront.net domain with the default cert. resource "aws_acm_certificate" "flip_cloudfront" { + count = var.manage_dns ? 1 : 0 provider = aws.us_east_1 domain_name = var.flip_alb_subdomain validation_method = "DNS" @@ -62,7 +68,7 @@ resource "aws_acm_certificate" "flip_cloudfront" { resource "aws_route53_record" "cloudfront_cert_validation" { for_each = { - for dvo in tolist(aws_acm_certificate.flip_cloudfront.domain_validation_options) : dvo.domain_name => { + for dvo in var.manage_dns ? tolist(aws_acm_certificate.flip_cloudfront[0].domain_validation_options) : [] : dvo.domain_name => { name = dvo.resource_record_name record = dvo.resource_record_value type = dvo.resource_record_type @@ -74,15 +80,29 @@ resource "aws_route53_record" "cloudfront_cert_validation" { records = [each.value.record] ttl = 60 type = each.value.type - zone_id = data.aws_route53_zone.subdomain.zone_id + zone_id = data.aws_route53_zone.subdomain[0].zone_id } resource "aws_acm_certificate_validation" "flip_cloudfront" { + count = var.manage_dns ? 1 : 0 provider = aws.us_east_1 - certificate_arn = aws_acm_certificate.flip_cloudfront.arn + certificate_arn = aws_acm_certificate.flip_cloudfront[0].arn validation_record_fqdns = [for record in aws_route53_record.cloudfront_cert_validation : record.fqdn] } +# State migration for the counts added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_acm_certificate.flip_cloudfront + to = aws_acm_certificate.flip_cloudfront[0] +} + +moved { + from = aws_acm_certificate_validation.flip_cloudfront + to = aws_acm_certificate_validation.flip_cloudfront[0] +} + ############################ # CloudFront VPC origin for the (internal) ALB # @@ -94,14 +114,27 @@ resource "aws_acm_certificate_validation" "flip_cloudfront" { ############################ resource "aws_cloudfront_vpc_origin" "flip_api" { + # Gated off on LZA (FLIP#749): the GRCLOUDFRONTVPCORIGIN SCP denies + # cloudfront:CreateVpcOrigin in workload accounts -- deliberately, since a + # VPC origin reaches the ALB inside the VPC and bypasses the TGW + central + # firewall. On LZA the networking account's edge distribution is the front + # door (aicentre-lza-iac); fl_ingress_lza.tf admits its relay path onto the + # ALB instead of the SG rule below. + count = var.lza_managed_network ? 0 : 1 + vpc_origin_endpoint_config { # CloudFront VPC origin names accept only alphanumerics, dashes, and # underscores — the subdomain contains dots, so replace them with dashes. - name = "flip-api-vpc-origin-${replace(var.flip_alb_subdomain, ".", "-")}" - arn = module.alb.arn - http_port = 80 + name = "flip-api-vpc-origin-${replace(var.flip_alb_subdomain, ".", "-")}" + arn = module.alb.arn + # Without a hosted zone the ALB cannot carry an ISSUED cert, so the private + # VPC-origin leg falls back to plain HTTP until DNS lands (FLIP#749; see the + # ALB listeners comment in main.tf). The ALB's main listener then serves + # plain HTTP on ALB_HTTPS_PORT, so the HTTP port follows it there. Viewer + # traffic stays HTTPS either way. + http_port = var.manage_dns ? 80 : var.ALB_HTTPS_PORT https_port = 443 - origin_protocol_policy = "https-only" + origin_protocol_policy = var.manage_dns ? "https-only" : "http-only" origin_ssl_protocols { items = ["TLSv1.2"] @@ -123,8 +156,9 @@ resource "aws_cloudfront_vpc_origin" "flip_api" { # checks against the service-managed SG (or the CloudFront managed prefix # list), not the ENI source IP. data "aws_security_group" "cloudfront_vpcorigins_service" { + count = var.lza_managed_network ? 0 : 1 name = "CloudFront-VPCOrigins-Service-SG" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id depends_on = [aws_cloudfront_vpc_origin.flip_api] } @@ -135,13 +169,32 @@ data "aws_security_group" "cloudfront_vpcorigins_service" { # lookup needs the VPC origin. Attaching the rule outside the module keeps the # chain linear. resource "aws_security_group_rule" "alb_ingress_https_from_cloudfront" { + count = var.lza_managed_network ? 0 : 1 description = "HTTPS from the CloudFront-VPCOrigins-Service-SG (Option 2 in AWS VPC origins docs)" type = "ingress" from_port = var.ALB_HTTPS_PORT to_port = var.ALB_HTTPS_PORT protocol = "tcp" security_group_id = module.alb_security_group.security_group.id - source_security_group_id = data.aws_security_group.cloudfront_vpcorigins_service.id + source_security_group_id = data.aws_security_group.cloudfront_vpcorigins_service[0].id +} + +# State migration for the counts added above (FLIP#749 WP3): keeps existing +# legacy states aligned without a manual `terraform state mv`. Safe to remove +# once every live state file has been migrated. +moved { + from = aws_cloudfront_vpc_origin.flip_api + to = aws_cloudfront_vpc_origin.flip_api[0] +} + +moved { + from = aws_security_group_rule.alb_ingress_https_from_cloudfront + to = aws_security_group_rule.alb_ingress_https_from_cloudfront[0] +} + +moved { + from = aws_cloudfront_distribution.flip_ui + to = aws_cloudfront_distribution.flip_ui[0] } ############################ @@ -154,7 +207,10 @@ resource "aws_security_group_rule" "alb_ingress_https_from_cloudfront" { ############################ resource "aws_s3_bucket" "cloudfront_logs" { - bucket = "flip-cf-logs-${var.flip_alb_subdomain}" + count = var.lza_managed_network ? 0 : 1 + # Derived-name-with-override, same rationale as access_logs_bucket_name in + # s3_logging.tf (global bucket names vs the shared subdomain, FLIP#749). + bucket = var.CF_LOGS_BUCKET_NAME != "" ? var.CF_LOGS_BUCKET_NAME : "flip-cf-logs-${var.flip_alb_subdomain}" tags = { Name = "flip-cloudfront-logs" @@ -162,15 +218,17 @@ resource "aws_s3_bucket" "cloudfront_logs" { } resource "aws_s3_bucket_ownership_controls" "cloudfront_logs" { - bucket = aws_s3_bucket.cloudfront_logs.id + count = var.lza_managed_network ? 0 : 1 + bucket = aws_s3_bucket.cloudfront_logs[0].id rule { object_ownership = "BucketOwnerPreferred" } } resource "aws_s3_bucket_acl" "cloudfront_logs" { - depends_on = [aws_s3_bucket_ownership_controls.cloudfront_logs] - bucket = aws_s3_bucket.cloudfront_logs.id + count = var.lza_managed_network ? 0 : 1 + depends_on = [aws_s3_bucket_ownership_controls.cloudfront_logs[0]] + bucket = aws_s3_bucket.cloudfront_logs[0].id access_control_policy { owner { @@ -201,7 +259,8 @@ resource "aws_s3_bucket_acl" "cloudfront_logs" { } resource "aws_s3_bucket_lifecycle_configuration" "cloudfront_logs" { - bucket = aws_s3_bucket.cloudfront_logs.id + count = var.lza_managed_network ? 0 : 1 + bucket = aws_s3_bucket.cloudfront_logs[0].id rule { id = "expire-cf-logs-after-30-days" @@ -216,7 +275,8 @@ resource "aws_s3_bucket_lifecycle_configuration" "cloudfront_logs" { } resource "aws_s3_bucket_server_side_encryption_configuration" "cloudfront_logs" { - bucket = aws_s3_bucket.cloudfront_logs.id + count = var.lza_managed_network ? 0 : 1 + bucket = aws_s3_bucket.cloudfront_logs[0].id rule { apply_server_side_encryption_by_default { @@ -230,7 +290,8 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "cloudfront_logs" # Neither is "public" under PAB semantics, so blocking public ACLs and # policies is safe regardless of which delivery mechanism is in use. resource "aws_s3_bucket_public_access_block" "cloudfront_logs" { - bucket = aws_s3_bucket.cloudfront_logs.id + count = var.lza_managed_network ? 0 : 1 + bucket = aws_s3_bucket.cloudfront_logs[0].id block_public_acls = true block_public_policy = true ignore_public_acls = true @@ -240,7 +301,8 @@ resource "aws_s3_bucket_public_access_block" "cloudfront_logs" { # Enforce HTTPS-only access to the CloudFront logs bucket. # CloudFront log delivery uses HTTPS only, so this is safe. resource "aws_s3_bucket_policy" "cloudfront_logs_https_only" { - bucket = aws_s3_bucket.cloudfront_logs.id + count = var.lza_managed_network ? 0 : 1 + bucket = aws_s3_bucket.cloudfront_logs[0].id policy = jsonencode({ Version = "2012-10-17" @@ -250,8 +312,8 @@ resource "aws_s3_bucket_policy" "cloudfront_logs_https_only" { Principal = "*" Action = "s3:*" Resource = [ - aws_s3_bucket.cloudfront_logs.arn, - "${aws_s3_bucket.cloudfront_logs.arn}/*", + aws_s3_bucket.cloudfront_logs[0].arn, + "${aws_s3_bucket.cloudfront_logs[0].arn}/*", ] Condition = { Bool = { @@ -371,7 +433,7 @@ resource "aws_s3_bucket_policy" "demo_assets" { Resource = "${data.aws_s3_bucket.demo_assets[0].arn}/ark_demo/assets/*" Condition = { StringEquals = { - "AWS:SourceArn" = aws_cloudfront_distribution.flip_ui.arn + "AWS:SourceArn" = var.lza_managed_network ? var.lza_web_edge_distribution_arn : aws_cloudfront_distribution.flip_ui[0].arn } } }] @@ -434,6 +496,16 @@ resource "aws_cloudfront_function" "spa_rewrite" { locals { cloudfront_policy_caching_optimized = "658327ea-f89d-4fab-a63d-7e88639e58f6" cloudfront_policy_caching_disabled = "4135ea2d-6df8-44a3-9df3-4b5a84be39ad" + + # Browser-facing origin of the UI, consumed by the S3 bucket CORS rules and + # the Cognito URLs in services.tf. With DNS managed it is the canonical + # subdomain (legacy shape, unchanged); on the zone-less first bring-up + # (FLIP#749) it is the CloudFront default domain, so uploads/downloads and + # sign-in keep working before any DNS exists. No dependency cycle: the + # distribution references neither the app buckets' CORS nor Cognito. + # On LZA the workload distribution is gated off -- the UI origin is the + # networking account's edge distribution (FLIP#749 WP3). + ui_origin = var.manage_dns ? "https://${var.flip_alb_subdomain}" : var.lza_managed_network ? "https://${var.lza_web_edge_domain}" : "https://${aws_cloudfront_distribution.flip_ui[0].domain_name}" } # Custom origin-request policy for /api/*. The managed AllViewer policy @@ -496,6 +568,7 @@ resource "aws_cloudfront_origin_request_policy" "flip_api" { # with no false positives, flip its `override_action` (for managed groups) # or `action` (for the custom rate-limit) to `block` / `none` + `block`. resource "aws_wafv2_web_acl" "flip_ui_cloudfront" { + count = var.lza_managed_network ? 0 : 1 # checkov:skip=CKV_AWS_192:AWSManagedRulesKnownBadInputsRuleSet (the Log4j AMR) is attached; count-mode rollout is deliberate — flip to block after sampled-traffic review provider = aws.us_east_1 name = "flip-ui-${replace(var.flip_alb_subdomain, "/[^a-zA-Z0-9]/", "-")}" @@ -609,15 +682,17 @@ resource "aws_wafv2_web_acl" "flip_ui_cloudfront" { # WAF logging destination. Name MUST start with `aws-waf-logs-` per AWS — # otherwise PutLoggingConfiguration rejects it. resource "aws_cloudwatch_log_group" "flip_ui_waf" { + count = var.lza_managed_network ? 0 : 1 provider = aws.us_east_1 name = "aws-waf-logs-flip-ui-${replace(var.flip_alb_subdomain, "/[^a-zA-Z0-9]/", "-")}" retention_in_days = local.log_retention_days } resource "aws_wafv2_web_acl_logging_configuration" "flip_ui_cloudfront" { + count = var.lza_managed_network ? 0 : 1 provider = aws.us_east_1 - resource_arn = aws_wafv2_web_acl.flip_ui_cloudfront.arn - log_destination_configs = [aws_cloudwatch_log_group.flip_ui_waf.arn] + resource_arn = aws_wafv2_web_acl.flip_ui_cloudfront[0].arn + log_destination_configs = [aws_cloudwatch_log_group.flip_ui_waf[0].arn] } ############################ @@ -878,14 +953,25 @@ resource "aws_cloudfront_response_headers_policy" "flip_api" { resource "aws_cloudfront_distribution" "flip_ui" { # checkov:skip=CKV2_AWS_47:the attached ACL carries AWSManagedRulesKnownBadInputsRuleSet (Log4j AMR); count-mode rollout is deliberate — see the web ACL above + # Gated off on LZA with the VPC origin above: the networking account's edge + # distribution serves the UI (cross-account OAC on aws_s3_bucket.flip_ui) + # and relays /api/* -- see aicentre-lza-iac. The WAF/OAC/function/response + # policies below stay standing unused on LZA to keep this diff and the + # legacy state churn minimal; the edge carries its own WAF. + count = var.lza_managed_network ? 0 : 1 + enabled = true is_ipv6_enabled = true http_version = "http2" default_root_object = "index.html" price_class = "PriceClass_100" - aliases = [var.flip_alb_subdomain] - comment = "flip-ui at ${var.flip_alb_subdomain}" - web_acl_id = aws_wafv2_web_acl.flip_ui_cloudfront.arn + # No aliases without DNS (FLIP#749): CloudFront only allows the default + # viewer cert when no aliases are set, and an alias without a record pointing + # at it is unreachable anyway. The distribution serves *.cloudfront.net until + # MANAGE_DNS flips to true. + aliases = var.manage_dns ? [var.flip_alb_subdomain] : [] + comment = "flip-ui at ${var.flip_alb_subdomain}" + web_acl_id = aws_wafv2_web_acl.flip_ui_cloudfront[0].arn origin { domain_name = aws_s3_bucket.flip_ui.bucket_regional_domain_name @@ -903,7 +989,7 @@ resource "aws_cloudfront_distribution" "flip_ui" { origin_id = "alb-api-origin" vpc_origin_config { - vpc_origin_id = aws_cloudfront_vpc_origin.flip_api.id + vpc_origin_id = aws_cloudfront_vpc_origin.flip_api[0].id } } @@ -1002,15 +1088,18 @@ resource "aws_cloudfront_distribution" "flip_ui" { } logging_config { - bucket = aws_s3_bucket.cloudfront_logs.bucket_domain_name + bucket = aws_s3_bucket.cloudfront_logs[0].bucket_domain_name include_cookies = false prefix = "standard-logs/" } viewer_certificate { - acm_certificate_arn = aws_acm_certificate_validation.flip_cloudfront.certificate_arn - ssl_support_method = "sni-only" - minimum_protocol_version = "TLSv1.2_2021" + acm_certificate_arn = var.manage_dns ? aws_acm_certificate_validation.flip_cloudfront[0].certificate_arn : null + cloudfront_default_certificate = var.manage_dns ? null : true + ssl_support_method = var.manage_dns ? "sni-only" : null + # AWS forces TLSv1 while the default *.cloudfront.net certificate is in + # use; pinning TLSv1.2_2021 there would just plan perpetual drift. + minimum_protocol_version = var.manage_dns ? "TLSv1.2_2021" : "TLSv1" } tags = { @@ -1032,7 +1121,7 @@ resource "aws_s3_bucket_policy" "flip_ui" { Resource = "${aws_s3_bucket.flip_ui.arn}/*" Condition = { StringEquals = { - "AWS:SourceArn" = aws_cloudfront_distribution.flip_ui.arn + "AWS:SourceArn" = var.lza_managed_network ? var.lza_web_edge_distribution_arn : aws_cloudfront_distribution.flip_ui[0].arn } } }, @@ -1073,15 +1162,72 @@ resource "aws_s3_bucket_logging" "flip_ui" { output "CloudfrontDistributionId" { description = "CloudFront distribution ID for flip-ui (used by make deploy-ui for cache invalidation)" - value = aws_cloudfront_distribution.flip_ui.id + value = one(aws_cloudfront_distribution.flip_ui[*].id) } output "CloudfrontDistributionDomain" { description = "CloudFront distribution CloudFront-assigned domain (*.cloudfront.net). Use for pre-cutover smoke tests." - value = aws_cloudfront_distribution.flip_ui.domain_name + value = one(aws_cloudfront_distribution.flip_ui[*].domain_name) } output "FlipUiBucketName" { description = "S3 bucket holding the UI static assets" value = aws_s3_bucket.flip_ui.bucket } + +# State migration for the counts added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Adding `count` renames +# each resource from X to X[0], which Terraform would otherwise plan as +# destroy-and-recreate — harmless on LZA where these are orphans, destructive on +# legacy where the WAF fronts live production traffic and the log bucket holds +# real objects under a 30-day lifecycle. Safe to remove once every live state +# file has been migrated. +moved { + from = aws_wafv2_web_acl.flip_ui_cloudfront + to = aws_wafv2_web_acl.flip_ui_cloudfront[0] +} + +moved { + from = aws_wafv2_web_acl_logging_configuration.flip_ui_cloudfront + to = aws_wafv2_web_acl_logging_configuration.flip_ui_cloudfront[0] +} + +moved { + from = aws_cloudwatch_log_group.flip_ui_waf + to = aws_cloudwatch_log_group.flip_ui_waf[0] +} + +moved { + from = aws_s3_bucket.cloudfront_logs + to = aws_s3_bucket.cloudfront_logs[0] +} + +moved { + from = aws_s3_bucket_ownership_controls.cloudfront_logs + to = aws_s3_bucket_ownership_controls.cloudfront_logs[0] +} + +moved { + from = aws_s3_bucket_acl.cloudfront_logs + to = aws_s3_bucket_acl.cloudfront_logs[0] +} + +moved { + from = aws_s3_bucket_lifecycle_configuration.cloudfront_logs + to = aws_s3_bucket_lifecycle_configuration.cloudfront_logs[0] +} + +moved { + from = aws_s3_bucket_server_side_encryption_configuration.cloudfront_logs + to = aws_s3_bucket_server_side_encryption_configuration.cloudfront_logs[0] +} + +moved { + from = aws_s3_bucket_public_access_block.cloudfront_logs + to = aws_s3_bucket_public_access_block.cloudfront_logs[0] +} + +moved { + from = aws_s3_bucket_policy.cloudfront_logs_https_only + to = aws_s3_bucket_policy.cloudfront_logs_https_only[0] +} diff --git a/deploy/providers/AWS/dhcp.tf b/deploy/providers/AWS/dhcp.tf index 93db20f44..62b09aafd 100644 --- a/deploy/providers/AWS/dhcp.tf +++ b/deploy/providers/AWS/dhcp.tf @@ -19,8 +19,15 @@ # # AmazonProvidedDNS is the default. Using it explicitly here prevents # Terraform from replacing it with nothing on re-apply. +# +# Gated off on the LZA platform-managed network (FLIP#749): the VPC's +# attributes are owned by the accelerator pipeline, so we don't re-associate +# its DHCP options. Only the bare-hostname search-domain convenience is lost — +# every in-repo consumer already resolves the Cloud Map FQDNs (see +# locals.service_discovery_names). resource "aws_vpc_dhcp_options" "flip" { + count = var.lza_managed_network ? 0 : 1 domain_name = local.flip_local_domain domain_name_servers = ["AmazonProvidedDNS"] @@ -30,6 +37,20 @@ resource "aws_vpc_dhcp_options" "flip" { } resource "aws_vpc_dhcp_options_association" "flip" { - vpc_id = module.flip_vpc.vpc_id - dhcp_options_id = aws_vpc_dhcp_options.flip.id + count = var.lza_managed_network ? 0 : 1 + vpc_id = local.vpc_id + dhcp_options_id = aws_vpc_dhcp_options.flip[0].id +} + +# State migration for the counts added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_vpc_dhcp_options.flip + to = aws_vpc_dhcp_options.flip[0] +} + +moved { + from = aws_vpc_dhcp_options_association.flip + to = aws_vpc_dhcp_options_association.flip[0] } diff --git a/deploy/providers/AWS/ecs_efs_provision.tf b/deploy/providers/AWS/ecs_efs_provision.tf index 42d1669c7..f186eebaa 100644 --- a/deploy/providers/AWS/ecs_efs_provision.tf +++ b/deploy/providers/AWS/ecs_efs_provision.tf @@ -59,7 +59,7 @@ resource "null_resource" "provision_efs_certs" { --cluster ${aws_ecs_cluster.flip.name} \ --task-definition ${aws_ecs_task_definition.efs_provision[0].arn} \ --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={subnets=[${join(",", module.flip_vpc.private_subnets)}],securityGroups=[${aws_security_group.ecs_fl_server.id}],assignPublicIp=DISABLED}" \ + --network-configuration "awsvpcConfiguration={subnets=[${join(",", local.app_subnet_ids)}],securityGroups=[${aws_security_group.ecs_fl_server.id}],assignPublicIp=DISABLED}" \ --count 1 \ --region ${var.AWS_REGION} \ --no-cli-pager \ @@ -103,8 +103,11 @@ resource "aws_ecs_task_definition" "efs_provision" { container_definitions = jsonencode([ { - name = "provision-efs-certs" - image = "amazon/aws-cli:2.22.35" + name = "provision-efs-certs" + # Default amazon/aws-cli:2.22.35 from Docker Hub; the egress-less LZA + # account overrides this to its ECR Public pull-through cache mirror via + # EFS_PROVISION_IMAGE in the env file (FLIP#749). + image = var.efs_provision_image # The amazon/aws-cli image's ENTRYPOINT is `aws`, so a command like # ["/bin/sh", "-c", ...] would get appended as args to aws and fail # with "Found invalid choice '/bin/sh'". Override entryPoint so the diff --git a/deploy/providers/AWS/ecs_flower.tf b/deploy/providers/AWS/ecs_flower.tf index 69ff22a3c..07697ecd7 100644 --- a/deploy/providers/AWS/ecs_flower.tf +++ b/deploy/providers/AWS/ecs_flower.tf @@ -118,7 +118,7 @@ resource "null_resource" "flower_register_supernode_keys" { --cluster ${aws_ecs_cluster.flip.name} \ --task-definition ${aws_ecs_task_definition.flower_register_supernode_keys[0].arn} \ --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={subnets=[${join(",", module.flip_vpc.private_subnets)}],securityGroups=[${aws_security_group.ecs_fl_api.id}],assignPublicIp=DISABLED}" \ + --network-configuration "awsvpcConfiguration={subnets=[${join(",", local.app_subnet_ids)}],securityGroups=[${aws_security_group.ecs_fl_api.id}],assignPublicIp=DISABLED}" \ --count 1 \ --region ${var.AWS_REGION} \ --no-cli-pager \ diff --git a/deploy/providers/AWS/ecs_services.tf b/deploy/providers/AWS/ecs_services.tf index 15e7e3746..c5fad17db 100644 --- a/deploy/providers/AWS/ecs_services.tf +++ b/deploy/providers/AWS/ecs_services.tf @@ -76,7 +76,7 @@ resource "aws_ecs_service" "flip_api" { enable_execute_command = var.ecs_exec_enabled network_configuration { - subnets = module.flip_vpc.private_subnets + subnets = local.app_subnet_ids security_groups = [aws_security_group.ecs_flip_api.id] assign_public_ip = false } @@ -125,7 +125,7 @@ resource "aws_ecs_service" "fl_api_net_1" { enable_execute_command = var.ecs_exec_enabled network_configuration { - subnets = module.flip_vpc.private_subnets + subnets = local.app_subnet_ids security_groups = [aws_security_group.ecs_fl_api.id] assign_public_ip = false } @@ -163,7 +163,7 @@ resource "aws_ecs_service" "fl_server_net_1" { enable_execute_command = var.ecs_exec_enabled network_configuration { - subnets = module.flip_vpc.private_subnets + subnets = local.app_subnet_ids security_groups = [aws_security_group.ecs_fl_server.id] assign_public_ip = false } @@ -173,10 +173,13 @@ resource "aws_ecs_service" "fl_server_net_1" { } # Register the running task's ENI IP with the NLB target group so FL - # clients reaching fl..flip.aicentre.co.uk:8002 hit the Fargate task - # over gRPC. NLB on TCP forwards the gRPC stream untouched. + # clients hit the Fargate task over gRPC. NLB on TCP forwards the gRPC + # stream untouched. Legacy: the internet-facing module.fl_server_nlb's TG, + # reached at fl..flip.aicentre.co.uk:8002. LZA (FLIP#749): the + # internal NLB's TG (fl_ingress_lza.tf), reached via the networking + # account's edge NLB over the TGW. load_balancer { - target_group_arn = aws_lb_target_group.ecs_fl_server_tcp.arn + target_group_arn = var.lza_managed_network ? aws_lb_target_group.ecs_fl_server_tcp_lza[0].arn : aws_lb_target_group.ecs_fl_server_tcp[0].arn container_name = "fl-server-net-1" # Backend-dependent container port (Flower: SuperLink Fleet 9092); the # NLB listener port trusts connect to stays var.FL_SERVER_PORT. diff --git a/deploy/providers/AWS/ecs_sg.tf b/deploy/providers/AWS/ecs_sg.tf index 8b855f6bd..d49e7ab7d 100644 --- a/deploy/providers/AWS/ecs_sg.tf +++ b/deploy/providers/AWS/ecs_sg.tf @@ -24,7 +24,7 @@ resource "aws_security_group" "ecs_flip_api" { name = "ecs-flip-api" description = "ECS flip-api task - inbound HTTP from ALB" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id tags = { FlipSG = "true" @@ -48,7 +48,7 @@ resource "aws_security_group_rule" "ecs_flip_api_ingress_vpc_http" { to_port = local.api_container_port protocol = "tcp" security_group_id = aws_security_group.ecs_flip_api.id - cidr_blocks = [var.vpc_cidr] + cidr_blocks = [local.vpc_cidr_block] } resource "aws_security_group_rule" "ecs_flip_api_egress_all" { @@ -68,7 +68,7 @@ resource "aws_security_group_rule" "ecs_flip_api_egress_all" { resource "aws_security_group" "ecs_fl_api" { name = "ecs-fl-api" description = "ECS fl-api-net-1 task - inbound HTTP from VPC" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id tags = { FlipSG = "true" @@ -82,7 +82,7 @@ resource "aws_security_group_rule" "ecs_fl_api_ingress_vpc_http" { to_port = local.api_container_port protocol = "tcp" security_group_id = aws_security_group.ecs_fl_api.id - cidr_blocks = [var.vpc_cidr] + cidr_blocks = [local.vpc_cidr_block] } resource "aws_security_group_rule" "ecs_fl_api_egress_all" { @@ -102,14 +102,16 @@ resource "aws_security_group_rule" "ecs_fl_api_egress_all" { resource "aws_security_group" "ecs_fl_server" { name = "ecs-fl-server" description = "ECS fl-server-net-1 task - inbound gRPC from NLB + HTTP from VPC" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id tags = { FlipSG = "true" } } +# Gated off with the NLB on LZA (FLIP#749): no NLB security group to reference. resource "aws_security_group_rule" "ecs_fl_server_ingress_nlb_grpc" { + count = var.lza_managed_network ? 0 : 1 type = "ingress" description = "gRPC from NLB (FL client connections)" # Backend-dependent container port (Flower: SuperLink Fleet 9092) — the @@ -121,6 +123,14 @@ resource "aws_security_group_rule" "ecs_fl_server_ingress_nlb_grpc" { source_security_group_id = module.fl_server_nlb.security_group_id } +# State migration for the count added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_security_group_rule.ecs_fl_server_ingress_nlb_grpc + to = aws_security_group_rule.ecs_fl_server_ingress_nlb_grpc[0] +} + # fl-api (the NVFLARE admin client) connects directly to fl-server on the # admin port (same 8002 as the gRPC client port; NVFLARE multiplexes admin # and client RPC over a single HTTP/2 stream). It does NOT go through the @@ -170,7 +180,7 @@ resource "aws_security_group_rule" "ecs_fl_server_ingress_vpc_http" { to_port = local.api_container_port protocol = "tcp" security_group_id = aws_security_group.ecs_fl_server.id - cidr_blocks = [var.vpc_cidr] + cidr_blocks = [local.vpc_cidr_block] } resource "aws_security_group_rule" "ecs_fl_server_egress_all" { diff --git a/deploy/providers/AWS/efs.tf b/deploy/providers/AWS/efs.tf index c916090b3..6a953691c 100644 --- a/deploy/providers/AWS/efs.tf +++ b/deploy/providers/AWS/efs.tf @@ -60,7 +60,7 @@ resource "aws_efs_file_system" "flip_fl" { resource "aws_security_group" "efs_mount_target" { name = "efs-mount-target" description = "NFS 2049 from ECS fl-api and fl-server tasks (rules added in PR 2); no egress by design" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id tags = { FlipSG = "true" @@ -72,7 +72,7 @@ resource "aws_security_group" "efs_mount_target" { ############################ resource "aws_efs_mount_target" "flip_fl" { - for_each = var.enable_efs ? toset(module.flip_vpc.private_subnets) : toset([]) + for_each = var.enable_efs ? toset(local.app_subnet_ids) : toset([]) file_system_id = aws_efs_file_system.flip_fl[0].id subnet_id = each.value security_groups = [aws_security_group.efs_mount_target.id] diff --git a/deploy/providers/AWS/fl_ingress_lza.tf b/deploy/providers/AWS/fl_ingress_lza.tf new file mode 100644 index 000000000..b7f57286f --- /dev/null +++ b/deploy/providers/AWS/fl_ingress_lza.tf @@ -0,0 +1,278 @@ +# Copyright (c) 2026 Guy's and St Thomas' NHS Foundation Trust & King's College London +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# LZA-only ingress plumbing (FLIP#749 WP3), the workload-account half of the +# two-tier design proven end-to-end by the e2e harness (FLIP#829/#830), from +# which this file is ported: +# +# FL: trust FL client → internet-facing edge NLB :{FL_SERVER_PORT} +# (networking account) → central firewall → TGW → the INTERNAL NLB +# below → fl-server ECS task. The legacy internet-facing +# module.fl_server_nlb is impossible in-account (no IGW + VPC BPA) and +# stays gated off; this internal NLB exists as the STATIC-IP anchor — +# its per-subnet IPs are assigned (not discovered) via subnet_mapping, +# so the networking account registers them once, with no sync +# machinery. +# Web: CloudFront + WAF → relay NLB (networking account, target-synced to +# this stack's ALB IPs) → firewall → TGW → module.alb. The ALB itself +# is unchanged; the only LZA addition is the ingress rule below — +# legacy admits CloudFront via the SG-drift stack, which is gated off +# here. +# +# Everything in this file is inert on legacy prod/stag (count/create-gated on +# var.lza_managed_network, list arguments emptied) — the legacy plan is +# unchanged. + +# Per-subnet detail for the static IP assignment below. Read at plan time +# (the subnets pre-exist), unlike an ENI-discovery data source, which would +# defer to apply and break for_each on the first plan. +data "aws_subnet" "lza_app" { + for_each = var.lza_managed_network ? toset(data.aws_subnets.lza_app[0].ids) : toset([]) + id = each.value +} + +# The prod VPC's TGW attachment — read to filter the PUBLISHED FL NLB IPs to +# AZs whose return path exists (AWS drops TGW traffic in AZs where the +# attachment has no ENI). Since lza#45 the attachment spans both AZs, so this +# publishes everything today; the filter keeps a future third AZ from being +# registered before its attachment subnet lands. +data "aws_ec2_transit_gateway_vpc_attachment" "lza" { + count = var.lza_managed_network ? 1 : 0 + + filter { + name = "vpc-id" + values = [data.aws_vpc.lza[0].id] + } + + filter { + name = "state" + values = ["available"] + } +} + +data "aws_subnet" "lza_tgw_attachment" { + for_each = var.lza_managed_network ? toset(data.aws_ec2_transit_gateway_vpc_attachment.lza[0].subnet_ids) : toset([]) + id = each.value +} + +locals { + # The FL NLB's private IPs are ASSIGNED via subnet_mapping rather than + # discovered from its ENIs afterwards: known at plan time, stable by + # construction, and publishable to the networking account before apply. + lza_fl_nlb_private_ips = var.lza_managed_network ? [ + for id in local.app_subnet_ids : cidrhost(data.aws_subnet.lza_app[id].cidr_block, var.lza_fl_nlb_host_num) + ] : [] + + lza_tgw_attachment_azs = var.lza_managed_network ? distinct([ + for s in values(data.aws_subnet.lza_tgw_attachment) : s.availability_zone + ]) : [] + + lza_fl_nlb_published_ips = var.lza_managed_network ? [ + for id in local.app_subnet_ids : + cidrhost(data.aws_subnet.lza_app[id].cidr_block, var.lza_fl_nlb_host_num) + if contains(local.lza_tgw_attachment_azs, data.aws_subnet.lza_app[id].availability_zone) + ] : [] +} + +module "fl_internal_nlb_security_group" { + count = var.lza_managed_network ? 1 : 0 + source = "./modules/secgroup" + name = "flip-fl-internal-nlb-sg" + vpc_id = local.vpc_id + description = "Internal FL NLB - FL TCP from the networking-account edge path and in-VPC callers" + ingress_rules = [ + { + # VPC CIDR: in-VPC verification (probe curls, fl-api). The networking + # ingress CIDRs are the edge-NLB → firewall → TGW path. Trust VPN + # CIDRs join this list when the VPN lands. + port = var.FL_SERVER_PORT + description = "FL TCP from the networking-account edge NLB path and VPC-internal callers" + cidr_blocks = concat([local.vpc_cidr_block], var.networking_ingress_cidrs) + } + ] +} + +resource "aws_ec2_tag" "fl_internal_nlb_security_group_flip_sg" { + count = var.lza_managed_network ? 1 : 0 + resource_id = module.fl_internal_nlb_security_group[0].security_group.id + key = "FlipSG" + value = "true" +} + +module "fl_server_internal_nlb" { + source = "terraform-aws-modules/alb/aws" + version = "~> 10.0" + # LZA counterpart of the legacy module.fl_server_nlb above — created on + # exactly the opposite gate, so precisely one FL NLB exists per environment. + create = var.lza_managed_network + name = "flip-fl-internal-nlb" + load_balancer_type = "network" + vpc_id = local.vpc_id + internal = true + + # Static private IP per subnet (see local.lza_fl_nlb_private_ips above). + subnet_mapping = var.lza_managed_network ? [ + for id in local.app_subnet_ids : { + subnet_id = id + private_ipv4_address = cidrhost(data.aws_subnet.lza_app[id].cidr_block, var.lza_fl_nlb_host_num) + } + ] : [] + + create_security_group = false + security_groups = var.lza_managed_network ? [module.fl_internal_nlb_security_group[0].security_group.id] : [] + enable_deletion_protection = false + + # Standalone Fargate TG below, not the module's target_groups map — same + # rationale as aws_lb_target_group.ecs_fl_server_tcp in main.tf. + listeners = { + "fl-server-tcp-listener" = { + port = var.FL_SERVER_PORT + protocol = "TCP" + forward = { + target_group_arn = var.lza_managed_network ? aws_lb_target_group.ecs_fl_server_tcp_lza[0].arn : null + } + } + } + + target_groups = {} +} + +# LZA counterpart of aws_lb_target_group.ecs_fl_server_tcp (main.tf) — +# identical semantics: backend-keyed name (port is ForceNew, see the comment +# there), container port per backend, registered by the ECS service's +# load_balancer block in ecs_services.tf, never by Terraform. +resource "aws_lb_target_group" "ecs_fl_server_tcp_lza" { + count = var.lza_managed_network ? 1 : 0 + name = var.fl_backend == "flower" ? "ecs-fl-server-flwr-lza" : "ecs-fl-server-lza" + port = local.fl_server_container_port + protocol = "TCP" + target_type = "ip" + vpc_id = local.vpc_id + + lifecycle { + create_before_destroy = true + } + + health_check { + enabled = true + protocol = "TCP" + port = "traffic-port" + healthy_threshold = 3 + unhealthy_threshold = 3 + interval = 30 + } + + deregistration_delay = 30 +} + +# Web leg: admit the networking-account relay (CloudFront VPC origin → +# relay NLB → firewall → TGW) onto the ALB's main listener. Legacy admits +# CloudFront through the SG-drift stack instead; empty ingress CIDRs create +# nothing, so the stack still applies standalone. +resource "aws_security_group_rule" "alb_ingress_web_from_networking" { + count = var.lza_managed_network ? 1 : 0 + type = "ingress" + description = "Web from the networking-account relay NLB path (CloudFront VPC origin) and VPC-internal callers" + protocol = "tcp" + from_port = var.ALB_HTTPS_PORT + to_port = var.ALB_HTTPS_PORT + # VPC CIDR: in-VPC verification (probe curls) and internal callers -- the + # same shape as the FL NLB SG above. The networking CIDRs are the relay path. + cidr_blocks = concat([local.vpc_cidr_block], var.networking_ingress_cidrs) + security_group_id = module.alb_security_group.security_group.id +} + +# SSM handoff parameters — the contract with the networking account's edge +# configuration (aicentre-lza-iac), replacing the e2e harness's +# /flip-e2e/networking/ prefix with the real one. +resource "aws_ssm_parameter" "lza_fl_nlb_private_ips" { + # checkov:skip=CKV2_AWS_34:non-secret networking value read CROSS-ACCOUNT by the networking account's edge stack (aicentre-lza-iac) — an AWS-managed CMK cannot be decrypted from another account + count = var.lza_managed_network ? 1 : 0 + name = "/flip/networking/fl_nlb_private_ips" + description = "Internal FL NLB static private IPs (comma-separated, assigned via subnet_mapping so they are stable by construction; TGW-reachable AZs only) - registered as IP targets on the networking account's edge NLB FL listener" + type = "StringList" + value = join(",", local.lza_fl_nlb_published_ips) +} + +resource "aws_ssm_parameter" "lza_fl_port" { + # checkov:skip=CKV2_AWS_34:non-secret networking value read CROSS-ACCOUNT by the networking account's edge stack (aicentre-lza-iac) — an AWS-managed CMK cannot be decrypted from another account + count = var.lza_managed_network ? 1 : 0 + name = "/flip/networking/fl_port" + description = "Workload-side FL ingress port (internal NLB listener) - consumed by the networking account's edge configuration" + type = "String" + value = tostring(var.FL_SERVER_PORT) +} + +resource "aws_ssm_parameter" "lza_alb_dns_name" { + # checkov:skip=CKV2_AWS_34:non-secret networking value read CROSS-ACCOUNT by the networking account's edge stack (aicentre-lza-iac) — an AWS-managed CMK cannot be decrypted from another account + count = var.lza_managed_network ? 1 : 0 + name = "/flip/networking/alb_dns_name" + description = "Internal web ALB DNS name - the networking account's target-sync Lambda resolves this on a cadence to keep the web relay NLB targets current (ALB IPs rotate)" + type = "String" + value = module.alb.dns_name +} + +resource "aws_ssm_parameter" "lza_web_port" { + # checkov:skip=CKV2_AWS_34:non-secret networking value read CROSS-ACCOUNT by the networking account's edge stack (aicentre-lza-iac) — an AWS-managed CMK cannot be decrypted from another account + count = var.lza_managed_network ? 1 : 0 + name = "/flip/networking/web_port" + description = "Workload-side web ingress port (the ALB's main listener; plain HTTP on the zone-less bring-up) - the relay NLB's target port on the networking side" + type = "String" + value = tostring(var.ALB_HTTPS_PORT) +} + +# Task-side admission for the internal NLB: the legacy +# ecs_fl_server_ingress_nlb_grpc rule (ecs_sg.tf) references the legacy NLB's +# SG and is gated off on LZA, which left the fl-server task admitting nothing +# on its FL port -- the NLB's health checks parked the target at unhealthy and +# ECS churned replacements forever. +resource "aws_security_group_rule" "ecs_fl_server_ingress_internal_nlb" { + count = var.lza_managed_network ? 1 : 0 + type = "ingress" + description = "gRPC from the internal FL NLB (edge-relayed FL clients + health checks)" + # Backend-dependent container port (Flower: SuperLink Fleet 9092) -- the + # NLB forwards its FL_SERVER_PORT listener here. + from_port = local.fl_server_container_port + to_port = local.fl_server_container_port + protocol = "tcp" + source_security_group_id = module.fl_internal_nlb_security_group[0].security_group.id + security_group_id = aws_security_group.ecs_fl_server.id +} + +# The NVFLARE admin kit (provisioned from net-1_project_prod.yml) targets the +# bare host `fl-server-net-1` — a SAN on the server cert alongside the public +# FQDN. Legacy resolves it through the flip.local DHCP search domain +# (dhcp.tf), which the LZA-managed VPC cannot carry, and Fargate's awsvpc +# network mode rejects extraHosts — so publish the bare name as a +# single-label private hosted zone whose apex A records are the internal FL +# NLB's deterministic static IPs. A bare-name lookup exhausts the VPC search +# domains and then matches this zone's apex, landing on the NLB → fl-server. +resource "aws_route53_zone" "fl_server_bare_name" { + count = var.lza_managed_network ? 1 : 0 + name = "fl-server-net-1" + comment = "LZA: resolve the NVFLARE admin kit's bare fl-server host inside the workload VPC (FLIP#749)" + + vpc { + vpc_id = local.vpc_id + } +} + +resource "aws_route53_record" "fl_server_bare_name_apex" { + count = var.lza_managed_network ? 1 : 0 + zone_id = aws_route53_zone.fl_server_bare_name[0].zone_id + name = "fl-server-net-1" + type = "A" + ttl = 60 + records = local.lza_fl_nlb_private_ips +} diff --git a/deploy/providers/AWS/iam_ecs.tf b/deploy/providers/AWS/iam_ecs.tf index 7f9f7835d..9d00c6c12 100644 --- a/deploy/providers/AWS/iam_ecs.tf +++ b/deploy/providers/AWS/iam_ecs.tf @@ -91,6 +91,39 @@ resource "aws_iam_role_policy" "ecs_task_execution_secrets" { policy = data.aws_iam_policy_document.ecs_task_execution_secrets.json } +# ECR pull-through cache import (FLIP#749, LZA only). On the LZA account the +# task images come from in-account pull-through caches instead of the public +# registries — the account has no internet egress. Two cache rules exist +# (created out-of-band, not managed here): `ghcr/` mirroring +# ghcr.io/londonaicentre via a read-only PAT in the +# `ecr-pullthroughcache/ghcr` secret, and a credential-less `ecr-public/` rule +# for the EFS-provision utility image. The first pull of a new tag triggers a +# service-side upstream import, which the pulling principal must be allowed to +# perform (plus repository creation on a cache miss); already-cached pulls need +# only the managed execution-role policy above. +data "aws_iam_policy_document" "ecs_task_execution_ecr_pull_through" { + count = var.lza_managed_network ? 1 : 0 + + statement { + sid = "EcrPullThroughCacheImport" + actions = [ + "ecr:BatchImportUpstreamImage", + "ecr:CreateRepository", + ] + resources = [ + "arn:aws:ecr:${var.AWS_REGION}:${data.aws_caller_identity.current.account_id}:repository/ghcr/*", + "arn:aws:ecr:${var.AWS_REGION}:${data.aws_caller_identity.current.account_id}:repository/ecr-public/*", + ] + } +} + +resource "aws_iam_role_policy" "ecs_task_execution_ecr_pull_through" { + count = var.lza_managed_network ? 1 : 0 + name = "flip-ecs-task-execution-ecr-pull-through" + role = aws_iam_role.ecs_task_execution.id + policy = data.aws_iam_policy_document.ecs_task_execution_ecr_pull_through[0].json +} + ############################ # flip-api task role ############################ diff --git a/deploy/providers/AWS/locals.tf b/deploy/providers/AWS/locals.tf index 2be680ec8..87250e5f4 100644 --- a/deploy/providers/AWS/locals.tf +++ b/deploy/providers/AWS/locals.tf @@ -193,7 +193,7 @@ locals { # Flower SuperLink (compose.production.flower.yml fl-server-net-1). TLS + # SuperNode-auth flags travel as the container command (ecs_tasks.tf), not # env. INTERNAL_SERVICE_KEY is injected via the secrets block. - fl_server_flower = { + fl_server_flower = merge({ LOCAL_DEV = "false" NET_ID = "net-1" MIN_CLIENTS = tostring(var.MIN_CLIENTS) @@ -201,7 +201,13 @@ locals { UPLOADED_FEDERATED_DATA_BUCKET = local.uploaded_federated_data_uri FLIP_API_INTERNAL_URL = "http://${local.service_discovery_names.flip_api}:${local.api_container_port}/api" INTERNAL_SERVICE_KEY_HEADER = var.INTERNAL_SERVICE_KEY_HEADER - } + }, var.lza_managed_network ? { + # Sealed egress (FLIP#749): the SuperLink's per-run `uv sync` cannot reach any + # package index, so the ServerApp runs in the image's preinstalled environment + # instead — Flower's documented restricted-network mode (exit-code 608 docs). + # Researcher package additions ship via image rebuild, not pyproject. + FLWR_DISABLE_RUNTIME_DEPENDENCY_INSTALLATION = "1" + } : {}) # Flower fl-api (compose.production.flower.yml fl-api-net-1). SuperLink # addresses use the Cloud Map name — the provisioned server cert must # carry it as a SAN (FLOWER_EXTRA_SERVER_SANS at provision time). diff --git a/deploy/providers/AWS/main.tf b/deploy/providers/AWS/main.tf index e6e1b0e95..26a75419b 100644 --- a/deploy/providers/AWS/main.tf +++ b/deploy/providers/AWS/main.tf @@ -45,8 +45,14 @@ data "aws_availability_zones" "available" {} # TGW VPC attachment. If you recreate or rename this VPC, plan against # aicentre-iac immediately afterwards. module "flip_vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 6.0" + source = "terraform-aws-modules/vpc/aws" + version = "~> 6.0" + # On the LZA account the network is platform-managed and VPC creation is + # SCP-denied — the module's own create flag empties it there (every internal + # resource is gated on it, so no NAT/IGW/EIPs either) without changing its + # state address for the legacy envs. Consumers read the network from the + # locals in network_lza.tf, which switch to data lookups (FLIP#749). + create_vpc = !var.lza_managed_network name = "flip-vpc" azs = slice(data.aws_availability_zones.available.names, 0, var.max_azs) cidr = var.vpc_cidr @@ -67,7 +73,7 @@ module "flip_vpc" { module "ec2_security_group" { source = "./modules/secgroup" name = "ec2-security-group" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id description = "Security group for the FLIP Central Hub SSM bastion (no inbound access)" ingress_rules = [] } @@ -183,8 +189,10 @@ locals { # private IPs), so it is documentation of intent rather than a control. When # var.enable_ecs_endpoints is false the endpoints don't exist and this rule is omitted entirely: # the 443 floor already carries that traffic, and re-stating it as a public rule here would - # collide with the floor on the same tuple. - var.enable_ecs_endpoints ? [ + # collide with the floor on the same tuple. Same on LZA (the gate mirrors the SG's own count in + # vpc_endpoints.tf): endpoints are centralised in the Network account there, so no local + # endpoint SG exists to reference. + var.enable_ecs_endpoints && !var.lza_managed_network ? [ merge(local.trust_egress_rule_defaults, { port = 443 source_security_group_id = aws_security_group.vpc_endpoints[0].id @@ -197,7 +205,7 @@ locals { module "trust_security_group" { source = "./modules/secgroup" name = "trust-security-group" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id description = "Security group for FLIP Trust EC2 instance (no inbound - access via SSM Session Manager and SSM port forwarding)" ingress_rules = [] @@ -216,7 +224,7 @@ resource "aws_ec2_tag" "trust_security_group_flip_sg" { module "rds_security_group" { source = "./modules/secgroup" name = "rds-security-group" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id description = "Security group for FLIP RDS instance" ingress_rules = [ { @@ -243,8 +251,12 @@ resource "aws_ec2_tag" "rds_security_group_flip_sg" { ############################ resource "aws_db_subnet_group" "flip_db_subnet_group" { - name = "flip-db-subnet-group" - subnet_ids = module.flip_vpc.private_subnets + name = "flip-db-subnet-group" + # Data subnets: on the LZA network these are the fully-isolated (local-routes + # only) subnets — RDS never initiates outbound traffic, and the proxy / + # bastion reach it over intra-VPC routing. On legacy these are the private + # subnets, unchanged (see network_lza.tf). + subnet_ids = local.data_subnet_ids } module "flip_db" { @@ -409,7 +421,7 @@ resource "aws_instance" "ec2_instance" { tags = { Name = "Ec2Instance" } - subnet_id = module.flip_vpc.private_subnets[0] + subnet_id = local.app_subnet_ids[0] associate_public_ip_address = false instance_type = "t3.micro" ami = data.aws_ssm_parameter.ubuntu.value @@ -462,7 +474,7 @@ resource "aws_instance" "ec2_instance" { module "alb_security_group" { source = "./modules/secgroup" name = "alb-security-group" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id description = "Security group for FLIP ALB" ingress_rules = [] } @@ -477,12 +489,31 @@ module "alb" { source = "terraform-aws-modules/alb/aws" version = "~> 10.0" name = "flip-alb" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id internal = true - subnets = module.flip_vpc.private_subnets + subnets = local.app_subnet_ids security_groups = [module.alb_security_group.security_group.id] enable_deletion_protection = false + # LZA (FLIP#749): the accelerator guardrail enables access logging to the + # LogArchive bucket out-of-band; mirror it so Terraform stops reverting it. + # The module's access_logs object requires `bucket`, so "no logging" must be + # null (its own default) — an empty map fails type conversion on legacy. + access_logs = var.lza_elb_access_logs_bucket != "" ? { + enabled = true + bucket = var.lza_elb_access_logs_bucket + prefix = "${data.aws_caller_identity.current.account_id}/elb-flip-alb" + } : null + + # The main listener's protocol depends on DNS availability (FLIP#749): with a + # hosted zone it terminates HTTPS with the DNS-validated ACM cert — the + # canonical shape, unchanged for legacy prod/stag. Without one + # (var.manage_dns = false, the zone-less first LZA bring-up) an ISSUED cert + # is impossible, so the same listener (key kept for state stability) serves + # plain HTTP on the same port; that leg only ever carries + # CloudFront-VPC-origin traffic over an AWS-managed ENI inside the VPC + # (viewers still get HTTPS on the default CloudFront domain), and it reverts + # to HTTPS as soon as the zone lands and MANAGE_DNS flips to true. listeners = { # HTTPS default action: return 404. CloudFront is the canonical front door # for user traffic; anything reaching the ALB default action (e.g. direct @@ -491,15 +522,19 @@ module "alb" { # /api/* behaviour and any direct trust access. "https-listener" = { port = var.ALB_HTTPS_PORT - protocol = "HTTPS" - certificate_arn = aws_acm_certificate.flip.arn - ssl_policy = "ELBSecurityPolicy-TLS13-1-3-2021-06" + protocol = var.manage_dns ? "HTTPS" : "HTTP" + certificate_arn = var.manage_dns ? aws_acm_certificate.flip[0].arn : null + ssl_policy = var.manage_dns ? "ELBSecurityPolicy-TLS13-1-3-2021-06" : null fixed_response = { content_type = "text/plain" message_body = "Not Found" status_code = "404" } }, + # On the zone-less bring-up this redirect points at what is temporarily a + # plain-HTTP listener — dead config there, but nothing dials port 80 (the + # VPC origin dials ALB_HTTPS_PORT and the ALB is internal), and keeping the + # key avoids a state churn on the flip back to HTTPS. "http-redirect" = { port = var.ALB_HTTP_PORT protocol = "HTTP" @@ -520,11 +555,17 @@ module "alb" { # Network Load Balancer for FL server TCP/TLS pass-through module "fl_server_nlb" { - source = "terraform-aws-modules/alb/aws" - version = "~> 10.0" + source = "terraform-aws-modules/alb/aws" + version = "~> 10.0" + # Not created on the LZA account (FLIP#749): no IGW + VPC Block Public Access + # make an internet-facing NLB impossible in-account, and the FL inbound + # architecture there (NLB in the central Ingress VPC vs FL-over-443 via the + # VPN) is an open WP2 decision — gate off rather than half-provision. Using + # the module's create flag keeps its state address stable for legacy envs. + create = !var.lza_managed_network name = "flip-fl-server-nlb" load_balancer_type = "network" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id subnets = module.flip_vpc.public_subnets enable_deletion_protection = false create_security_group = true @@ -538,7 +579,9 @@ module "fl_server_nlb" { ip_protocol = "tcp" from_port = tostring(var.FL_SERVER_PORT) to_port = tostring(var.FL_SERVER_PORT) - cidr_ipv4 = "${module.flip_vpc.nat_public_ips[0]}/32" + # Guarded because module arguments are evaluated even with create = + # false: on LZA the VPC module is empty, so there is no NAT EIP to index. + cidr_ipv4 = var.lza_managed_network ? null : "${module.flip_vpc.nat_public_ips[0]}/32" } } @@ -550,7 +593,7 @@ module "fl_server_nlb" { # the listener/ingress side which stays on FL_SERVER_PORT. from_port = tostring(local.fl_server_container_port) to_port = tostring(local.fl_server_container_port) - cidr_ipv4 = var.vpc_cidr + cidr_ipv4 = local.vpc_cidr_block } } @@ -563,7 +606,8 @@ module "fl_server_nlb" { port = var.FL_SERVER_PORT protocol = "TCP" forward = { - target_group_arn = aws_lb_target_group.ecs_fl_server_tcp.arn + # Guarded like the ingress rule above: the TG is count-gated on LZA. + target_group_arn = var.lza_managed_network ? null : aws_lb_target_group.ecs_fl_server_tcp[0].arn } } } @@ -575,12 +619,16 @@ module "fl_server_nlb" { target_groups = {} } +# Skipped when the account has no hosted zone (var.manage_dns = false — the +# zone-less first LZA bring-up, FLIP#749): the lookup would hard-fail there. data "aws_route53_zone" "subdomain" { - name = var.flip_alb_subdomain + count = var.manage_dns ? 1 : 0 + name = var.flip_alb_subdomain } resource "aws_route53_record" "alb" { - zone_id = data.aws_route53_zone.subdomain.zone_id + count = var.manage_dns ? 1 : 0 + zone_id = data.aws_route53_zone.subdomain[0].zone_id name = var.flip_alb_subdomain type = "A" @@ -588,18 +636,28 @@ resource "aws_route53_record" "alb" { # (Resource is still named "alb" for TF-state backwards compatibility; a # rename would recreate the record. The alias target is now CloudFront.) alias { - name = aws_cloudfront_distribution.flip_ui.domain_name - zone_id = aws_cloudfront_distribution.flip_ui.hosted_zone_id + name = aws_cloudfront_distribution.flip_ui[0].domain_name + zone_id = aws_cloudfront_distribution.flip_ui[0].hosted_zone_id evaluate_target_health = false } } +# State migration for the count added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_route53_record.alb + to = aws_route53_record.alb[0] +} + # Target group for the fl-server-net-1 ECS Fargate service. Registered by # the ECS service via the load_balancer block in ecs_services.tf - we never # attach instance/IP targets here. target_type=ip is required for awsvpc # Fargate tasks. NLB protocol must be TCP - HTTP/2 gRPC framing is opaque # to the NLB and forwarded as-is. +# Gated off with the NLB on LZA (FLIP#749): a TG with no LB is dead config. resource "aws_lb_target_group" "ecs_fl_server_tcp" { + count = var.lza_managed_network ? 0 : 1 # Container port per backend (NVFLARE: FL_SERVER_PORT; Flower: SuperLink # Fleet 9092 — see local.fl_server_container_port in ecs_tasks.tf). The # NLB LISTENER stays on var.FL_SERVER_PORT for both. Port is ForceNew, so @@ -610,7 +668,7 @@ resource "aws_lb_target_group" "ecs_fl_server_tcp" { port = local.fl_server_container_port protocol = "TCP" target_type = "ip" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id lifecycle { create_before_destroy = true @@ -630,8 +688,10 @@ resource "aws_lb_target_group" "ecs_fl_server_tcp" { deregistration_delay = 30 } +# Gated off with the NLB on LZA, and with the zone when DNS is unmanaged (FLIP#749). resource "aws_route53_record" "fl_server_nlb" { - zone_id = data.aws_route53_zone.subdomain.zone_id + count = var.manage_dns && !var.lza_managed_network ? 1 : 0 + zone_id = data.aws_route53_zone.subdomain[0].zone_id name = var.flip_nlb_subdomain type = "A" @@ -642,6 +702,19 @@ resource "aws_route53_record" "fl_server_nlb" { } } +# State migration for the counts added to the NLB stack (FLIP#749): keeps +# existing legacy states aligned without a manual `terraform state mv`. Safe to +# remove once every live state file has been migrated. +moved { + from = aws_lb_target_group.ecs_fl_server_tcp + to = aws_lb_target_group.ecs_fl_server_tcp[0] +} + +moved { + from = aws_route53_record.fl_server_nlb + to = aws_route53_record.fl_server_nlb[0] +} + # Target group for the flip-api ECS Fargate service. Registered by the ECS # service itself via the load_balancer block in ecs_services.tf - we never # attach instance/IP targets here from terraform. target_type=ip is required @@ -652,7 +725,7 @@ resource "aws_lb_target_group" "ecs_flip_api" { port = local.api_container_port protocol = "HTTP" target_type = "ip" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id health_check { enabled = true @@ -698,8 +771,9 @@ resource "aws_lb_listener_rule" "api_routing" { # Allow on-prem trust FL clients to reach the FL server via the NLB. # Without this rule the NLB security group drops the connection before it reaches the EC2. +# Emptied on LZA (FLIP#749): there is no NLB (or NLB security group) to attach to. resource "aws_security_group_rule" "local_trust_fl_server_nlb" { - for_each = toset(var.local_trust_public_ips) + for_each = toset(var.lza_managed_network ? [] : var.local_trust_public_ips) type = "ingress" from_port = var.FL_SERVER_PORT to_port = var.FL_SERVER_PORT @@ -723,7 +797,8 @@ resource "aws_security_group_rule" "local_trust_fl_server_nlb" { resource "aws_security_group_rule" "k8s_trust_fl_server_nlb" { # The deprecated scalar is marked sensitive for backwards compatibility, # but an address used as a resource key is necessarily disclosed in state. - for_each = toset(concat( + # Emptied on LZA (FLIP#749) like the on-prem rule above: no NLB there. + for_each = toset(var.lza_managed_network ? [] : concat( var.k8s_trust_public_ips, nonsensitive(var.K8S_TRUST_IP) != "" ? [nonsensitive(var.K8S_TRUST_IP)] : [] )) @@ -747,8 +822,8 @@ output "SsmCommand" { } output "NatGatewayPublicIp" { - description = "NAT Gateway public IP (Central Hub outbound traffic source)" - value = module.flip_vpc.nat_public_ips[0] + description = "NAT Gateway public IP (Central Hub outbound traffic source; null on the LZA platform-managed network, where egress is via the Network account — FLIP#749)" + value = var.lza_managed_network ? null : module.flip_vpc.nat_public_ips[0] } output "TrustEc2InstanceId" { @@ -787,12 +862,12 @@ output "CognitoAppClientId" { } output "FlServerEndpoint" { - description = "FL server DNS endpoint (NLB pass-through)" + description = "FL server DNS endpoint (NLB pass-through; on LZA there is no NLB yet — FL inbound is a FLIP#749 WP2 decision)" value = var.flip_nlb_subdomain } output "FlServerRawNlbDns" { - description = "Raw AWS NLB DNS name for FL server debugging" + description = "Raw AWS NLB DNS name for FL server debugging (null on LZA — FLIP#749)" value = module.fl_server_nlb.dns_name } @@ -850,7 +925,7 @@ module "trust_ec2" { name_prefix = "trust" instance_type = "t3.xlarge" key_name = aws_key_pair.host_key.key_name - subnet_id = element(module.flip_vpc.private_subnets, 0) + subnet_id = element(local.app_subnet_ids, 0) # use the trust SG, not the central EC2 SG security_group_ids = [module.trust_security_group.security_group.id] diff --git a/deploy/providers/AWS/modules/cognito/main.tf b/deploy/providers/AWS/modules/cognito/main.tf index ed76212e5..f4ee8c041 100644 --- a/deploy/providers/AWS/modules/cognito/main.tf +++ b/deploy/providers/AWS/modules/cognito/main.tf @@ -15,6 +15,14 @@ # FLIP Cognito user pool + client + domain + seed users. resource "aws_cognito_user_pool" "flip_user_pool" { + # Feature tier. Null = never sent (legacy pools keep their grandfathered + # tier). The LZA mode pins LITE: ESSENTIALS -- the default for new pools -- + # counts as "ManagedLogin configured", which Cognito refuses to serve over + # PrivateLink, and the interface endpoint is that account's only route to + # cognito-idp (FLIP#749). FLIP uses classic flows (SDK sign-in, TOTP MFA), + # all LITE features. + user_pool_tier = var.user_pool_tier + name = var.user_pool_name username_attributes = ["email"] auto_verified_attributes = ["email"] @@ -111,9 +119,17 @@ resource "random_string" "cognito_domain" { } resource "aws_cognito_user_pool_domain" "main" { + # Gated off on LZA (FLIP#749): FLIP never uses the hosted UI (the app signs + # in via the SDK), and a domain object born with Managed Login keeps its + # "ManagedLogin configured" state through an in-place downgrade to v1 -- + # Cognito then refuses PrivateLink API access to the whole pool, which is + # the LZA account's only route to cognito-idp. Proven empirically 2026-08-20: + # ListUsers through the interface endpoint returned the ManagedLogin error + # with the domain present (any version) and 200 once it was deleted. + count = var.create_hosted_ui_domain ? 1 : 0 domain = random_string.cognito_domain.result user_pool_id = aws_cognito_user_pool.flip_user_pool.id - managed_login_version = 2 + managed_login_version = var.managed_login_version lifecycle { prevent_destroy = true diff --git a/deploy/providers/AWS/modules/cognito/outputs.tf b/deploy/providers/AWS/modules/cognito/outputs.tf index 7231abc2a..1605cb9cf 100644 --- a/deploy/providers/AWS/modules/cognito/outputs.tf +++ b/deploy/providers/AWS/modules/cognito/outputs.tf @@ -29,5 +29,5 @@ output "app_client_id" { output "domain" { description = "Cognito hosted-UI subdomain (the random 8-char prefix, not the full URL)" - value = aws_cognito_user_pool_domain.main.domain + value = one(aws_cognito_user_pool_domain.main[*].domain) } diff --git a/deploy/providers/AWS/modules/cognito/variables.tf b/deploy/providers/AWS/modules/cognito/variables.tf index a965cdcb5..8954d86a3 100644 --- a/deploy/providers/AWS/modules/cognito/variables.tf +++ b/deploy/providers/AWS/modules/cognito/variables.tf @@ -70,3 +70,21 @@ variable "mfa_configuration" { error_message = "mfa_configuration must be one of: OFF, OPTIONAL, ON." } } + +variable "managed_login_version" { + description = "Hosted-UI generation for the user pool domain: 2 = Managed Login (branding designer), 1 = classic hosted UI. Cognito refuses PrivateLink (interface-endpoint) API access to pools with Managed Login configured, so the LZA mode -- whose only route to cognito-idp is the central interface endpoint -- must run 1 (FLIP#749). FLIP's own UI performs SDK-based sign-in, so the hosted-UI generation is cosmetic." + type = number + default = 2 +} + +variable "user_pool_tier" { + description = "Cognito feature tier for the pool (LITE | ESSENTIALS | PLUS). Null omits the argument so existing pools keep their current tier. See the resource comment for why the LZA mode requires LITE." + type = string + default = null +} + +variable "create_hosted_ui_domain" { + description = "Whether to create the hosted-UI user pool domain. False on LZA -- see the resource comment: a domain that has ever had Managed Login blocks PrivateLink access to the pool, and FLIP's flows never use the hosted UI." + type = bool + default = true +} diff --git a/deploy/providers/AWS/network_lza.tf b/deploy/providers/AWS/network_lza.tf new file mode 100644 index 000000000..d9539df37 --- /dev/null +++ b/deploy/providers/AWS/network_lza.tf @@ -0,0 +1,92 @@ +# Copyright (c) 2026 Guy's and St Thomas' NHS Foundation Trust & King's College London +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################ +# Platform-managed network (LZA) — FLIP#749 +############################ +# +# On an LZA-governed workload account the VPC layer is provisioned by the AWS +# Accelerator pipeline (londonaicentre/lza) and the GRNETSEC2 SCP denies +# CreateVpc / CreateSubnet / CreateInternetGateway / AllocateAddress / +# CreateVpcEndpoint to everything but that pipeline (GRTGWVPN makes even +# route-table edits pipeline-only), so this stack cannot create its own network +# there. When var.lza_managed_network is set the VPC module is skipped +# (create_vpc = false in main.tf) and the platform VPC + subnets are discovered +# by Name tag below. +# +# Subnet semantics on the LZA VPC (WP0 probe findings on FLIP#749): +# - app subnets (-app-*): default-route 0.0.0.0/0 to the Transit +# Gateway, so they reach the central interface endpoints in the Network +# account. Anything that talks to AWS APIs, pulls images, or terminates +# traffic lives here: ECS tasks, the internal ALB, the RDS Proxy, EFS mount +# targets, and the EC2 hosts. +# - data subnets (-data-*): local routes only — fully isolated (same-VPC +# traffic works; no TGW, endpoint, or internet reach). Only the RDS +# instances live here; everything that connects to them does so over +# intra-VPC local routing. +# The lookups match ALL Name-tag hits so the multi-AZ -b subnets the platform +# team is adding appear on the next plan without a code change. + +data "aws_vpc" "lza" { + count = var.lza_managed_network ? 1 : 0 + + filter { + name = "tag:Name" + values = [var.lza_vpc_name] + } +} + +data "aws_subnets" "lza_app" { + count = var.lza_managed_network ? 1 : 0 + + filter { + name = "vpc-id" + values = [data.aws_vpc.lza[0].id] + } + + filter { + name = "tag:Name" + values = ["${var.lza_vpc_name}-app-*"] + } +} + +data "aws_subnets" "lza_data" { + count = var.lza_managed_network ? 1 : 0 + + filter { + name = "vpc-id" + values = [data.aws_vpc.lza[0].id] + } + + filter { + name = "tag:Name" + values = ["${var.lza_vpc_name}-data-*"] + } +} + +# Network source of truth for BOTH paths — every consumer references these +# locals; only this file and main.tf's module "flip_vpc" know which path is +# active. The data-lookup branches are sort()ed because aws_subnets returns ids +# in no guaranteed order and positional consumers (e.g. the bastion's +# app_subnet_ids[0]) must not be reshuffled between plans; the legacy branches +# keep the module outputs untouched so existing state never sees an order +# change. +locals { + vpc_id = var.lza_managed_network ? data.aws_vpc.lza[0].id : module.flip_vpc.vpc_id + vpc_cidr_block = var.lza_managed_network ? data.aws_vpc.lza[0].cidr_block : var.vpc_cidr + + # Legacy envs have no app/data split — both roles map onto the private subnets. + app_subnet_ids = var.lza_managed_network ? sort(data.aws_subnets.lza_app[0].ids) : module.flip_vpc.private_subnets + data_subnet_ids = var.lza_managed_network ? sort(data.aws_subnets.lza_data[0].ids) : module.flip_vpc.private_subnets +} diff --git a/deploy/providers/AWS/parameter_store.tf b/deploy/providers/AWS/parameter_store.tf index c9e22ab40..952c47952 100644 --- a/deploy/providers/AWS/parameter_store.tf +++ b/deploy/providers/AWS/parameter_store.tf @@ -59,9 +59,14 @@ resource "aws_ssm_parameter" "flip_app_bundles_bucket" { # network_account_flip module reads these from the FLIP-Prod account to # back the cross-account TGW VPC attachment (single authoritative value # avoids tag-collision ambiguity during VPC migrations). +# +# Gated off on the LZA account (FLIP#749): its TGW attachment is provisioned by +# the accelerator pipeline, not by aicentre-iac, so nothing consumes these +# there — they are the legacy TGW coupling only. resource "aws_ssm_parameter" "vpc_id" { # checkov:skip=CKV2_AWS_34:non-secret networking value read CROSS-ACCOUNT by aicentre-iac — an AWS-managed CMK cannot be decrypted from another account + count = var.lza_managed_network ? 0 : 1 name = "${local.ssm_prefix}/networking/vpc_id" description = "FLIP-Prod VPC ID — consumed cross-account by aicentre-iac's TGW VPC attachment" type = "String" @@ -70,12 +75,25 @@ resource "aws_ssm_parameter" "vpc_id" { resource "aws_ssm_parameter" "private_subnet_ids" { # checkov:skip=CKV2_AWS_34:non-secret networking value read CROSS-ACCOUNT by aicentre-iac — an AWS-managed CMK cannot be decrypted from another account + count = var.lza_managed_network ? 0 : 1 name = "${local.ssm_prefix}/networking/private_subnet_ids" description = "FLIP-Prod private subnet IDs (comma-separated) — consumed cross-account by aicentre-iac's TGW VPC attachment" type = "StringList" value = join(",", module.flip_vpc.private_subnets) } +# State migration for the counts added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_ssm_parameter.vpc_id + to = aws_ssm_parameter.vpc_id[0] +} + +moved { + from = aws_ssm_parameter.private_subnet_ids + to = aws_ssm_parameter.private_subnet_ids[0] +} # FL kit-slot pool names — flip-api's runtime source in production, read at boot # seeding and re-read when a trust registration finds the pool exhausted diff --git a/deploy/providers/AWS/rds_proxy.tf b/deploy/providers/AWS/rds_proxy.tf index 6ae58d589..40729d6f3 100644 --- a/deploy/providers/AWS/rds_proxy.tf +++ b/deploy/providers/AWS/rds_proxy.tf @@ -57,7 +57,7 @@ locals { resource "aws_security_group" "rds_proxy" { name = "rds-proxy" description = "RDS Proxy - inbound 5432 from flip-api, outbound 5432 to RDS" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id tags = { FlipSG = "true" @@ -151,12 +151,16 @@ resource "aws_iam_role_policy" "rds_proxy" { ############################ resource "aws_db_proxy" "flip_db" { - name = "flip-database-proxy" - engine_family = "POSTGRESQL" - require_tls = true - idle_client_timeout = 1800 - role_arn = aws_iam_role.rds_proxy.arn - vpc_subnet_ids = module.flip_vpc.private_subnets + name = "flip-database-proxy" + engine_family = "POSTGRESQL" + require_tls = true + idle_client_timeout = 1800 + role_arn = aws_iam_role.rds_proxy.arn + # App subnets on the LZA network: the proxy's secret retrieval is believed + # to be service-side (verified during FLIP#749 WP3), but app placement keeps + # the central secretsmanager endpoint reachable either way; RDS itself sits + # in the isolated data subnets, reached over intra-VPC local routing. + vpc_subnet_ids = local.app_subnet_ids vpc_security_group_ids = [aws_security_group.rds_proxy.id] auth { diff --git a/deploy/providers/AWS/s3_logging.tf b/deploy/providers/AWS/s3_logging.tf index 9042c7ddc..8d2970ef2 100644 --- a/deploy/providers/AWS/s3_logging.tf +++ b/deploy/providers/AWS/s3_logging.tf @@ -29,7 +29,10 @@ locals { # argument" on any environment where this bucket does not yet exist. The # module callers pass this local; first-apply ordering is still guaranteed # by their `depends_on = [aws_s3_bucket_acl.flip_access_logs]`. - access_logs_bucket_name = "flip-access-logs-${var.flip_alb_subdomain}" + # Name is derived from the subdomain unless overridden — the override exists + # because bucket names are global and the LZA env shares the eventual + # subdomain with legacy prod, which still owns the derived name (FLIP#749). + access_logs_bucket_name = var.ACCESS_LOGS_BUCKET_NAME != "" ? var.ACCESS_LOGS_BUCKET_NAME : "flip-access-logs-${var.flip_alb_subdomain}" } resource "aws_s3_bucket" "flip_access_logs" { diff --git a/deploy/providers/AWS/scripts/add_fl_kits.sh b/deploy/providers/AWS/scripts/add_fl_kits.sh index 2c1d5a6d3..3e5670437 100755 --- a/deploy/providers/AWS/scripts/add_fl_kits.sh +++ b/deploy/providers/AWS/scripts/add_fl_kits.sh @@ -32,7 +32,8 @@ # trust registration — flip-api reconciles its pool on a miss, so no restart and no # task-definition change is needed. # -# Invoked by `make -C deploy/providers/AWS add-fl-kits N= PROD=stag|true [YES=1]`, +# Invoked by `make -C deploy/providers/AWS add-fl-kits N= PROD= [YES=1]` +# (PROD=stag|true, or lza|lza-stag for the LZA estate — FLIP#749), # which exports the env-file vars this script reads (AICENTRE_BUCKET_NAME, # FLARE_KIT_DATE, AWS_PROFILE, FL_BACKEND, PROD) plus N / YES / MAIN_ENV_FILE. # @@ -52,9 +53,9 @@ set -euo pipefail N="${N:?N is required (number of kit slots to add, e.g. N=2)}" -PROD="${PROD:?PROD is required (stag|true)}" -AICENTRE_BUCKET_NAME="${AICENTRE_BUCKET_NAME:?AICENTRE_BUCKET_NAME must be set (run via make with PROD=stag|true)}" -FLARE_KIT_DATE="${FLARE_KIT_DATE:?FLARE_KIT_DATE must be set (run via make with PROD=stag|true)}" +PROD="${PROD:?PROD is required (stag|true|lza|lza-stag)}" +AICENTRE_BUCKET_NAME="${AICENTRE_BUCKET_NAME:?AICENTRE_BUCKET_NAME must be set (run via make with PROD=)}" +FLARE_KIT_DATE="${FLARE_KIT_DATE:?FLARE_KIT_DATE must be set (run via make with PROD=)}" MAIN_ENV_FILE="${MAIN_ENV_FILE:?MAIN_ENV_FILE must be set (the env file to append FL_KIT_SLOT_NAMES to)}" YES="${YES:-}" @@ -69,11 +70,16 @@ if ! [[ "${N}" =~ ^[0-9]+$ ]] || [[ "${N}" -lt 1 ]]; then exit 1 fi +# ENV_NAME selects the CA workspace and project YAML (provision/workspace-, +# net-N_project_.yml). The LZA environments deliberately reuse the legacy +# ones: their kits carry the same server FQDN and root CA, so trusts keep their +# client kits across the DNS cutover (FLIP#749). A staging estate provisioned +# with a fresh CA instead trips the fingerprint guard below, never a silent mint. case "${PROD}" in -true) ENV_NAME="prod" ;; -stag) ENV_NAME="stag" ;; +true | lza) ENV_NAME="prod" ;; +stag | lza-stag) ENV_NAME="stag" ;; *) - echo "❌ PROD must be 'stag' or 'true', got '${PROD}'." >&2 + echo "❌ PROD must be one of 'stag', 'true', 'lza', 'lza-stag' — got '${PROD}'." >&2 exit 1 ;; esac diff --git a/deploy/providers/AWS/scripts/compose-ci-env.sh b/deploy/providers/AWS/scripts/compose-ci-env.sh index b08e708d4..01b95bda9 100755 --- a/deploy/providers/AWS/scripts/compose-ci-env.sh +++ b/deploy/providers/AWS/scripts/compose-ci-env.sh @@ -162,6 +162,28 @@ OPTIONAL_KEYS=( # configuration. Only stag sets it (to "false", for testing). ENFORCE_MFA + # The LZA keys (FLIP#749). CI drives only the two self-contained accounts — + # `TF_ENV` admits stag and prod alone, and a platform-managed estate is + # applied from a laptop with PROD=lza / PROD=lza-stag — so all five are + # expected to be *absent* from the aws-stag / aws-prod GitHub environments, + # and absent is the legacy value in every case: + # + # ACCESS_LOGS_BUCKET_NAME — "" derives flip-access-logs- + # CF_LOGS_BUCKET_NAME — "" derives flip-cf-logs- + # EFS_PROVISION_IMAGE — Makefile exports it only when set; variables.tf + # keeps the Docker Hub amazon/aws-cli default + # LZA_VPC_NAME — same `ifneq` guard; names the accelerator VPC + # MANAGE_DNS — Makefile `?= true`, i.e. the DNS-managed shape + # + # They are still carried through the workflows and listed here rather than + # exempted, so the drift guard above keeps covering them if CI ever gains an + # LZA environment. + ACCESS_LOGS_BUCKET_NAME + CF_LOGS_BUCKET_NAME + EFS_PROVISION_IMAGE + LZA_VPC_NAME + MANAGE_DNS + ) # Deliberately absent: PRESERVE_VPC. It is a make-level flag read only by # `make destroy` (scripts/destroy-selective.sh), never a Terraform input — and CI diff --git a/deploy/providers/AWS/scripts/register-trusts.sh b/deploy/providers/AWS/scripts/register-trusts.sh index 734a89bae..bcc980cb8 100755 --- a/deploy/providers/AWS/scripts/register-trusts.sh +++ b/deploy/providers/AWS/scripts/register-trusts.sh @@ -29,7 +29,8 @@ # Usage: # KIT= register-trusts.sh # register one kit (trust/.env..) # register-trusts.sh # register every live trust/.env.*. kit -# PROD=true selects .production kits; anything else selects .stag. +# PROD=true selects .production kits, PROD=lza selects .lza-prod kits and +# PROD=lza-stag selects .lza-stag kits (FLIP#749); anything else selects .stag. set -eo pipefail # Default file mode 077 so any tempfile / redirect this script (or sourced @@ -50,10 +51,16 @@ ECS_SERVICE="${ECS_SERVICE:-flip-api}" TASK_FAMILY="${TASK_FAMILY:-flip-api}" LOG_GROUP="${LOG_GROUP:-/ecs/flip-api}" -# Kit-file env suffix: PROD=true → .production kits, otherwise → .stag. Matches -# the env file the deploy Makefile included (.env.production / .env.stag). -ENV_SUFFIX="production" -[ "${PROD:-stag}" = "true" ] || ENV_SUFFIX="stag" +# Kit-file env suffix: PROD=true → .production kits, PROD=lza → .lza-prod kits, +# PROD=lza-stag → .lza-stag kits, otherwise → .stag. Must match KIT_ENV_SUFFIX in +# the deploy Makefile (which reads the kit this script writes back) and the env +# file it included (.env.production / .env.lza-prod / .env.lza-stag / .env.stag). +case "${PROD:-stag}" in + true) ENV_SUFFIX="production" ;; + lza) ENV_SUFFIX="lza-prod" ;; + lza-stag) ENV_SUFFIX="lza-stag" ;; + *) ENV_SUFFIX="stag" ;; +esac log_info "Discovering live $ECS_SERVICE service network config..." SVC_JSON="$(aws_cmd ecs describe-services --cluster "$ECS_CLUSTER" --services "$ECS_SERVICE" --query 'services[0]')" diff --git a/deploy/providers/AWS/scripts/tests/test_add_fl_kits.sh b/deploy/providers/AWS/scripts/tests/test_add_fl_kits.sh index 9209a7ef1..c03529b2f 100644 --- a/deploy/providers/AWS/scripts/tests/test_add_fl_kits.sh +++ b/deploy/providers/AWS/scripts/tests/test_add_fl_kits.sh @@ -120,6 +120,7 @@ FAIL=0 STARTUP_OVERRIDES=() KEYCOUNT_OVERRIDES=() YES_MODE="" +PROD_MODE="" # PROD value for the next run_case (default stag) run_case() { local name="$1" N="$2" env_json="$3" nets="$4" @@ -144,7 +145,7 @@ run_case() { LAST_ENVF="$(mktemp --tmpdir="${TEST_ROOT}")" printf 'SOMETHING=1\nFL_KIT_SLOT_NAMES=%s\nOTHER=2\n' "${env_json}" > "${LAST_ENVF}" LAST_OUT="$(printf 'n\n' | env PATH="${MOCKBIN}:${PATH}" \ - N="${N}" PROD=stag AICENTRE_BUCKET_NAME=testbucket FLARE_KIT_DATE=2026-01-01 \ + N="${N}" PROD="${PROD_MODE:-stag}" AICENTRE_BUCKET_NAME=testbucket FLARE_KIT_DATE=2026-01-01 \ MAIN_ENV_FILE="${LAST_ENVF}" FL_BACKEND=nvflare FIXTURE_DIR="${fx}" YES="${YES_MODE}" \ bash "${SCRIPT}" 2>&1)" LAST_RC=$? @@ -154,6 +155,7 @@ run_case() { STARTUP_OVERRIDES=() KEYCOUNT_OVERRIDES=() YES_MODE="" + PROD_MODE="" } check() { # assert the plan's activate/mint lines @@ -289,6 +291,24 @@ check_refuses "refuses net-2 overwrite" "already has objects — refusing to ove check_refuses "reports the net-1 upload" "Kits already uploaded this run: net-1/Trust_3" check_refuses "warns against the instinctive re-run" "Do NOT re-run add-fl-kits" +# --- PROD values (FLIP#749): the LZA pair rides the legacy env workspaces; anything else is refused --- + +PROD_MODE=lza-stag +run_case "PROD=lza-stag plans as the stag env (N=2)" 2 '["Trust_1", "Trust_2"]' "net-1 net-2" \ + "net-1=Trust_1 Trust_2 fl-server-net-1" "net-2=Trust_1 Trust_2 fl-server-net-2" +check "activate none, mint Trust_3 Trust_4" "(none — no spare kits)" "Trust_3 Trust_4 (existing max: Trust_2)" +check_contains "plan header names the stag env" 1 "Plan (stag):" + +PROD_MODE=lza +run_case "PROD=lza plans as the prod env (N=1)" 1 '["Trust_1", "Trust_2"]' "net-1 net-2" \ + "net-1=Trust_1 Trust_2 fl-server-net-1" "net-2=Trust_1 Trust_2 fl-server-net-2" +check_contains "plan header names the prod env" 1 "Plan (prod):" + +PROD_MODE=lza-bogus +run_case "unknown PROD value refused (N=1)" 1 '["Trust_1", "Trust_2"]' "net-1 net-2" \ + "net-1=Trust_1 Trust_2 fl-server-net-1" "net-2=Trust_1 Trust_2 fl-server-net-2" +check_refuses "rejects PROD=lza-bogus before touching anything" "PROD must be one of" + echo "" echo "==== ${PASS} passed, ${FAIL} failed ====" [[ "${FAIL}" -eq 0 ]] diff --git a/deploy/providers/AWS/security.tf b/deploy/providers/AWS/security.tf index 4d602ae3f..79975208b 100644 --- a/deploy/providers/AWS/security.tf +++ b/deploy/providers/AWS/security.tf @@ -20,17 +20,26 @@ # DeleteSecurityGroup, ModifySecurityGroupRules, # UpdateSecurityGroupRuleDescriptions{Ingress,Egress}, CreateSecurityGroup, # and CreateTags/DeleteTags on SG resources (mitigates tag-evasion). +# +# The whole stack is gated off on the LZA account (FLIP#749): the org security +# baseline there (Control Tower org trail, GuardDuty, Security Hub, AWS Config) +# already owns change monitoring, so FLIP's own CloudTrail→EventBridge→Lambda +# plumbing would be redundant noise alongside it. ############################ # SNS Topic for SG drift alerts ############################ resource "aws_sns_topic" "sg_drift" { + count = var.lza_managed_network ? 0 : 1 + name = "flip-sg-drift" } resource "aws_sns_topic_subscription" "sg_drift_email" { - topic_arn = aws_sns_topic.sg_drift.arn + count = var.lza_managed_network ? 0 : 1 + + topic_arn = aws_sns_topic.sg_drift[0].arn protocol = "email" endpoint = var.SES_VERIFIED_EMAIL } @@ -40,6 +49,8 @@ resource "aws_sns_topic_subscription" "sg_drift_email" { ############################ resource "aws_sqs_queue" "sg_drift_dlq" { + count = var.lza_managed_network ? 0 : 1 + name = "flip-sg-drift-dlq" message_retention_seconds = 1209600 # 14 days visibility_timeout_seconds = 30 @@ -50,6 +61,8 @@ resource "aws_sqs_queue" "sg_drift_dlq" { ############################ data "archive_file" "sg_drift_lambda" { + count = var.lza_managed_network ? 0 : 1 + type = "zip" output_path = "${path.module}/.terraform-build/sg_drift_lambda.zip" @@ -164,23 +177,25 @@ PYTHON } resource "aws_lambda_function" "sg_drift_filter" { - filename = data.archive_file.sg_drift_lambda.output_path + count = var.lza_managed_network ? 0 : 1 + + filename = data.archive_file.sg_drift_lambda[0].output_path function_name = "flip-sg-drift-filter" - role = aws_iam_role.sg_drift_lambda_role.arn + role = aws_iam_role.sg_drift_lambda_role[0].arn handler = "lambda_function.handler" runtime = "python3.12" architectures = ["arm64"] - source_code_hash = data.archive_file.sg_drift_lambda.output_base64sha256 + source_code_hash = data.archive_file.sg_drift_lambda[0].output_base64sha256 timeout = 10 memory_size = 128 dead_letter_config { - target_arn = aws_sqs_queue.sg_drift_dlq.arn + target_arn = aws_sqs_queue.sg_drift_dlq[0].arn } environment { variables = { - SNS_TOPIC_ARN = aws_sns_topic.sg_drift.arn + SNS_TOPIC_ARN = aws_sns_topic.sg_drift[0].arn } } } @@ -190,14 +205,18 @@ resource "aws_lambda_function" "sg_drift_filter" { ############################ resource "aws_iam_role" "sg_drift_lambda_role" { + count = var.lza_managed_network ? 0 : 1 + name = "flip-sg-drift-lambda-role" - assume_role_policy = data.aws_iam_policy_document.lambda_assume.json + assume_role_policy = data.aws_iam_policy_document.lambda_assume[0].json # FLIP#962: the CI apply role may only create or grant to a boundary-carrying # role. See locals.tf and ci/main.tf. permissions_boundary = local.iam_permissions_boundary_arn } data "aws_iam_policy_document" "lambda_assume" { + count = var.lza_managed_network ? 0 : 1 + statement { actions = ["sts:AssumeRole"] principals { @@ -208,6 +227,8 @@ data "aws_iam_policy_document" "lambda_assume" { } data "aws_iam_policy_document" "sg_drift_lambda_policy" { + count = var.lza_managed_network ? 0 : 1 + statement { sid = "DescribeSecurityGroups" actions = ["ec2:DescribeSecurityGroups"] @@ -217,13 +238,13 @@ data "aws_iam_policy_document" "sg_drift_lambda_policy" { statement { sid = "PublishSns" actions = ["sns:Publish"] - resources = [aws_sns_topic.sg_drift.arn] + resources = [aws_sns_topic.sg_drift[0].arn] } statement { sid = "SendToDlq" actions = ["sqs:SendMessage"] - resources = [aws_sqs_queue.sg_drift_dlq.arn] + resources = [aws_sqs_queue.sg_drift_dlq[0].arn] } # Basic Lambda logging — scoped to this function's log group. @@ -241,9 +262,11 @@ data "aws_iam_policy_document" "sg_drift_lambda_policy" { } resource "aws_iam_role_policy" "sg_drift_lambda_policy" { + count = var.lza_managed_network ? 0 : 1 + name = "flip-sg-drift-lambda-policy" - role = aws_iam_role.sg_drift_lambda_role.id - policy = data.aws_iam_policy_document.sg_drift_lambda_policy.json + role = aws_iam_role.sg_drift_lambda_role[0].id + policy = data.aws_iam_policy_document.sg_drift_lambda_policy[0].json } ############################ @@ -251,6 +274,8 @@ resource "aws_iam_role_policy" "sg_drift_lambda_policy" { ############################ resource "aws_cloudwatch_event_rule" "sg_drift" { + count = var.lza_managed_network ? 0 : 1 + name = "flip-sg-drift" description = "Capture SG modification events for FlipSG-tagged security groups" @@ -279,17 +304,21 @@ resource "aws_cloudwatch_event_rule" "sg_drift" { } resource "aws_cloudwatch_event_target" "sg_drift" { - rule = aws_cloudwatch_event_rule.sg_drift.name + count = var.lza_managed_network ? 0 : 1 + + rule = aws_cloudwatch_event_rule.sg_drift[0].name target_id = "sg-drift-filter-lambda" - arn = aws_lambda_function.sg_drift_filter.arn + arn = aws_lambda_function.sg_drift_filter[0].arn } resource "aws_lambda_permission" "sg_drift_allow_eventbridge" { + count = var.lza_managed_network ? 0 : 1 + statement_id = "AllowExecutionFromEventBridge" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.sg_drift_filter.function_name + function_name = aws_lambda_function.sg_drift_filter[0].function_name principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.sg_drift.arn + source_arn = aws_cloudwatch_event_rule.sg_drift[0].arn } ############################ @@ -297,6 +326,8 @@ resource "aws_lambda_permission" "sg_drift_allow_eventbridge" { ############################ resource "aws_cloudwatch_metric_alarm" "sg_drift_dlq_depth" { + count = var.lza_managed_network ? 0 : 1 + alarm_name = "flip-sg-drift-dlq-depth" comparison_operator = "GreaterThanThreshold" evaluation_periods = "1" @@ -306,8 +337,67 @@ resource "aws_cloudwatch_metric_alarm" "sg_drift_dlq_depth" { statistic = "Sum" threshold = "0" alarm_description = "SG drift DLQ has undelivered events — Lambda failed after EventBridge retry window" - alarm_actions = [aws_sns_topic.sg_drift.arn] + alarm_actions = [aws_sns_topic.sg_drift[0].arn] dimensions = { - QueueName = aws_sqs_queue.sg_drift_dlq.name + QueueName = aws_sqs_queue.sg_drift_dlq[0].name } } + +############################ +# State migration (FLIP#749) +############################ +# +# The counts added above change every address from to [0]; these +# moved blocks keep existing legacy states aligned without a manual +# `terraform state mv`. Safe to remove once every live state file has been +# migrated. + +moved { + from = aws_sns_topic.sg_drift + to = aws_sns_topic.sg_drift[0] +} + +moved { + from = aws_sns_topic_subscription.sg_drift_email + to = aws_sns_topic_subscription.sg_drift_email[0] +} + +moved { + from = aws_sqs_queue.sg_drift_dlq + to = aws_sqs_queue.sg_drift_dlq[0] +} + +moved { + from = aws_lambda_function.sg_drift_filter + to = aws_lambda_function.sg_drift_filter[0] +} + +moved { + from = aws_iam_role.sg_drift_lambda_role + to = aws_iam_role.sg_drift_lambda_role[0] +} + +moved { + from = aws_iam_role_policy.sg_drift_lambda_policy + to = aws_iam_role_policy.sg_drift_lambda_policy[0] +} + +moved { + from = aws_cloudwatch_event_rule.sg_drift + to = aws_cloudwatch_event_rule.sg_drift[0] +} + +moved { + from = aws_cloudwatch_event_target.sg_drift + to = aws_cloudwatch_event_target.sg_drift[0] +} + +moved { + from = aws_lambda_permission.sg_drift_allow_eventbridge + to = aws_lambda_permission.sg_drift_allow_eventbridge[0] +} + +moved { + from = aws_cloudwatch_metric_alarm.sg_drift_dlq_depth + to = aws_cloudwatch_metric_alarm.sg_drift_dlq_depth[0] +} diff --git a/deploy/providers/AWS/service_discovery.tf b/deploy/providers/AWS/service_discovery.tf index c43c51d61..7845f47d9 100644 --- a/deploy/providers/AWS/service_discovery.tf +++ b/deploy/providers/AWS/service_discovery.tf @@ -26,7 +26,7 @@ resource "aws_service_discovery_private_dns_namespace" "flip_local" { name = local.flip_local_domain description = "Private DNS for ECS service-to-service resolution" - vpc = module.flip_vpc.vpc_id + vpc = local.vpc_id } ############################ diff --git a/deploy/providers/AWS/services.tf b/deploy/providers/AWS/services.tf index ccae1fc33..0e08a4794 100644 --- a/deploy/providers/AWS/services.tf +++ b/deploy/providers/AWS/services.tf @@ -52,8 +52,10 @@ module "flip_model_files_uploads_bucket" { # GET is for model-file downloads: `download_file.py` hands the browser a # presigned GET URL (FLIP#784) instead of proxying bytes through flip-api, # so the bucket needs to accept a direct browser GET. - cors_methods = ["POST", "GET"] - cors_allowed_origins = ["https://${var.flip_alb_subdomain}"] + cors_methods = ["POST", "GET"] + # local.ui_origin (cloudfront.tf) — the canonical subdomain, or the default + # CloudFront domain on a zone-less bring-up (FLIP#749). + cors_allowed_origins = [local.ui_origin] kms_key_arn = aws_kms_key.flip_app_key.arn logging_target_bucket = local.access_logs_bucket_name mfa_delete_protection = true @@ -74,7 +76,7 @@ module "flip_fl_results_bucket" { source = "./modules/flip_s3_bucket" bucket_name = var.FLIP_FL_RESULTS_BUCKET_NAME cors_methods = ["GET"] - cors_allowed_origins = ["https://${var.flip_alb_subdomain}"] + cors_allowed_origins = [local.ui_origin] kms_key_arn = aws_kms_key.flip_app_key.arn logging_target_bucket = local.access_logs_bucket_name mfa_delete_protection = true @@ -135,7 +137,7 @@ resource "aws_s3_bucket_cors_configuration" "aicentre_bucket_cors" { cors_rule { allowed_headers = ["*"] allowed_methods = ["PUT", "GET"] - allowed_origins = ["https://${var.flip_alb_subdomain}"] + allowed_origins = [local.ui_origin] expose_headers = [] } } @@ -191,13 +193,24 @@ resource "aws_s3_bucket_logging" "aicentre_bucket" { module "cognito" { source = "./modules/cognito" - user_pool_name = var.flip_user_pool_name - client_name = var.flip_cognito_client - sign_in_hostname = var.flip_alb_subdomain - admin_email = var.flip_cognito_admin_email - researcher_email = var.flip_cognito_researcher_email - seed_user_password = var.ADMIN_USER_PASSWORD - templates_dir = "${path.module}/templates/cognito" + user_pool_name = var.flip_user_pool_name + client_name = var.flip_cognito_client + # Hostname stamped into invite emails; derived from local.ui_origin + # (cloudfront.tf) so a zone-less bring-up (FLIP#749) sends the reachable + # CloudFront default domain rather than a dead subdomain. Identical to + # var.flip_alb_subdomain whenever DNS is managed (all legacy envs). + sign_in_hostname = trimprefix(local.ui_origin, "https://") + # Managed Login (v2) blocks PrivateLink access to cognito-idp, and the LZA + # account's ONLY path to Cognito is the central interface endpoint -- flip-api + # cannot boot against a v2 pool there (FLIP#749). Cosmetic either way: the + # FLIP UI signs in via the SDK, not the hosted UI. + managed_login_version = var.lza_managed_network ? 1 : 2 + user_pool_tier = var.lza_managed_network ? "LITE" : null + create_hosted_ui_domain = !var.lza_managed_network + admin_email = var.flip_cognito_admin_email + researcher_email = var.flip_cognito_researcher_email + seed_user_password = var.ADMIN_USER_PASSWORD + templates_dir = "${path.module}/templates/cognito" # callback_urls is NOT cosmetic — it is this environment's live browser CORS # allowlist. Do not trim it without reading the following. # @@ -222,7 +235,14 @@ module "cognito" { # localhost is deliberately absent: this root stack is stag/prod only. Local # development uses the separate ./dev root, whose var.cognito_callback_urls # carries the localhost origins. - callback_urls = ["https://${var.flip_alb_subdomain}"] + # + # The one origin is local.ui_origin (cloudfront.tf) rather than a literal + # "https://${var.flip_alb_subdomain}": the two are the same string wherever DNS + # is managed — every legacy environment — and on a zone-less LZA bring-up + # (FLIP#749) the local resolves to the reachable edge/CloudFront domain, which + # is the browser origin there. It is the same value fed to the S3 bucket CORS + # rules above, so the two allowlists cannot drift apart. + callback_urls = [local.ui_origin] # Not read by flip-api; only Cognito's hosted UI would honour these as # post-logout redirect targets, and allowed_oauth_flows_user_pool_client is @@ -231,7 +251,7 @@ module "cognito" { # no redirect target that isn't a real FLIP origin. logout_urls is Optional in # the provider schema and need not mirror callback_urls — the ./dev root # already passes lists of differing length. - logout_urls = ["https://${var.flip_alb_subdomain}"] + logout_urls = [local.ui_origin] } # State migration: Cognito resources used to live at the root of this stack and @@ -255,6 +275,11 @@ moved { to = module.cognito.aws_cognito_user_pool_domain.main } +moved { + from = module.cognito.aws_cognito_user_pool_domain.main + to = module.cognito.aws_cognito_user_pool_domain.main[0] +} + moved { from = aws_cognito_user_pool_client.client to = module.cognito.aws_cognito_user_pool_client.client diff --git a/deploy/providers/AWS/tests/test_cognito_callback_urls.py b/deploy/providers/AWS/tests/test_cognito_callback_urls.py index 8b3c3ecee..96de28da9 100644 --- a/deploy/providers/AWS/tests/test_cognito_callback_urls.py +++ b/deploy/providers/AWS/tests/test_cognito_callback_urls.py @@ -31,6 +31,12 @@ The dev root is asserted in the opposite direction: it *must* keep localhost, so that a well-meant "no localhost in Terraform" sweep cannot break local development instead. + +The stag/prod list holds a ``local.`` reference rather than a quoted URL, because the +canonical origin differs between a DNS-managed environment and a zone-less LZA bring-up +(FLIP#749) and ``cloudfront.tf`` resolves the two into one expression. So the guard reads +raw list entries rather than quoted literals — a reference entry would otherwise read as +no entry at all — and follows each reference to its definition, asserting over that too. """ import re @@ -40,6 +46,11 @@ STAG_PROD_SERVICES_TF = AWS_PROVIDER_DIR / "services.tf" DEV_VARIABLES_TF = AWS_PROVIDER_DIR / "dev" / "variables.tf" +# Where a `local.` entry in the list may be defined. +LOCALS_SOURCES = (AWS_PROVIDER_DIR / "locals.tf", AWS_PROVIDER_DIR / "cloudfront.tf") + +LOCAL_REFERENCE = re.compile(r"^local\.([A-Za-z0-9_]+)$") + def _block(source: str, header: str) -> str: """Extract a brace-balanced HCL block by its opening header. @@ -64,19 +75,37 @@ def _block(source: str, header: str) -> str: raise AssertionError(f"unbalanced braces after {header!r}") -def _string_list(block: str, argument: str) -> list[str]: - """Read a single-line list-of-strings argument out of an HCL block. +def _list_entries(block: str, argument: str) -> list[str]: + """Read a list-valued argument out of an HCL block as its raw entry expressions. Args: block (str): An HCL block body. argument (str): The argument name, e.g. ``callback_urls``. Returns: - list[str]: The quoted entries, with any ``${...}`` interpolation left verbatim. + list[str]: One entry per element, verbatim — a quoted URL keeps its quotes and any + ``${...}`` interpolation, an unquoted expression (``local.ui_origin``) comes back as + written. """ match = re.search(rf"^\s*{re.escape(argument)}\s*=\s*\[(.*?)\]", block, re.MULTILINE | re.DOTALL) assert match is not None, f"{argument} is not set as a literal list — the guard below cannot see it" - return re.findall(r'"([^"]*)"', match.group(1)) + return [entry.strip() for entry in match.group(1).split(",") if entry.strip()] + + +def _local_definition(name: str) -> str: + """Return the expression a ``local.`` reference is defined as. + + Args: + name (str): The local's name, without the ``local.`` prefix. + + Returns: + str: The right-hand side of its definition, as one line of source. + """ + for path in LOCALS_SOURCES: + match = re.search(rf"^\s*{re.escape(name)}\s*=\s*(.+)$", path.read_text(), re.MULTILINE) + if match is not None: + return match.group(1) + raise AssertionError(f"local.{name} is defined in none of {[path.name for path in LOCALS_SOURCES]}") def _cognito_module_argument(argument: str) -> list[str]: @@ -89,7 +118,28 @@ def _cognito_module_argument(argument: str) -> list[str]: list[str]: The argument's entries. """ block = _block(STAG_PROD_SERVICES_TF.read_text(), 'module "cognito"') - return _string_list(block, argument) + return _list_entries(block, argument) + + +def _cognito_origin_sources(argument: str) -> list[str]: + """Every piece of source a stag/prod browser origin could be written into. + + An entry that is a ``local.`` reference contributes the reference *and* the expression + it resolves to, so a localhost origin cannot hide one indirection away. + + Args: + argument (str): The argument name, e.g. ``callback_urls``. + + Returns: + list[str]: Entry expressions, plus the definition of each local they reference. + """ + sources = [] + for entry in _cognito_module_argument(argument): + sources.append(entry) + reference = LOCAL_REFERENCE.match(entry) + if reference is not None: + sources.append(_local_definition(reference.group(1))) + return sources def test_stag_prod_passes_callback_urls_explicitly() -> None: @@ -102,8 +152,8 @@ def test_stag_prod_passes_callback_urls_explicitly() -> None: def test_stag_prod_callback_urls_carry_no_localhost() -> None: - """No stag/prod callback URL may name localhost.""" - offenders = [url for url in _cognito_module_argument("callback_urls") if "localhost" in url.lower()] + """No stag/prod callback URL may name localhost, directly or via the local it references.""" + offenders = [source for source in _cognito_origin_sources("callback_urls") if "localhost" in source.lower()] assert not offenders, f"localhost is not a stag/prod browser origin: {offenders}" @@ -113,7 +163,7 @@ def test_stag_prod_logout_urls_carry_no_localhost() -> None: flip-api does not read these, but the client should advertise no redirect target that isn't a real FLIP origin. """ - offenders = [url for url in _cognito_module_argument("logout_urls") if "localhost" in url.lower()] + offenders = [source for source in _cognito_origin_sources("logout_urls") if "localhost" in source.lower()] assert not offenders, f"localhost is not a stag/prod redirect target: {offenders}" @@ -124,5 +174,5 @@ def test_dev_root_keeps_localhost_callback_urls() -> None: localhost, so stripping it here would break dev sign-in via the same CORS path. """ block = _block(DEV_VARIABLES_TF.read_text(), 'variable "cognito_callback_urls"') - urls = _string_list(block, "default") + urls = _list_entries(block, "default") assert any("localhost" in url.lower() for url in urls), f"dev callback_urls lost its localhost origins: {urls}" diff --git a/deploy/providers/AWS/variables.tf b/deploy/providers/AWS/variables.tf index f1772f4ae..c6938e133 100644 --- a/deploy/providers/AWS/variables.tf +++ b/deploy/providers/AWS/variables.tf @@ -30,6 +30,42 @@ variable "VPC_NAME" { type = string } +variable "lza_managed_network" { + description = "Run against the LZA-provisioned (platform-managed) network instead of creating one (FLIP#749). true skips the VPC module (and with it NAT/IGW/EIPs), the VPC endpoints, the DHCP options, and the legacy /flip/networking/* SSM params (fl_ingress_lza.tf publishes the LZA edge-handoff set under that prefix instead), and discovers the AWSAccelerator VPC + subnets by Name tag instead (see network_lza.tf). Orthogonal to var.environment: the LZA account is prod-grade, so it deploys with environment=prod AND this flag. Set via PROD=lza in the Makefile." + type = bool + default = false +} + +variable "lza_vpc_name" { + description = "Name tag of the platform-managed VPC to discover when lza_managed_network is set. The subnet lookups derive their Name-tag patterns from it (-app-*, -data-*)." + type = string + default = "AWSAccelerator-eu-west-2-prod" +} + +variable "networking_ingress_cidrs" { + description = "CIDRs of the networking account's ingress-VPC subnets (the edge NLB and CloudFront relay path over the TGW), admitted onto the LZA FL NLB and the ALB's main listener (fl_ingress_lza.tf). Empty by default so the stack applies standalone; the value comes from the networking account. LZA-only — legacy prod/stag never reads it." + type = list(string) + default = [] +} + +variable "lza_fl_nlb_host_num" { + description = "Host number (cidrhost index) assigned to the internal FL NLB's static private IP in each app subnet via subnet_mapping (fl_ingress_lza.tf) — e.g. 251 in 10.12.0.0/24 gives 10.12.0.251. Assigned (not discovered) so the IPs are known at plan time and stable by construction; keep it high, clear of DHCP-assigned task/ALB ENIs, and DIFFERENT from the e2e harness's fl_nlb_host_num (250) while both stacks share the app subnets (FLIP#829)." + type = number + default = 251 +} + +variable "lza_web_edge_domain" { + description = "Domain name of the networking account's edge CloudFront distribution -- the user-facing front door on LZA (FLIP#749 WP3; the workload distribution is gated off there). Used for local.ui_origin (CORS / UI origin URL). LZA-only." + type = string + default = "" +} + +variable "lza_web_edge_distribution_arn" { + description = "ARN of the networking account's edge CloudFront distribution, granted s3:GetObject on the flip-ui (and demo-assets) buckets via cross-account OAC on LZA. LZA-only -- legacy grants the in-account distribution instead." + type = string + default = "" +} + variable "max_azs" { type = number } @@ -169,11 +205,17 @@ variable "flip_fl_image_tag" { } variable "docker_registry" { - description = "Docker image registry prefix (e.g. ghcr.io/londonaicentre/)" + description = "Docker image registry prefix (e.g. ghcr.io/londonaicentre/). On the LZA account point it at the ECR pull-through cache mirror (.dkr.ecr..amazonaws.com/ghcr/londonaicentre/) — there is no internet egress to reach GHCR directly (FLIP#749)." type = string default = "ghcr.io/londonaicentre/" } +variable "efs_provision_image" { + description = "Image for the one-shot EFS provisioning task (ecs_efs_provision.tf). Default is the Docker-Hub-hosted amazon/aws-cli, unchanged for legacy envs; on the egress-less LZA account point it at the credential-less ECR Public pull-through cache mirror (.dkr.ecr..amazonaws.com/ecr-public/aws-cli/aws-cli:) via EFS_PROVISION_IMAGE in the env file (FLIP#749)." + type = string + default = "amazon/aws-cli:2.22.35" +} + variable "fl_api_name" { description = "FL API Docker image name (backend-specific: flare-fl-api or flower-fl-api)" type = string @@ -262,6 +304,18 @@ variable "enable_service_discovery" { default = true } +variable "ACCESS_LOGS_BUCKET_NAME" { + description = "Override for the S3 server-access-logs bucket name; empty derives flip-access-logs-. Bucket names are global, so set this where the derived name is already owned by another account — e.g. the LZA env, whose flip_alb_subdomain deliberately keeps its post-cutover value while legacy prod still owns the derived name (FLIP#749)." + type = string + default = "" +} + +variable "CF_LOGS_BUCKET_NAME" { + description = "Override for the CloudFront standard-logs bucket name; empty derives flip-cf-logs-. Same global-name rationale as ACCESS_LOGS_BUCKET_NAME (FLIP#749)." + type = string + default = "" +} + variable "FLIP_MODEL_FILES_UPLOADS_BUCKET_NAME" { description = "Globally-unique S3 bucket name for researcher-uploaded model files (browser presigned-PUT surface today; narrows to presigned POST once PR #438 lands). Required, no default — must be set per environment in the matching .env.*." type = string @@ -367,6 +421,12 @@ variable "FL_SERVER_PORT" { } +variable "manage_dns" { + description = "Whether this account hosts the Route53 zone for flip_alb_subdomain. false (first LZA bring-up, before the zone moves in the platform DNS migration — FLIP#749) skips the zone lookup, every Route53 record, and both DNS-validated ACM certs: CloudFront then serves on its default *.cloudfront.net domain with the default viewer certificate (allowed only when no aliases are set), and the CloudFront→ALB origin leg falls back to plain HTTP over the private VPC-origin ENI, because an ALB HTTPS listener needs an ISSUED certificate and issuance needs DNS validation. Legacy prod/stag keep the default true." + type = bool + default = true +} + variable "flip_alb_subdomain" { description = "Public canonical subdomain for FLIP. Aliased via Route53 to the CloudFront distribution; CloudFront fronts both the SPA (from S3) and the API (/api/* -> ALB). Name is retained for Terraform-state backwards compatibility - see main.tf:492-494." type = string @@ -437,3 +497,16 @@ variable "ecs_exec_enabled" { type = bool default = false } + +variable "lza_elb_access_logs_bucket" { + description = <<-EOT + Name of the LZA LogArchive account's central ELB access-logs bucket. On the + LZA estate the accelerator's guardrail auto-enables ALB access logging to + this bucket out-of-band; setting it here codifies that state so plans stop + proposing to disable platform-managed logging. Empty (the default, and the + only valid value on legacy) omits the access_logs block entirely, keeping + legacy plans byte-identical. + EOT + type = string + default = "" +} diff --git a/deploy/providers/AWS/vpc_endpoints.tf b/deploy/providers/AWS/vpc_endpoints.tf index dd2b9cabb..6e2705357 100644 --- a/deploy/providers/AWS/vpc_endpoints.tf +++ b/deploy/providers/AWS/vpc_endpoints.tf @@ -20,16 +20,21 @@ # Gated behind enable_ecs_endpoints because each interface endpoint incurs an # hourly charge for an ENI in every configured availability zone. # The S3 gateway endpoint is always created (no hourly charge). +# +# On the LZA platform-managed network (FLIP#749) this whole file is gated off: +# CreateVpcEndpoint is SCP-denied there, interface endpoints are centralised in +# the Network account (resolved via PHZs over the TGW), and the LZA pipeline +# already provides in-account S3 + DynamoDB gateway endpoints. ############################ # Security group for interface endpoints ############################ resource "aws_security_group" "vpc_endpoints" { - count = var.enable_ecs_endpoints ? 1 : 0 + count = var.enable_ecs_endpoints && !var.lza_managed_network ? 1 : 0 name = "vpc-endpoints" description = "TLS 443 to AWS interface endpoints from VPC tasks" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id tags = { FlipSG = "true" @@ -37,18 +42,18 @@ resource "aws_security_group" "vpc_endpoints" { } resource "aws_security_group_rule" "vpc_endpoints_ingress_from_vpc" { - count = var.enable_ecs_endpoints ? 1 : 0 + count = var.enable_ecs_endpoints && !var.lza_managed_network ? 1 : 0 type = "ingress" description = "HTTPS from anywhere in the VPC (ECS tasks)" from_port = 443 to_port = 443 protocol = "tcp" security_group_id = aws_security_group.vpc_endpoints[0].id - cidr_blocks = [var.vpc_cidr] + cidr_blocks = [local.vpc_cidr_block] } resource "aws_security_group_rule" "vpc_endpoints_egress_all" { - count = var.enable_ecs_endpoints ? 1 : 0 + count = var.enable_ecs_endpoints && !var.lza_managed_network ? 1 : 0 type = "egress" description = "Default egress for endpoint ENIs" from_port = 0 @@ -71,12 +76,21 @@ data "aws_ec2_managed_prefix_list" "s3" { } resource "aws_vpc_endpoint" "s3" { - vpc_id = module.flip_vpc.vpc_id + count = var.lza_managed_network ? 0 : 1 + vpc_id = local.vpc_id service_name = "com.amazonaws.${var.AWS_REGION}.s3" vpc_endpoint_type = "Gateway" route_table_ids = module.flip_vpc.private_route_table_ids } +# State migration for the count added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_vpc_endpoint.s3 + to = aws_vpc_endpoint.s3[0] +} + ############################ # Interface endpoints ############################ @@ -90,11 +104,11 @@ locals { } resource "aws_vpc_endpoint" "interface" { - for_each = var.enable_ecs_endpoints ? local.interface_endpoint_services : toset([]) - vpc_id = module.flip_vpc.vpc_id + for_each = var.enable_ecs_endpoints && !var.lza_managed_network ? local.interface_endpoint_services : toset([]) + vpc_id = local.vpc_id service_name = "com.amazonaws.${var.AWS_REGION}.${each.value}" vpc_endpoint_type = "Interface" - subnet_ids = module.flip_vpc.private_subnets + subnet_ids = local.app_subnet_ids security_group_ids = [aws_security_group.vpc_endpoints[0].id] private_dns_enabled = true } diff --git a/docs/source/deploy-flip/deploy-central-hub.rst b/docs/source/deploy-flip/deploy-central-hub.rst index 33fce4228..4d5307ae5 100644 --- a/docs/source/deploy-flip/deploy-central-hub.rst +++ b/docs/source/deploy-flip/deploy-central-hub.rst @@ -14,11 +14,55 @@ and :doc:`deploy-flip-node-in-tre`. :local: :depth: 2 +**************** +Deployment modes +**************** + +The Terraform root in ``deploy/providers/AWS/`` supports **two deployment modes**, both +permanently supported. They share one root module — the second is selected by an environment +flag, not a fork of the code. + +.. list-table:: + :header-rows: 1 + :widths: 20 18 31 31 + + * - Mode + - Selected by + - Network + - Ingress + * - **Self-contained** (default) + - ``PROD=stag`` / ``PROD=true`` + - FLIP creates its own VPC, subnets, internet gateway and NAT + - In-account CloudFront for the UI and ``/api/*``; a public NLB for FL traffic + * - **Platform-managed** + - ``PROD=lza`` (production) / ``PROD=lza-stag`` (staging) + - Discovered from a VPC provisioned by the AWS `Landing Zone Accelerator + `_; + FLIP creates no network resources + - A shared networking account's edge, reached over a Transit Gateway; no public load + balancer in the workload account + +**Self-contained** is the default, and the shape the rest of this page assumes. Everything +FLIP needs lives in one AWS account, which is the simplest way to stand up a Central Hub. + +**Platform-managed** suits an AWS estate already governed by the Landing Zone Accelerator, +where a platform team owns the network, the guardrails and a shared edge, and workload +accounts are denied VPC-layer creation. FLIP then discovers the network it is given instead +of building one, and ingress arrives through the estate's own edge. Deploying this way means +coordinating with whoever operates that estate — the account needs its VPC, Transit Gateway +attachment, egress path and edge listeners in place before FLIP applies. The operator-facing +detail (what the flag gates on and off, the environment file, and the handoff to the +networking account) is in the "Deploying onto an LZA estate" section of +`deploy/providers/AWS/README.md +`_. + +Where a command below differs between the two, the difference is called out inline. + ************ Architecture ************ -The Central Hub stack runs in a custom VPC with public and private subnets across two AZs: +In the self-contained mode the Central Hub stack runs in a custom VPC with public and private subnets across two AZs: - **flip-ui** — static assets served from S3 behind CloudFront at the canonical subdomain (``stag.flip.aicentre.co.uk`` / ``app.flip.aicentre.co.uk``). CloudFront also forwards ``/api/*`` to the ALB. - **flip-api** — the central application API. @@ -28,6 +72,11 @@ The Central Hub stack runs in a custom VPC with public and private subnets acros - **SES** — transactional email sent by the application: access-request notifications and XNAT credential emails. Cognito invites and password-reset codes do **not** go through SES (see below). - **Secrets Manager** — AES key, database password, internal service key hash. +Under the platform-managed mode the same services run unchanged; what differs is where they +sit and how traffic reaches them. The VPC and subnets are the accelerator's, there is no +in-account CloudFront distribution and no public NLB, and both web and FL traffic arrive from +the shared networking account over the Transit Gateway. + Operator access is via AWS Systems Manager (SSM) Session Manager — port 22 is **not** open on any security group. @@ -41,13 +90,14 @@ Prerequisites 4. **GitHub CLI** — needed to authenticate against GitHub Container Registry for image pulls. 5. **SSH key pair** at ``~/.ssh/host-aws`` — uploaded to AWS and used as the identity file for the SSM ProxyCommand-based SSH config. -6. **Environment file** — ``.env.stag`` (staging) or ``.env.production`` - (production) in the project root. +6. **Environment file** — ``.env.stag`` (staging), ``.env.production`` + (production), or ``.env.lza-prod`` / ``.env.lza-stag`` (platform-managed) in + the project root. 7. **AWS Session Manager plugin** — required for ``ssh flip`` and ``make forward-trust``. -AWS profile aliases (``prod``, ``stag``, ``dev``) should be configured in -``~/.aws/config`` so the Makefile guards can verify the active profile against -the chosen environment. +AWS profile aliases (``prod``, ``stag``, ``dev``, and ``lza-prod`` / ``lza-stag`` +for a platform-managed deployment) should be configured in ``~/.aws/config`` so +the Makefile guards can verify the active profile against the chosen environment. ************************ Required IAM permissions @@ -113,9 +163,23 @@ This runs, in order: 13. ``status`` — comprehensive health checks. The ``PROD`` variable selects the environment file (``stag`` → ``.env.stag``, -``true`` → ``.env.production``) and is mapped onto ``TF_VAR_environment`` -(``stag`` or ``prod``) so Terraform can gate prod-only RDS hardening (deletion -protection, final snapshot). +``true`` → ``.env.production``, ``lza`` → ``.env.lza-prod``, ``lza-stag`` → +``.env.lza-stag``) and is mapped onto ``TF_VAR_environment`` — ``prod`` for +``true`` and ``lza``, ``stag`` otherwise — so Terraform can gate prod-only RDS +hardening (deletion protection, final snapshot). The two ``lza`` values +additionally set the platform-managed-network flag, which is orthogonal to the +environment name: ``PROD=lza`` is a production estate and keeps every prod-only +control, while ``PROD=lza-stag`` is staging semantics on the same +platform-managed network. + +.. note:: + + ``full-deploy`` is the self-contained chain. Some of its steps — the cloud + trust EC2, the SSH and Ansible provisioning — do not apply to a + platform-managed deployment, and the chain, along with ``make status`` and + ``make destroy``, is not yet exercised there. On the ``lza`` values run + ``init`` / ``plan`` / ``apply`` followed by ``deploy-centralhub`` and + ``deploy-ui``; see the AWS README section linked under `Deployment modes`_. Subsequent UI-only deploys do not need Terraform: @@ -135,7 +199,7 @@ For debugging or selective steps: .. code-block:: shell - export PROD=stag # or: export PROD=true + export PROD=stag # or: export PROD=true, or PROD=lza / PROD=lza-stag (platform-managed) make github-login make aws-login @@ -173,8 +237,9 @@ for full details): trust API key and written into the trust's kit file. ``make generate-internal-service-key`` populates the active env file -(``.env.stag`` or ``.env.production``) and preserves any keys that already -exist; ``make register-trusts`` writes the per-trust keys into the kit files. +(``.env.stag``, ``.env.production`` or ``.env.lza-prod``) and preserves any keys +that already exist; ``make register-trusts`` writes the per-trust keys into the +kit files. *********************** Applying schema changes diff --git a/fl-services/flower/Makefile b/fl-services/flower/Makefile index 9183ebfd2..7642dee90 100644 --- a/fl-services/flower/Makefile +++ b/fl-services/flower/Makefile @@ -19,13 +19,17 @@ REPO_ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/../..) # Load the env file selected by PROD (dev by default), exactly as -# fl-services/nvflare/Makefile does — with PROD=stag|true this brings in the -# upload vars (AICENTRE_BUCKET_NAME, FLOWER_KIT_DATE) used by -# upload-creds-to-s3. +# fl-services/nvflare/Makefile does — with PROD=stag|true (or lza|lza-stag for +# the LZA estate, FLIP#749) this brings in the upload vars (AICENTRE_BUCKET_NAME, +# FLOWER_KIT_DATE) used by upload-creds-to-s3. ifeq ($(PROD),true) MAIN_ENV_FILE := $(REPO_ROOT)/.env.production else ifeq ($(PROD),stag) MAIN_ENV_FILE := $(REPO_ROOT)/.env.stag +else ifeq ($(PROD),lza) +MAIN_ENV_FILE := $(REPO_ROOT)/.env.lza-prod +else ifeq ($(PROD),lza-stag) +MAIN_ENV_FILE := $(REPO_ROOT)/.env.lza-stag else MAIN_ENV_FILE := $(REPO_ROOT)/.env.development endif @@ -71,7 +75,7 @@ provision: # make provision NET_NUMBER=1 # make upload-creds-to-s3 NET_NUMBER=1 PROD=stag upload-creds-to-s3: - @[ -n "$(AICENTRE_BUCKET_NAME)" ] || { echo "❌ AICENTRE_BUCKET_NAME not set — run with PROD=stag|true"; exit 1; } + @[ -n "$(AICENTRE_BUCKET_NAME)" ] || { echo "❌ AICENTRE_BUCKET_NAME not set — run with PROD=stag|true|lza|lza-stag"; exit 1; } @[ -n "$(FLOWER_KIT_DATE)" ] || { echo "❌ FLOWER_KIT_DATE not set — add it to the env file (e.g. 20260811)"; exit 1; } @echo "⬆️ Uploading Flower creds (net-$(NET_NUMBER)) → s3://$(AICENTRE_BUCKET_NAME)/fl-flower-participant-kits/$(FLOWER_KIT_DATE)/net-$(NET_NUMBER) $(DRYRUN)" aws s3 sync ./provision/creds/net-$(NET_NUMBER) s3://$(AICENTRE_BUCKET_NAME)/fl-flower-participant-kits/$(FLOWER_KIT_DATE)/net-$(NET_NUMBER) --delete --exclude "certificates/ca.key" $(DRYRUN) diff --git a/fl-services/flower/fl-api-flower/fl_api/app.py b/fl-services/flower/fl-api-flower/fl_api/app.py index 71a74307d..ea9af91d3 100644 --- a/fl-services/flower/fl-api-flower/fl_api/app.py +++ b/fl-services/flower/fl-api-flower/fl_api/app.py @@ -122,6 +122,9 @@ def _extract_json_from_stdout(stdout: str) -> dict[str, Any]: def _run_flwr_command(command: list[str], cwd: Path, action_name: str) -> subprocess.CompletedProcess[str]: + # The flwr CLI is this service's own baked venv binary (flwr is a direct dependency), + # deliberately NOT `uvx flwr`: uvx resolves its tool environment from PyPI at call + # time, which the egress-less LZA account cannot reach (FLIP#749). try: return subprocess.run( command, @@ -182,7 +185,7 @@ def _validate_tutorial_folder(tutorial_name: str) -> Path: def _get_federation_nodes(src_root: Path) -> list[dict[str, Any]]: """Query the SuperLink Control API for connected SuperNode statuses.""" - command = ["uvx", "flwr", "federation", "list", "--federation", "@none/default", "local", "--format", "json"] + command = ["flwr", "federation", "list", "--federation", "@none/default", "local", "--format", "json"] result = _run_flwr_command(command, src_root, "federation list") payload = _parse_flwr_payload(result, "federation list") @@ -310,7 +313,7 @@ def check_client_status( @app.get("/list_jobs", include_in_schema=False) # alias, hide from docs def list_runs() -> list[JobMetadata]: src_root = _get_src_root() - command = ["uvx", "flwr", "list", "local", "--format", "json"] + command = ["flwr", "list", "local", "--format", "json"] result = _run_flwr_command(command, src_root, "list") payload = _parse_flwr_payload(result, "list") @@ -336,7 +339,7 @@ def _submit_from_job_dir(job_dir: Path, label: str) -> str: run_config_path: str | None = None try: - command = ["uvx", "flwr", "run", ".", "local", "--format", "json"] + command = ["flwr", "run", ".", "local", "--format", "json"] if config_toml_path.is_file(): logger.info("Using config.toml overrides from %s for job submission.", job_dir) # flip-job-dir points the evaluation ServerApp at the app directory, @@ -418,7 +421,7 @@ def _find_terminal_run(src_root: Path, run_id: str) -> JobMetadata | None: Used to make ``DELETE /abort_run`` idempotent: a failed ``flwr stop`` followed by a ``flwr list`` showing the run in a terminal state is treated as a successful no-op. """ - list_command = ["uvx", "flwr", "list", "local", "--format", "json"] + list_command = ["flwr", "list", "local", "--format", "json"] list_result = _run_flwr_command(list_command, src_root, "list") if list_result.returncode != 0: return None @@ -449,7 +452,7 @@ def abort_run(run_id: int) -> JobMetadata: # the string form. src_root = _get_src_root() run_id_str = str(run_id) - command = ["uvx", "flwr", "stop", run_id_str, "local", "--format", "json"] + command = ["flwr", "stop", run_id_str, "local", "--format", "json"] result = _run_flwr_command(command, src_root, "stop") if result.returncode == 0: diff --git a/fl-services/flower/fl-api-flower/tests/conftest.py b/fl-services/flower/fl-api-flower/tests/conftest.py index 6f1259232..b6e7bfc44 100644 --- a/fl-services/flower/fl-api-flower/tests/conftest.py +++ b/fl-services/flower/fl-api-flower/tests/conftest.py @@ -60,9 +60,9 @@ def _raise(*_args, **_kwargs): if by_command is not None: # by_command: {flwr_subcommand: {"returncode": int, "stdout": str, "stderr": str}} - # e.g. {"stop": {...}, "list": {...}}. command is ["uvx", "flwr", "", ...]. + # e.g. {"stop": {...}, "list": {...}}. command is ["flwr", "", ...]. def _dispatch(command, *_args, **_kwargs): - subcommand = command[2] if len(command) > 2 else "" + subcommand = command[1] if len(command) > 1 else "" spec = by_command.get(subcommand, {}) return subprocess.CompletedProcess( args=command, diff --git a/fl-services/nvflare/Makefile b/fl-services/nvflare/Makefile index 38036d932..23421e47c 100644 --- a/fl-services/nvflare/Makefile +++ b/fl-services/nvflare/Makefile @@ -21,12 +21,17 @@ REPO_ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/../..) # Load the env file selected by PROD (dev by default), exactly as the root Makefile # does, so compose + the provisioning targets see the dev vars (FL_API_PORT, -# LOG_LEVEL, NET_ENDPOINTS, FL_PORT, …) and, with PROD=stag|true, the upload vars -# (AICENTRE_BUCKET_NAME, FLARE_KIT_DATE) used by upload-kits-to-s3. +# LOG_LEVEL, NET_ENDPOINTS, FL_PORT, …) and, with PROD=stag|true (or lza|lza-stag +# for the LZA estate, FLIP#749), the upload vars (AICENTRE_BUCKET_NAME, +# FLARE_KIT_DATE) used by upload-kits-to-s3. ifeq ($(PROD),true) MAIN_ENV_FILE := $(REPO_ROOT)/.env.production else ifeq ($(PROD),stag) MAIN_ENV_FILE := $(REPO_ROOT)/.env.stag +else ifeq ($(PROD),lza) +MAIN_ENV_FILE := $(REPO_ROOT)/.env.lza-prod +else ifeq ($(PROD),lza-stag) +MAIN_ENV_FILE := $(REPO_ROOT)/.env.lza-stag else MAIN_ENV_FILE := $(REPO_ROOT)/.env.development endif @@ -110,13 +115,14 @@ provision-add-client-prod: # Upload the provisioned FLARE participant kits to S3 under the FLARE_KIT_DATE prefix # the deploy flow pulls from (deploy/providers/AWS Makefile `provision-local-trust`, # which reads s3://$(AICENTRE_BUCKET_NAME)/fl-flare-participant-kits/$(FLARE_KIT_DATE)/ -# net-N/services//). Run with PROD=stag|true so AICENTRE_BUCKET_NAME + FLARE_KIT_DATE -# load from the matching .env. Defaults to a dry run — pass DRYRUN= to upload for real. -FL_KIT_WORKSPACE := $(if $(filter true,$(PROD)),$(PROD_WORKSPACE_DIR),$(STAG_WORKSPACE_DIR)) +# net-N/services//). Run with PROD=stag|true (or lza|lza-stag) so AICENTRE_BUCKET_NAME + +# FLARE_KIT_DATE load from the matching .env. Defaults to a dry run — pass DRYRUN= to upload +# for real. The LZA pair maps onto the legacy workspaces (same FQDN + CA by design, FLIP#749). +FL_KIT_WORKSPACE := $(if $(filter true lza,$(PROD)),$(PROD_WORKSPACE_DIR),$(STAG_WORKSPACE_DIR)) DRYRUN ?= --dryrun upload-kits-to-s3: - @[ -n "$(AICENTRE_BUCKET_NAME)" ] || { echo "❌ AICENTRE_BUCKET_NAME not set — run with PROD=stag|true"; exit 1; } - @[ -n "$(FLARE_KIT_DATE)" ] || { echo "❌ FLARE_KIT_DATE not set — run with PROD=stag|true"; exit 1; } + @[ -n "$(AICENTRE_BUCKET_NAME)" ] || { echo "❌ AICENTRE_BUCKET_NAME not set — run with PROD=stag|true|lza|lza-stag"; exit 1; } + @[ -n "$(FLARE_KIT_DATE)" ] || { echo "❌ FLARE_KIT_DATE not set — run with PROD=stag|true|lza|lza-stag"; exit 1; } @echo "⬆️ Uploading FLARE kits (net-$(NET_NUMBER), $(FL_KIT_WORKSPACE)) → s3://$(AICENTRE_BUCKET_NAME)/fl-flare-participant-kits/$(FLARE_KIT_DATE)/net-$(NET_NUMBER) $(DRYRUN)" aws s3 sync ./$(FL_KIT_WORKSPACE)/net-$(NET_NUMBER) s3://$(AICENTRE_BUCKET_NAME)/fl-flare-participant-kits/$(FLARE_KIT_DATE)/net-$(NET_NUMBER) --delete $(DRYRUN) diff --git a/flip-api/entrypoint.sh b/flip-api/entrypoint.sh index 016cdd798..eb78d314e 100755 --- a/flip-api/entrypoint.sh +++ b/flip-api/entrypoint.sh @@ -17,6 +17,22 @@ DEBUG=${DEBUG:-false} ENV=${ENV:-development} +# The venv is baked at image build time (uv sync --locked in the Dockerfile). +# Never re-resolve it at boot: uv would otherwise try to rebuild the project's +# editable install, which fetches its build backend (setuptools) from PyPI -- +# on the egress-less LZA account (FLIP#749) that hangs and kills every boot, +# and on legacy it silently re-downloads setuptools each start. Dependency +# changes ship by rebuilding the image (BUILD=true in dev), never at boot. +UV_NO_SYNC=1 +export UV_NO_SYNC + +# With the boot-time sync gone, nothing installs the flip_api package into the +# venv (the Dockerfile syncs dependencies before src/ exists, and dev +# bind-mounts src/ over the image). Put the src layout on the import path +# directly -- one mechanism for prod images and dev mounts alike. +PYTHONPATH="/app/src${PYTHONPATH:+:$PYTHONPATH}" +export PYTHONPATH + echo "🚀 Starting flip-api entrypoint script..." echo "🌍 Environment: $ENV" echo "🐛 Debug mode: $DEBUG" diff --git a/scripts/new_trust.py b/scripts/new_trust.py index 74fac6843..8ea43d2e6 100644 --- a/scripts/new_trust.py +++ b/scripts/new_trust.py @@ -31,13 +31,23 @@ from pathlib import Path +# Kit-file env suffix → the PROD flag that selects it (root Makefile ENV / deploy +# Makefile KIT_ENV_SUFFIX). Suffixes that spell their own flag are absent. +_PROD_FLAG_FOR_ENV = {"production": "true", "lza-prod": "lza"} + + def main() -> None: """CLI entry point: scaffold trust/.env.. from the base template.""" parser = argparse.ArgumentParser(description="Scaffold a new trust kit file.") parser.add_argument("--code", required=True, help="Trust short code — the kit-file handle (e.g. KCH).") parser.add_argument("--name", required=True, help="Trust display name registered on the hub.") parser.add_argument("--region", default=None, help="Optional NHS region.") - parser.add_argument("--env", required=True, choices=("development", "stag", "production"), help="Target env.") + parser.add_argument( + "--env", + required=True, + choices=("development", "stag", "production", "lza-prod", "lza-stag"), + help="Target env — the kit-file suffix (lza-prod / lza-stag = the LZA estate, FLIP#749).", + ) parser.add_argument("--template", default=None, help="Base template (default trust/.env.example).") parser.add_argument("--trust-dir", default=None, help="Override the trust/ directory (testing).") args = parser.parse_args() @@ -65,7 +75,7 @@ def main() -> None: target.write_text("\n".join(identity) + "\n" + template.read_text()) target.chmod(0o600) print(f"✅ Scaffolded {target}") - print(f" Next: make register-trust KIT={args.code} PROD={'true' if args.env == 'production' else args.env}") + print(f" Next: make register-trust KIT={args.code} PROD={_PROD_FLAG_FOR_ENV.get(args.env, args.env)}") if __name__ == "__main__": diff --git a/scripts/sync_trust_kit.py b/scripts/sync_trust_kit.py index 7c47e0685..0be9765c2 100755 --- a/scripts/sync_trust_kit.py +++ b/scripts/sync_trust_kit.py @@ -25,9 +25,9 @@ The caller (the root Makefile) is responsible for sourcing the right env file: `include $(MAIN_ENV_FILE)` + `export` populates os.environ with -.env.development / .env.stag / .env.production values before invoking this -script. No docker compose exec, no ECS round-trip, no per-trust lookup — just a -file→file copy keyed on KIT, portable across dev/stag/prod. +.env.development / .env.stag / .env.production (or .env.lza-prod / .env.lza-stag) +values before invoking this script. No docker compose exec, no ECS round-trip, no +per-trust lookup — just a file→file copy keyed on KIT, portable across dev/stag/prod. """ from __future__ import annotations diff --git a/scripts/tests/test_new_trust.py b/scripts/tests/test_new_trust.py index b78e7ea8c..c6ab7548c 100644 --- a/scripts/tests/test_new_trust.py +++ b/scripts/tests/test_new_trust.py @@ -125,12 +125,27 @@ def test_region_optional() -> None: shutil.rmtree(root) +def test_lza_envs_name_the_kit_and_hint_the_prod_flag() -> None: + print("▶ LZA envs scaffold .env..lza-{prod,stag} and hint the PROD flag that reads them back") + for env, prod_flag in (("lza-stag", "lza-stag"), ("lza-prod", "lza")): + root = _temp_repo() + try: + res = _run(root, "--code", "LZA", "--name", "LZA Trust", "--env", env) + target = root / "trust" / f".env.LZA.{env}" + _assert(res.returncode == 0, f"--env {env}: exit 0", res.stderr) + _assert(target.exists(), f"--env {env}: kit scaffolded at .env..{env}") + _assert(f"PROD={prod_flag}" in res.stdout, f"--env {env}: next-step hint says PROD={prod_flag}", res.stdout) + finally: + shutil.rmtree(root) + + def main() -> None: if not NEW_TRUST.is_file(): sys.exit(f"❌ {NEW_TRUST} not found") test_scaffolds_kit_with_identity_and_defaults() test_refuses_to_overwrite() test_region_optional() + test_lza_envs_name_the_kit_and_hint_the_prod_flag() print("—") print(f"PASS={PASS} FAIL={FAIL}") sys.exit(0 if FAIL == 0 else 1)