From bfb6a2a05ba68f59662202da5b869ae405f87dfc Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 9 Jul 2026 13:19:11 +0100 Subject: [PATCH 01/41] feat(deploy/AWS): add the PROD=lza environment plumbing for the LZA account (#749) PROD=lza selects the gitignored root .env.lza-prod, guards on the FLIPAdminAccess-893493035022 profile (override via LZA_AWS_PROFILE), keeps the prod-grade TF_VAR_environment=prod hardening and origin/main deploy ref, and namespaces trust kits as trust/.env..lza-prod so the parallel-running legacy prod kits are never touched. Legacy PROD=true/stag recipes and exported TF_VARs are byte-identical to before. Signed-off-by: at24_bioeng625-pc --- .gitignore | 1 + deploy/providers/AWS/Makefile | 33 +++++++++++++++---- .../providers/AWS/scripts/register-trusts.sh | 15 ++++++--- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index c28172148..309527b49 100644 --- a/.gitignore +++ b/.gitignore @@ -337,6 +337,7 @@ flip-fl-api/transfer/*/* .env.development .env.stag .env.production +.env.lza-prod # Per-trust operator kits β€” one file per trust, named by trust CODE and # environment (e.g. trust/.env.KCH.development, trust/.env..production). # Minted by the hub via register-trust / POST /admin/trusts; contain plaintext diff --git a/deploy/providers/AWS/Makefile b/deploy/providers/AWS/Makefile index 6220fea47..10b015266 100644 --- a/deploy/providers/AWS/Makefile +++ b/deploy/providers/AWS/Makefile @@ -12,6 +12,12 @@ ifeq ($(PROD),true) MAIN_ENV_FILE=../../../.env.production +else ifeq ($(PROD),lza) +# LZA FLIPProduction account (FLIP#749): a prod-grade environment on the +# platform-managed (AWS Accelerator) network, running in parallel with legacy +# prod until cutover. Gitignored like the other env files β€” see the +# "Deploying to the LZA account" section in README.md for the required values. +MAIN_ENV_FILE=../../../.env.lza-prod else # Sometimes PROD may not be set, default to staging environment in that case PROD=stag @@ -54,6 +60,10 @@ $(if $(FL_KIT_DATE),,$(error 🧠 $(if $(filter nvflare,$(FL_BACKEND)),FLARE_KIT # the env file or on the command line if your local profile names differ. PROD_AWS_PROFILE ?= prod STAG_AWS_PROFILE ?= stag +# The LZA FLIPProduction account (893493035022) is reached via the Identity +# Center `FLIPAdminAccess` permission set; the default matches the profile name +# `aws configure sso` generates for it (FLIP#749). +LZA_AWS_PROFILE ?= FLIPAdminAccess-893493035022 # Refuse to operate against an unintended AWS account. Profile name is the safety # guard because Make has no easy way to read the account ID without an `aws` call. @@ -61,6 +71,10 @@ ifeq ($(PROD),true) ifneq ($(AWS_PROFILE),$(PROD_AWS_PROFILE)) $(error "For production deployments, AWS_PROFILE must be set to '$(PROD_AWS_PROFILE)' (override via PROD_AWS_PROFILE)") endif +else ifeq ($(PROD),lza) +ifneq ($(AWS_PROFILE),$(LZA_AWS_PROFILE)) +$(error "For LZA deployments, AWS_PROFILE must be set to '$(LZA_AWS_PROFILE)' (override via LZA_AWS_PROFILE)") +endif else ifneq ($(AWS_PROFILE),$(STAG_AWS_PROFILE)) $(error "For staging deployments, AWS_PROFILE must be set to '$(STAG_AWS_PROFILE)' (override via STAG_AWS_PROFILE)") @@ -68,13 +82,17 @@ endif endif ## SETTING VARIABLES FOR TERRAFORM -# Map the Make-level PROD flag (true|stag) onto a Terraform-friendly environment name. +# Map the Make-level PROD flag (true|lza|stag) onto a Terraform-friendly environment name. # Used in main.tf to gate prod-only RDS hardening (deletion protection, final snapshot). -export TF_VAR_environment=$(if $(filter true,$(PROD)),prod,stag) - -# Kit-file env suffix: PROD=true β†’ .production kits, otherwise β†’ .stag. Selects -# the trust kit file trust/.env.. for the trust-deploy targets. -KIT_ENV_SUFFIX := $(if $(filter true,$(PROD)),production,stag) +# PROD=lza is a production estate, so it keeps the prod hardening β€” what differs there +# is the platform-managed network, toggled by the orthogonal flag below (FLIP#749). +export TF_VAR_environment=$(if $(filter true lza,$(PROD)),prod,stag) + +# Kit-file env suffix: PROD=true β†’ .production kits, PROD=lza β†’ .lza-prod kits (a +# separate namespace so the parallel-running legacy prod kits are never touched +# during the FLIP#749 migration), otherwise β†’ .stag. Selects the trust kit file +# trust/.env.. for the trust-deploy targets. +KIT_ENV_SUFFIX := $(if $(filter true,$(PROD)),production,$(if $(filter lza,$(PROD)),lza-prod,stag)) export TF_VAR_max_azs=2 export TF_VAR_vpc_cidr=10.0.0.0/16 export TF_VAR_public_subnets=["10.0.1.0/24", "10.0.2.0/24"] @@ -193,7 +211,8 @@ CENTRALHUB_ECS_SERVICES = flip-api fl-api-net-1 fl-server-net-1 # recorded commit and rollback-centralhub can repoint at the previous revision. The # env-file tags (DOCKER_TAG / DOCKER_FL_TAG) remain the bootstrap-only defaults Terraform # bakes into a (re)created task-definition family β€” day-to-day deploys never touch them. -DEPLOY_GIT_REF := $(if $(filter true,$(PROD)),origin/main,origin/develop) +# PROD=lza tracks origin/main like legacy prod (both are production estates, FLIP#749). +DEPLOY_GIT_REF := $(if $(filter true lza,$(PROD)),origin/main,origin/develop) TRUST_IMAGES = \ $(DOCKER_REGISTRY)trust-api:$(DOCKER_TAG) \ diff --git a/deploy/providers/AWS/scripts/register-trusts.sh b/deploy/providers/AWS/scripts/register-trusts.sh index 734a89bae..e2060ccef 100755 --- a/deploy/providers/AWS/scripts/register-trusts.sh +++ b/deploy/providers/AWS/scripts/register-trusts.sh @@ -29,7 +29,8 @@ # Usage: # KIT= register-trusts.sh # register one kit (trust/.env..) # register-trusts.sh # register every live trust/.env.*. kit -# PROD=true selects .production kits; anything else selects .stag. +# PROD=true selects .production kits, PROD=lza selects .lza-prod kits (FLIP#749); +# anything else selects .stag. set -eo pipefail # Default file mode 077 so any tempfile / redirect this script (or sourced @@ -50,10 +51,14 @@ ECS_SERVICE="${ECS_SERVICE:-flip-api}" TASK_FAMILY="${TASK_FAMILY:-flip-api}" LOG_GROUP="${LOG_GROUP:-/ecs/flip-api}" -# Kit-file env suffix: PROD=true β†’ .production kits, otherwise β†’ .stag. Matches -# the env file the deploy Makefile included (.env.production / .env.stag). -ENV_SUFFIX="production" -[ "${PROD:-stag}" = "true" ] || ENV_SUFFIX="stag" +# Kit-file env suffix: PROD=true β†’ .production kits, PROD=lza β†’ .lza-prod kits, +# otherwise β†’ .stag. Matches KIT_ENV_SUFFIX in the deploy Makefile and the env +# file it included (.env.production / .env.lza-prod / .env.stag). +case "${PROD:-stag}" in + true) ENV_SUFFIX="production" ;; + lza) ENV_SUFFIX="lza-prod" ;; + *) ENV_SUFFIX="stag" ;; +esac log_info "Discovering live $ECS_SERVICE service network config..." SVC_JSON="$(aws_cmd ecs describe-services --cluster "$ECS_CLUSTER" --services "$ECS_SERVICE" --query 'services[0]')" From e317c470cf902d9d4ad9530fc9dee2475203fa28 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 9 Jul 2026 13:23:04 +0100 Subject: [PATCH 02/41] feat(deploy/AWS): conditional network layer for the LZA platform-managed VPC (#749) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When TF_VAR_lza_managed_network is set (PROD=lza) the VPC module is emptied via its create flag and the AWSAccelerator VPC + subnets are discovered by Name tag instead (new network_lza.tf), with every consumer rewired through local.vpc_id / local.vpc_cidr_block / local.app_subnet_ids / local.data_subnet_ids. Subnet placement on LZA: RDS goes to the isolated data subnets; ECS tasks, ALB, RDS Proxy, EFS mount targets and EC2 hosts go to the TGW-routed app subnets. The in-account VPC endpoints, DHCP options and /flip/networking/* SSM params (legacy TGW coupling) are gated off there β€” all SCP-denied or platform-provided. moved blocks keep legacy state aligned where a count was added; on legacy envs the flag is false and the resolved config is unchanged. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/Makefile | 12 +++ deploy/providers/AWS/cloudfront.tf | 2 +- deploy/providers/AWS/dhcp.tf | 25 +++++- deploy/providers/AWS/ecs_efs_provision.tf | 2 +- deploy/providers/AWS/ecs_services.tf | 6 +- deploy/providers/AWS/ecs_sg.tf | 12 +-- deploy/providers/AWS/efs.tf | 4 +- deploy/providers/AWS/main.tf | 44 ++++++----- deploy/providers/AWS/network_lza.tf | 92 +++++++++++++++++++++++ deploy/providers/AWS/parameter_store.tf | 18 +++++ deploy/providers/AWS/rds_proxy.tf | 18 +++-- deploy/providers/AWS/service_discovery.tf | 2 +- deploy/providers/AWS/variables.tf | 12 +++ deploy/providers/AWS/vpc_endpoints.tf | 32 +++++--- 14 files changed, 232 insertions(+), 49 deletions(-) create mode 100644 deploy/providers/AWS/network_lza.tf diff --git a/deploy/providers/AWS/Makefile b/deploy/providers/AWS/Makefile index 10b015266..58f35e51a 100644 --- a/deploy/providers/AWS/Makefile +++ b/deploy/providers/AWS/Makefile @@ -88,6 +88,18 @@ endif # is the platform-managed network, toggled by the orthogonal flag below (FLIP#749). export TF_VAR_environment=$(if $(filter true lza,$(PROD)),prod,stag) +# Platform-managed network toggle (FLIP#749): on the LZA FLIPProduction account +# the VPC layer is provisioned by the AWS Accelerator pipeline and SCP-denied to +# this stack, so Terraform switches from creating the network to data lookups +# (see network_lza.tf). Off for every legacy environment. +export TF_VAR_lza_managed_network=$(if $(filter lza,$(PROD)),true,false) +# Name tag of the platform-managed VPC; only consulted when PROD=lza. Set +# LZA_VPC_NAME in the env file to override the variables.tf default +# (AWSAccelerator-eu-west-2-prod) if the platform template is ever renamed. +ifneq ($(LZA_VPC_NAME),) +export TF_VAR_lza_vpc_name=$(LZA_VPC_NAME) +endif + # Kit-file env suffix: PROD=true β†’ .production kits, PROD=lza β†’ .lza-prod kits (a # separate namespace so the parallel-running legacy prod kits are never touched # during the FLIP#749 migration), otherwise β†’ .stag. Selects the trust kit file diff --git a/deploy/providers/AWS/cloudfront.tf b/deploy/providers/AWS/cloudfront.tf index e631568a5..69df45c43 100644 --- a/deploy/providers/AWS/cloudfront.tf +++ b/deploy/providers/AWS/cloudfront.tf @@ -124,7 +124,7 @@ resource "aws_cloudfront_vpc_origin" "flip_api" { # list), not the ENI source IP. data "aws_security_group" "cloudfront_vpcorigins_service" { name = "CloudFront-VPCOrigins-Service-SG" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id depends_on = [aws_cloudfront_vpc_origin.flip_api] } diff --git a/deploy/providers/AWS/dhcp.tf b/deploy/providers/AWS/dhcp.tf index 93db20f44..62b09aafd 100644 --- a/deploy/providers/AWS/dhcp.tf +++ b/deploy/providers/AWS/dhcp.tf @@ -19,8 +19,15 @@ # # AmazonProvidedDNS is the default. Using it explicitly here prevents # Terraform from replacing it with nothing on re-apply. +# +# Gated off on the LZA platform-managed network (FLIP#749): the VPC's +# attributes are owned by the accelerator pipeline, so we don't re-associate +# its DHCP options. Only the bare-hostname search-domain convenience is lost β€” +# every in-repo consumer already resolves the Cloud Map FQDNs (see +# locals.service_discovery_names). resource "aws_vpc_dhcp_options" "flip" { + count = var.lza_managed_network ? 0 : 1 domain_name = local.flip_local_domain domain_name_servers = ["AmazonProvidedDNS"] @@ -30,6 +37,20 @@ resource "aws_vpc_dhcp_options" "flip" { } resource "aws_vpc_dhcp_options_association" "flip" { - vpc_id = module.flip_vpc.vpc_id - dhcp_options_id = aws_vpc_dhcp_options.flip.id + count = var.lza_managed_network ? 0 : 1 + vpc_id = local.vpc_id + dhcp_options_id = aws_vpc_dhcp_options.flip[0].id +} + +# State migration for the counts added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_vpc_dhcp_options.flip + to = aws_vpc_dhcp_options.flip[0] +} + +moved { + from = aws_vpc_dhcp_options_association.flip + to = aws_vpc_dhcp_options_association.flip[0] } diff --git a/deploy/providers/AWS/ecs_efs_provision.tf b/deploy/providers/AWS/ecs_efs_provision.tf index 42d1669c7..7b69f0385 100644 --- a/deploy/providers/AWS/ecs_efs_provision.tf +++ b/deploy/providers/AWS/ecs_efs_provision.tf @@ -59,7 +59,7 @@ resource "null_resource" "provision_efs_certs" { --cluster ${aws_ecs_cluster.flip.name} \ --task-definition ${aws_ecs_task_definition.efs_provision[0].arn} \ --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={subnets=[${join(",", module.flip_vpc.private_subnets)}],securityGroups=[${aws_security_group.ecs_fl_server.id}],assignPublicIp=DISABLED}" \ + --network-configuration "awsvpcConfiguration={subnets=[${join(",", local.app_subnet_ids)}],securityGroups=[${aws_security_group.ecs_fl_server.id}],assignPublicIp=DISABLED}" \ --count 1 \ --region ${var.AWS_REGION} \ --no-cli-pager \ diff --git a/deploy/providers/AWS/ecs_services.tf b/deploy/providers/AWS/ecs_services.tf index 15e7e3746..fec59c801 100644 --- a/deploy/providers/AWS/ecs_services.tf +++ b/deploy/providers/AWS/ecs_services.tf @@ -76,7 +76,7 @@ resource "aws_ecs_service" "flip_api" { enable_execute_command = var.ecs_exec_enabled network_configuration { - subnets = module.flip_vpc.private_subnets + subnets = local.app_subnet_ids security_groups = [aws_security_group.ecs_flip_api.id] assign_public_ip = false } @@ -125,7 +125,7 @@ resource "aws_ecs_service" "fl_api_net_1" { enable_execute_command = var.ecs_exec_enabled network_configuration { - subnets = module.flip_vpc.private_subnets + subnets = local.app_subnet_ids security_groups = [aws_security_group.ecs_fl_api.id] assign_public_ip = false } @@ -163,7 +163,7 @@ resource "aws_ecs_service" "fl_server_net_1" { enable_execute_command = var.ecs_exec_enabled network_configuration { - subnets = module.flip_vpc.private_subnets + subnets = local.app_subnet_ids security_groups = [aws_security_group.ecs_fl_server.id] assign_public_ip = false } diff --git a/deploy/providers/AWS/ecs_sg.tf b/deploy/providers/AWS/ecs_sg.tf index 8b855f6bd..dc6230227 100644 --- a/deploy/providers/AWS/ecs_sg.tf +++ b/deploy/providers/AWS/ecs_sg.tf @@ -24,7 +24,7 @@ resource "aws_security_group" "ecs_flip_api" { name = "ecs-flip-api" description = "ECS flip-api task - inbound HTTP from ALB" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id tags = { FlipSG = "true" @@ -48,7 +48,7 @@ resource "aws_security_group_rule" "ecs_flip_api_ingress_vpc_http" { to_port = local.api_container_port protocol = "tcp" security_group_id = aws_security_group.ecs_flip_api.id - cidr_blocks = [var.vpc_cidr] + cidr_blocks = [local.vpc_cidr_block] } resource "aws_security_group_rule" "ecs_flip_api_egress_all" { @@ -68,7 +68,7 @@ resource "aws_security_group_rule" "ecs_flip_api_egress_all" { resource "aws_security_group" "ecs_fl_api" { name = "ecs-fl-api" description = "ECS fl-api-net-1 task - inbound HTTP from VPC" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id tags = { FlipSG = "true" @@ -82,7 +82,7 @@ resource "aws_security_group_rule" "ecs_fl_api_ingress_vpc_http" { to_port = local.api_container_port protocol = "tcp" security_group_id = aws_security_group.ecs_fl_api.id - cidr_blocks = [var.vpc_cidr] + cidr_blocks = [local.vpc_cidr_block] } resource "aws_security_group_rule" "ecs_fl_api_egress_all" { @@ -102,7 +102,7 @@ resource "aws_security_group_rule" "ecs_fl_api_egress_all" { resource "aws_security_group" "ecs_fl_server" { name = "ecs-fl-server" description = "ECS fl-server-net-1 task - inbound gRPC from NLB + HTTP from VPC" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id tags = { FlipSG = "true" @@ -170,7 +170,7 @@ resource "aws_security_group_rule" "ecs_fl_server_ingress_vpc_http" { to_port = local.api_container_port protocol = "tcp" security_group_id = aws_security_group.ecs_fl_server.id - cidr_blocks = [var.vpc_cidr] + cidr_blocks = [local.vpc_cidr_block] } resource "aws_security_group_rule" "ecs_fl_server_egress_all" { diff --git a/deploy/providers/AWS/efs.tf b/deploy/providers/AWS/efs.tf index c916090b3..6a953691c 100644 --- a/deploy/providers/AWS/efs.tf +++ b/deploy/providers/AWS/efs.tf @@ -60,7 +60,7 @@ resource "aws_efs_file_system" "flip_fl" { resource "aws_security_group" "efs_mount_target" { name = "efs-mount-target" description = "NFS 2049 from ECS fl-api and fl-server tasks (rules added in PR 2); no egress by design" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id tags = { FlipSG = "true" @@ -72,7 +72,7 @@ resource "aws_security_group" "efs_mount_target" { ############################ resource "aws_efs_mount_target" "flip_fl" { - for_each = var.enable_efs ? toset(module.flip_vpc.private_subnets) : toset([]) + for_each = var.enable_efs ? toset(local.app_subnet_ids) : toset([]) file_system_id = aws_efs_file_system.flip_fl[0].id subnet_id = each.value security_groups = [aws_security_group.efs_mount_target.id] diff --git a/deploy/providers/AWS/main.tf b/deploy/providers/AWS/main.tf index cdcbc69fb..c2c1f6fc0 100644 --- a/deploy/providers/AWS/main.tf +++ b/deploy/providers/AWS/main.tf @@ -45,8 +45,14 @@ data "aws_availability_zones" "available" {} # TGW VPC attachment. If you recreate or rename this VPC, plan against # aicentre-iac immediately afterwards. module "flip_vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 6.0" + source = "terraform-aws-modules/vpc/aws" + version = "~> 6.0" + # On the LZA account the network is platform-managed and VPC creation is + # SCP-denied β€” the module's own create flag empties it there (every internal + # resource is gated on it, so no NAT/IGW/EIPs either) without changing its + # state address for the legacy envs. Consumers read the network from the + # locals in network_lza.tf, which switch to data lookups (FLIP#749). + create_vpc = !var.lza_managed_network name = "flip-vpc" azs = slice(data.aws_availability_zones.available.names, 0, var.max_azs) cidr = var.vpc_cidr @@ -67,7 +73,7 @@ module "flip_vpc" { module "ec2_security_group" { source = "./modules/secgroup" name = "ec2-security-group" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id description = "Security group for the FLIP Central Hub SSM bastion (no inbound access)" ingress_rules = [] } @@ -197,7 +203,7 @@ locals { module "trust_security_group" { source = "./modules/secgroup" name = "trust-security-group" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id description = "Security group for FLIP Trust EC2 instance (no inbound - access via SSM Session Manager and SSM port forwarding)" ingress_rules = [] @@ -216,7 +222,7 @@ resource "aws_ec2_tag" "trust_security_group_flip_sg" { module "rds_security_group" { source = "./modules/secgroup" name = "rds-security-group" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id description = "Security group for FLIP RDS instance" ingress_rules = [ { @@ -243,8 +249,12 @@ resource "aws_ec2_tag" "rds_security_group_flip_sg" { ############################ resource "aws_db_subnet_group" "flip_db_subnet_group" { - name = "flip-db-subnet-group" - subnet_ids = module.flip_vpc.private_subnets + name = "flip-db-subnet-group" + # Data subnets: on the LZA network these are the fully-isolated (local-routes + # only) subnets β€” RDS never initiates outbound traffic, and the proxy / + # bastion reach it over intra-VPC routing. On legacy these are the private + # subnets, unchanged (see network_lza.tf). + subnet_ids = local.data_subnet_ids } module "flip_db" { @@ -405,7 +415,7 @@ resource "aws_instance" "ec2_instance" { tags = { Name = "Ec2Instance" } - subnet_id = module.flip_vpc.private_subnets[0] + subnet_id = local.app_subnet_ids[0] associate_public_ip_address = false instance_type = "t3.micro" ami = data.aws_ssm_parameter.ubuntu.value @@ -450,7 +460,7 @@ resource "aws_instance" "ec2_instance" { module "alb_security_group" { source = "./modules/secgroup" name = "alb-security-group" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id description = "Security group for FLIP ALB" ingress_rules = [] } @@ -464,9 +474,9 @@ resource "aws_ec2_tag" "alb_security_group_flip_sg" { module "alb" { source = "terraform-aws-modules/alb/aws" name = "flip-alb" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id internal = true - subnets = module.flip_vpc.private_subnets + subnets = local.app_subnet_ids security_groups = [module.alb_security_group.security_group.id] enable_deletion_protection = false @@ -510,7 +520,7 @@ module "fl_server_nlb" { source = "terraform-aws-modules/alb/aws" name = "flip-fl-server-nlb" load_balancer_type = "network" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id subnets = module.flip_vpc.public_subnets enable_deletion_protection = false create_security_group = true @@ -596,7 +606,7 @@ resource "aws_lb_target_group" "ecs_fl_server_tcp" { port = local.fl_server_container_port protocol = "TCP" target_type = "ip" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id lifecycle { create_before_destroy = true @@ -638,7 +648,7 @@ resource "aws_lb_target_group" "ecs_flip_api" { port = local.api_container_port protocol = "HTTP" target_type = "ip" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id health_check { enabled = true @@ -733,8 +743,8 @@ output "SsmCommand" { } output "NatGatewayPublicIp" { - description = "NAT Gateway public IP (Central Hub outbound traffic source)" - value = module.flip_vpc.nat_public_ips[0] + description = "NAT Gateway public IP (Central Hub outbound traffic source; null on the LZA platform-managed network, where egress is via the Network account β€” FLIP#749)" + value = var.lza_managed_network ? null : module.flip_vpc.nat_public_ips[0] } output "TrustEc2InstanceId" { @@ -836,7 +846,7 @@ module "trust_ec2" { name_prefix = "trust" instance_type = "t3.xlarge" key_name = aws_key_pair.host_key.key_name - subnet_id = element(module.flip_vpc.private_subnets, 0) + subnet_id = element(local.app_subnet_ids, 0) # use the trust SG, not the central EC2 SG security_group_ids = [module.trust_security_group.security_group.id] diff --git a/deploy/providers/AWS/network_lza.tf b/deploy/providers/AWS/network_lza.tf new file mode 100644 index 000000000..6ebf40988 --- /dev/null +++ b/deploy/providers/AWS/network_lza.tf @@ -0,0 +1,92 @@ +# Copyright (c) 2026 Guy's and St Thomas' NHS Foundation Trust & King's College London +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################ +# Platform-managed network (LZA) β€” FLIP#749 +############################ +# +# On the LZA FLIPProduction account (893493035022) the VPC layer is provisioned +# by the AWS Accelerator pipeline (londonaicentre/lza) and the GRNETSEC2 SCP +# denies CreateVpc / CreateSubnet / CreateInternetGateway / AllocateAddress / +# CreateVpcEndpoint to everything but that pipeline (GRTGWVPN makes even +# route-table edits pipeline-only), so this stack cannot create its own network +# there. When var.lza_managed_network is set the VPC module is skipped +# (create_vpc = false in main.tf) and the platform VPC + subnets are discovered +# by Name tag below. +# +# Subnet semantics on the LZA VPC (WP0 probe findings on FLIP#749): +# - app subnets (-app-*): default-route 0.0.0.0/0 to the Transit +# Gateway, so they reach the central interface endpoints in the Network +# account. Anything that talks to AWS APIs, pulls images, or terminates +# traffic lives here: ECS tasks, the internal ALB, the RDS Proxy, EFS mount +# targets, and the EC2 hosts. +# - data subnets (-data-*): local routes only β€” fully isolated (same-VPC +# traffic works; no TGW, endpoint, or internet reach). Only the RDS +# instances live here; everything that connects to them does so over +# intra-VPC local routing. +# The lookups match ALL Name-tag hits so the multi-AZ -b subnets the platform +# team is adding appear on the next plan without a code change. + +data "aws_vpc" "lza" { + count = var.lza_managed_network ? 1 : 0 + + filter { + name = "tag:Name" + values = [var.lza_vpc_name] + } +} + +data "aws_subnets" "lza_app" { + count = var.lza_managed_network ? 1 : 0 + + filter { + name = "vpc-id" + values = [data.aws_vpc.lza[0].id] + } + + filter { + name = "tag:Name" + values = ["${var.lza_vpc_name}-app-*"] + } +} + +data "aws_subnets" "lza_data" { + count = var.lza_managed_network ? 1 : 0 + + filter { + name = "vpc-id" + values = [data.aws_vpc.lza[0].id] + } + + filter { + name = "tag:Name" + values = ["${var.lza_vpc_name}-data-*"] + } +} + +# Network source of truth for BOTH paths β€” every consumer references these +# locals; only this file and main.tf's module "flip_vpc" know which path is +# active. The data-lookup branches are sort()ed because aws_subnets returns ids +# in no guaranteed order and positional consumers (e.g. the bastion's +# app_subnet_ids[0]) must not be reshuffled between plans; the legacy branches +# keep the module outputs untouched so existing state never sees an order +# change. +locals { + vpc_id = var.lza_managed_network ? data.aws_vpc.lza[0].id : module.flip_vpc.vpc_id + vpc_cidr_block = var.lza_managed_network ? data.aws_vpc.lza[0].cidr_block : var.vpc_cidr + + # Legacy envs have no app/data split β€” both roles map onto the private subnets. + app_subnet_ids = var.lza_managed_network ? sort(data.aws_subnets.lza_app[0].ids) : module.flip_vpc.private_subnets + data_subnet_ids = var.lza_managed_network ? sort(data.aws_subnets.lza_data[0].ids) : module.flip_vpc.private_subnets +} diff --git a/deploy/providers/AWS/parameter_store.tf b/deploy/providers/AWS/parameter_store.tf index 1055f3847..79fe52add 100644 --- a/deploy/providers/AWS/parameter_store.tf +++ b/deploy/providers/AWS/parameter_store.tf @@ -55,8 +55,13 @@ resource "aws_ssm_parameter" "flip_app_bundles_bucket" { # network_account_flip module reads these from the FLIP-Prod account to # back the cross-account TGW VPC attachment (single authoritative value # avoids tag-collision ambiguity during VPC migrations). +# +# Gated off on the LZA account (FLIP#749): its TGW attachment is provisioned by +# the accelerator pipeline, not by aicentre-iac, so nothing consumes these +# there β€” they are the legacy TGW coupling only. resource "aws_ssm_parameter" "vpc_id" { + count = var.lza_managed_network ? 0 : 1 name = "${local.ssm_prefix}/networking/vpc_id" description = "FLIP-Prod VPC ID β€” consumed cross-account by aicentre-iac's TGW VPC attachment" type = "String" @@ -64,12 +69,25 @@ resource "aws_ssm_parameter" "vpc_id" { } resource "aws_ssm_parameter" "private_subnet_ids" { + count = var.lza_managed_network ? 0 : 1 name = "${local.ssm_prefix}/networking/private_subnet_ids" description = "FLIP-Prod private subnet IDs (comma-separated) β€” consumed cross-account by aicentre-iac's TGW VPC attachment" type = "StringList" value = join(",", module.flip_vpc.private_subnets) } +# State migration for the counts added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_ssm_parameter.vpc_id + to = aws_ssm_parameter.vpc_id[0] +} + +moved { + from = aws_ssm_parameter.private_subnet_ids + to = aws_ssm_parameter.private_subnet_ids[0] +} # FL kit-slot pool names β€” flip-api's runtime source in production, read at boot # seeding and re-read when a trust registration finds the pool exhausted diff --git a/deploy/providers/AWS/rds_proxy.tf b/deploy/providers/AWS/rds_proxy.tf index 8398c8b99..df07f7932 100644 --- a/deploy/providers/AWS/rds_proxy.tf +++ b/deploy/providers/AWS/rds_proxy.tf @@ -57,7 +57,7 @@ locals { resource "aws_security_group" "rds_proxy" { name = "rds-proxy" description = "RDS Proxy - inbound 5432 from flip-api, outbound 5432 to RDS" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id tags = { FlipSG = "true" @@ -148,12 +148,16 @@ resource "aws_iam_role_policy" "rds_proxy" { ############################ resource "aws_db_proxy" "flip_db" { - name = "flip-database-proxy" - engine_family = "POSTGRESQL" - require_tls = true - idle_client_timeout = 1800 - role_arn = aws_iam_role.rds_proxy.arn - vpc_subnet_ids = module.flip_vpc.private_subnets + name = "flip-database-proxy" + engine_family = "POSTGRESQL" + require_tls = true + idle_client_timeout = 1800 + role_arn = aws_iam_role.rds_proxy.arn + # App subnets on the LZA network: the proxy's secret retrieval is believed + # to be service-side (verified during FLIP#749 WP3), but app placement keeps + # the central secretsmanager endpoint reachable either way; RDS itself sits + # in the isolated data subnets, reached over intra-VPC local routing. + vpc_subnet_ids = local.app_subnet_ids vpc_security_group_ids = [aws_security_group.rds_proxy.id] auth { diff --git a/deploy/providers/AWS/service_discovery.tf b/deploy/providers/AWS/service_discovery.tf index c43c51d61..7845f47d9 100644 --- a/deploy/providers/AWS/service_discovery.tf +++ b/deploy/providers/AWS/service_discovery.tf @@ -26,7 +26,7 @@ resource "aws_service_discovery_private_dns_namespace" "flip_local" { name = local.flip_local_domain description = "Private DNS for ECS service-to-service resolution" - vpc = module.flip_vpc.vpc_id + vpc = local.vpc_id } ############################ diff --git a/deploy/providers/AWS/variables.tf b/deploy/providers/AWS/variables.tf index a4415b9bc..a339a0945 100644 --- a/deploy/providers/AWS/variables.tf +++ b/deploy/providers/AWS/variables.tf @@ -30,6 +30,18 @@ variable "VPC_NAME" { type = string } +variable "lza_managed_network" { + description = "Run against the LZA-provisioned (platform-managed) network instead of creating one (FLIP#749). true skips the VPC module (and with it NAT/IGW/EIPs), the VPC endpoints, the DHCP options, and the /flip/networking/* SSM params, and discovers the AWSAccelerator VPC + subnets by Name tag instead (see network_lza.tf). Orthogonal to var.environment: the LZA account is prod-grade, so it deploys with environment=prod AND this flag. Set via PROD=lza in the Makefile." + type = bool + default = false +} + +variable "lza_vpc_name" { + description = "Name tag of the platform-managed VPC to discover when lza_managed_network is set. The subnet lookups derive their Name-tag patterns from it (-app-*, -data-*)." + type = string + default = "AWSAccelerator-eu-west-2-prod" +} + variable "max_azs" { type = number } diff --git a/deploy/providers/AWS/vpc_endpoints.tf b/deploy/providers/AWS/vpc_endpoints.tf index dd2b9cabb..6e2705357 100644 --- a/deploy/providers/AWS/vpc_endpoints.tf +++ b/deploy/providers/AWS/vpc_endpoints.tf @@ -20,16 +20,21 @@ # Gated behind enable_ecs_endpoints because each interface endpoint incurs an # hourly charge for an ENI in every configured availability zone. # The S3 gateway endpoint is always created (no hourly charge). +# +# On the LZA platform-managed network (FLIP#749) this whole file is gated off: +# CreateVpcEndpoint is SCP-denied there, interface endpoints are centralised in +# the Network account (resolved via PHZs over the TGW), and the LZA pipeline +# already provides in-account S3 + DynamoDB gateway endpoints. ############################ # Security group for interface endpoints ############################ resource "aws_security_group" "vpc_endpoints" { - count = var.enable_ecs_endpoints ? 1 : 0 + count = var.enable_ecs_endpoints && !var.lza_managed_network ? 1 : 0 name = "vpc-endpoints" description = "TLS 443 to AWS interface endpoints from VPC tasks" - vpc_id = module.flip_vpc.vpc_id + vpc_id = local.vpc_id tags = { FlipSG = "true" @@ -37,18 +42,18 @@ resource "aws_security_group" "vpc_endpoints" { } resource "aws_security_group_rule" "vpc_endpoints_ingress_from_vpc" { - count = var.enable_ecs_endpoints ? 1 : 0 + count = var.enable_ecs_endpoints && !var.lza_managed_network ? 1 : 0 type = "ingress" description = "HTTPS from anywhere in the VPC (ECS tasks)" from_port = 443 to_port = 443 protocol = "tcp" security_group_id = aws_security_group.vpc_endpoints[0].id - cidr_blocks = [var.vpc_cidr] + cidr_blocks = [local.vpc_cidr_block] } resource "aws_security_group_rule" "vpc_endpoints_egress_all" { - count = var.enable_ecs_endpoints ? 1 : 0 + count = var.enable_ecs_endpoints && !var.lza_managed_network ? 1 : 0 type = "egress" description = "Default egress for endpoint ENIs" from_port = 0 @@ -71,12 +76,21 @@ data "aws_ec2_managed_prefix_list" "s3" { } resource "aws_vpc_endpoint" "s3" { - vpc_id = module.flip_vpc.vpc_id + count = var.lza_managed_network ? 0 : 1 + vpc_id = local.vpc_id service_name = "com.amazonaws.${var.AWS_REGION}.s3" vpc_endpoint_type = "Gateway" route_table_ids = module.flip_vpc.private_route_table_ids } +# State migration for the count added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_vpc_endpoint.s3 + to = aws_vpc_endpoint.s3[0] +} + ############################ # Interface endpoints ############################ @@ -90,11 +104,11 @@ locals { } resource "aws_vpc_endpoint" "interface" { - for_each = var.enable_ecs_endpoints ? local.interface_endpoint_services : toset([]) - vpc_id = module.flip_vpc.vpc_id + for_each = var.enable_ecs_endpoints && !var.lza_managed_network ? local.interface_endpoint_services : toset([]) + vpc_id = local.vpc_id service_name = "com.amazonaws.${var.AWS_REGION}.${each.value}" vpc_endpoint_type = "Interface" - subnet_ids = module.flip_vpc.private_subnets + subnet_ids = local.app_subnet_ids security_group_ids = [aws_security_group.vpc_endpoints[0].id] private_dns_enabled = true } From 65f052ba158402ddff94d8eb24d1d976863dd672 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 9 Jul 2026 13:28:12 +0100 Subject: [PATCH 03/41] feat(deploy/AWS): gate the SG-drift stack and public FL NLB off on LZA (#749) On the LZA account the org security baseline (Control Tower org trail, GuardDuty, Security Hub, Config) owns change monitoring, so FLIP's own CloudTrail->EventBridge->Lambda SG-drift plumbing is gated off there; and with no IGW + VPC Block Public Access an internet-facing NLB is impossible in-account, so the FL-server NLB, its target group, DNS record, SG rules and the ECS load_balancer wiring are gated off too (the fl-server/fl-api services themselves stay deployable) until the WP2 FL-inbound decision. Adds the LZA-only ECR pull-through execution-role grant (ghcr/ + ecr-public/ cache prefixes) and makes the Docker-Hub EFS-provision utility image env-overridable so LZA can pull it through the credential-less ecr-public cache. Legacy envs resolve to identical config; moved blocks cover every added count. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/Makefile | 8 ++ deploy/providers/AWS/ecs_efs_provision.tf | 7 +- deploy/providers/AWS/ecs_services.tf | 21 ++-- deploy/providers/AWS/ecs_sg.tf | 10 ++ deploy/providers/AWS/iam_ecs.tf | 33 ++++++ deploy/providers/AWS/main.tf | 44 ++++++-- deploy/providers/AWS/security.tf | 124 +++++++++++++++++++--- deploy/providers/AWS/variables.tf | 8 +- 8 files changed, 220 insertions(+), 35 deletions(-) diff --git a/deploy/providers/AWS/Makefile b/deploy/providers/AWS/Makefile index 58f35e51a..e76274d02 100644 --- a/deploy/providers/AWS/Makefile +++ b/deploy/providers/AWS/Makefile @@ -166,6 +166,14 @@ endif export TF_VAR_docker_image_tag=${DOCKER_TAG} export TF_VAR_flip_fl_image_tag=${DOCKER_FL_TAG} export TF_VAR_docker_registry=${DOCKER_REGISTRY} +# One-shot EFS-provision utility image (default amazon/aws-cli from Docker Hub +# in variables.tf). The LZA account has no internet egress, so .env.lza-prod +# points this at the in-account ECR Public pull-through cache mirror +# (…/ecr-public/aws-cli/aws-cli:, FLIP#749). Export only when set so +# legacy envs keep the variables.tf default rather than an empty string. +ifneq ($(EFS_PROVISION_IMAGE),) +export TF_VAR_efs_provision_image=${EFS_PROVISION_IMAGE} +endif export TF_VAR_fl_api_name=${DOCKER_FL_API_NAME} export TF_VAR_fl_server_name=${DOCKER_FL_SERVER_NAME} export TF_VAR_fl_client_name=${DOCKER_FL_CLIENT_NAME} diff --git a/deploy/providers/AWS/ecs_efs_provision.tf b/deploy/providers/AWS/ecs_efs_provision.tf index 7b69f0385..f186eebaa 100644 --- a/deploy/providers/AWS/ecs_efs_provision.tf +++ b/deploy/providers/AWS/ecs_efs_provision.tf @@ -103,8 +103,11 @@ resource "aws_ecs_task_definition" "efs_provision" { container_definitions = jsonencode([ { - name = "provision-efs-certs" - image = "amazon/aws-cli:2.22.35" + name = "provision-efs-certs" + # Default amazon/aws-cli:2.22.35 from Docker Hub; the egress-less LZA + # account overrides this to its ECR Public pull-through cache mirror via + # EFS_PROVISION_IMAGE in the env file (FLIP#749). + image = var.efs_provision_image # The amazon/aws-cli image's ENTRYPOINT is `aws`, so a command like # ["/bin/sh", "-c", ...] would get appended as args to aws and fail # with "Found invalid choice '/bin/sh'". Override entryPoint so the diff --git a/deploy/providers/AWS/ecs_services.tf b/deploy/providers/AWS/ecs_services.tf index fec59c801..a7e9c2bec 100644 --- a/deploy/providers/AWS/ecs_services.tf +++ b/deploy/providers/AWS/ecs_services.tf @@ -175,15 +175,22 @@ resource "aws_ecs_service" "fl_server_net_1" { # Register the running task's ENI IP with the NLB target group so FL # clients reaching fl..flip.aicentre.co.uk:8002 hit the Fargate task # over gRPC. NLB on TCP forwards the gRPC stream untouched. - load_balancer { - target_group_arn = aws_lb_target_group.ecs_fl_server_tcp.arn - container_name = "fl-server-net-1" - # Backend-dependent container port (Flower: SuperLink Fleet 9092); the - # NLB listener port trusts connect to stays var.FL_SERVER_PORT. - container_port = local.fl_server_container_port + # Skipped on LZA (FLIP#749): there is no NLB there yet β€” the service still + # deploys, it just has no inbound FL path until the WP2 ingress decision. + dynamic "load_balancer" { + for_each = var.lza_managed_network ? [] : [1] + content { + target_group_arn = aws_lb_target_group.ecs_fl_server_tcp[0].arn + container_name = "fl-server-net-1" + # Backend-dependent container port (Flower: SuperLink Fleet 9092); the + # NLB listener port trusts connect to stays var.FL_SERVER_PORT. + container_port = local.fl_server_container_port + } } - health_check_grace_period_seconds = 120 + # The grace period is only valid on services with a load balancer β€” ECS + # rejects it outright when the block above is skipped on LZA. + health_check_grace_period_seconds = var.lza_managed_network ? null : 120 deployment_minimum_healthy_percent = 100 deployment_maximum_percent = 200 diff --git a/deploy/providers/AWS/ecs_sg.tf b/deploy/providers/AWS/ecs_sg.tf index dc6230227..d49e7ab7d 100644 --- a/deploy/providers/AWS/ecs_sg.tf +++ b/deploy/providers/AWS/ecs_sg.tf @@ -109,7 +109,9 @@ resource "aws_security_group" "ecs_fl_server" { } } +# Gated off with the NLB on LZA (FLIP#749): no NLB security group to reference. resource "aws_security_group_rule" "ecs_fl_server_ingress_nlb_grpc" { + count = var.lza_managed_network ? 0 : 1 type = "ingress" description = "gRPC from NLB (FL client connections)" # Backend-dependent container port (Flower: SuperLink Fleet 9092) β€” the @@ -121,6 +123,14 @@ resource "aws_security_group_rule" "ecs_fl_server_ingress_nlb_grpc" { source_security_group_id = module.fl_server_nlb.security_group_id } +# State migration for the count added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_security_group_rule.ecs_fl_server_ingress_nlb_grpc + to = aws_security_group_rule.ecs_fl_server_ingress_nlb_grpc[0] +} + # fl-api (the NVFLARE admin client) connects directly to fl-server on the # admin port (same 8002 as the gRPC client port; NVFLARE multiplexes admin # and client RPC over a single HTTP/2 stream). It does NOT go through the diff --git a/deploy/providers/AWS/iam_ecs.tf b/deploy/providers/AWS/iam_ecs.tf index 55833b950..4c1a3db10 100644 --- a/deploy/providers/AWS/iam_ecs.tf +++ b/deploy/providers/AWS/iam_ecs.tf @@ -88,6 +88,39 @@ resource "aws_iam_role_policy" "ecs_task_execution_secrets" { policy = data.aws_iam_policy_document.ecs_task_execution_secrets.json } +# ECR pull-through cache import (FLIP#749, LZA only). On the LZA account the +# task images come from in-account pull-through caches instead of the public +# registries β€” the account has no internet egress. Two cache rules exist +# (created out-of-band, not managed here): `ghcr/` mirroring +# ghcr.io/londonaicentre via a read-only PAT in the +# `ecr-pullthroughcache/ghcr` secret, and a credential-less `ecr-public/` rule +# for the EFS-provision utility image. The first pull of a new tag triggers a +# service-side upstream import, which the pulling principal must be allowed to +# perform (plus repository creation on a cache miss); already-cached pulls need +# only the managed execution-role policy above. +data "aws_iam_policy_document" "ecs_task_execution_ecr_pull_through" { + count = var.lza_managed_network ? 1 : 0 + + statement { + sid = "EcrPullThroughCacheImport" + actions = [ + "ecr:BatchImportUpstreamImage", + "ecr:CreateRepository", + ] + resources = [ + "arn:aws:ecr:${var.AWS_REGION}:${data.aws_caller_identity.current.account_id}:repository/ghcr/*", + "arn:aws:ecr:${var.AWS_REGION}:${data.aws_caller_identity.current.account_id}:repository/ecr-public/*", + ] + } +} + +resource "aws_iam_role_policy" "ecs_task_execution_ecr_pull_through" { + count = var.lza_managed_network ? 1 : 0 + name = "flip-ecs-task-execution-ecr-pull-through" + role = aws_iam_role.ecs_task_execution.id + policy = data.aws_iam_policy_document.ecs_task_execution_ecr_pull_through[0].json +} + ############################ # flip-api task role ############################ diff --git a/deploy/providers/AWS/main.tf b/deploy/providers/AWS/main.tf index c2c1f6fc0..e9ee284ba 100644 --- a/deploy/providers/AWS/main.tf +++ b/deploy/providers/AWS/main.tf @@ -517,7 +517,13 @@ module "alb" { # Network Load Balancer for FL server TCP/TLS pass-through module "fl_server_nlb" { - source = "terraform-aws-modules/alb/aws" + source = "terraform-aws-modules/alb/aws" + # Not created on the LZA account (FLIP#749): no IGW + VPC Block Public Access + # make an internet-facing NLB impossible in-account, and the FL inbound + # architecture there (NLB in the central Ingress VPC vs FL-over-443 via the + # VPN) is an open WP2 decision β€” gate off rather than half-provision. Using + # the module's create flag keeps its state address stable for legacy envs. + create = !var.lza_managed_network name = "flip-fl-server-nlb" load_balancer_type = "network" vpc_id = local.vpc_id @@ -534,7 +540,9 @@ module "fl_server_nlb" { ip_protocol = "tcp" from_port = tostring(var.FL_SERVER_PORT) to_port = tostring(var.FL_SERVER_PORT) - cidr_ipv4 = "${module.flip_vpc.nat_public_ips[0]}/32" + # Guarded because module arguments are evaluated even with create = + # false: on LZA the VPC module is empty, so there is no NAT EIP to index. + cidr_ipv4 = var.lza_managed_network ? null : "${module.flip_vpc.nat_public_ips[0]}/32" } } @@ -546,7 +554,7 @@ module "fl_server_nlb" { # the listener/ingress side which stays on FL_SERVER_PORT. from_port = tostring(local.fl_server_container_port) to_port = tostring(local.fl_server_container_port) - cidr_ipv4 = var.vpc_cidr + cidr_ipv4 = local.vpc_cidr_block } } @@ -559,7 +567,8 @@ module "fl_server_nlb" { port = var.FL_SERVER_PORT protocol = "TCP" forward = { - target_group_arn = aws_lb_target_group.ecs_fl_server_tcp.arn + # Guarded like the ingress rule above: the TG is count-gated on LZA. + target_group_arn = var.lza_managed_network ? null : aws_lb_target_group.ecs_fl_server_tcp[0].arn } } } @@ -595,7 +604,9 @@ resource "aws_route53_record" "alb" { # attach instance/IP targets here. target_type=ip is required for awsvpc # Fargate tasks. NLB protocol must be TCP - HTTP/2 gRPC framing is opaque # to the NLB and forwarded as-is. +# Gated off with the NLB on LZA (FLIP#749): a TG with no LB is dead config. resource "aws_lb_target_group" "ecs_fl_server_tcp" { + count = var.lza_managed_network ? 0 : 1 # Container port per backend (NVFLARE: FL_SERVER_PORT; Flower: SuperLink # Fleet 9092 β€” see local.fl_server_container_port in ecs_tasks.tf). The # NLB LISTENER stays on var.FL_SERVER_PORT for both. Port is ForceNew, so @@ -626,7 +637,9 @@ resource "aws_lb_target_group" "ecs_fl_server_tcp" { deregistration_delay = 30 } +# Gated off with the NLB on LZA (FLIP#749). resource "aws_route53_record" "fl_server_nlb" { + count = var.lza_managed_network ? 0 : 1 zone_id = data.aws_route53_zone.subdomain.zone_id name = var.flip_nlb_subdomain type = "A" @@ -638,6 +651,19 @@ resource "aws_route53_record" "fl_server_nlb" { } } +# State migration for the counts added to the NLB stack (FLIP#749): keeps +# existing legacy states aligned without a manual `terraform state mv`. Safe to +# remove once every live state file has been migrated. +moved { + from = aws_lb_target_group.ecs_fl_server_tcp + to = aws_lb_target_group.ecs_fl_server_tcp[0] +} + +moved { + from = aws_route53_record.fl_server_nlb + to = aws_route53_record.fl_server_nlb[0] +} + # Target group for the flip-api ECS Fargate service. Registered by the ECS # service itself via the load_balancer block in ecs_services.tf - we never # attach instance/IP targets here from terraform. target_type=ip is required @@ -694,8 +720,9 @@ resource "aws_lb_listener_rule" "api_routing" { # Allow on-prem trust FL clients to reach the FL server via the NLB. # Without this rule the NLB security group drops the connection before it reaches the EC2. +# Emptied on LZA (FLIP#749): there is no NLB (or NLB security group) to attach to. resource "aws_security_group_rule" "local_trust_fl_server_nlb" { - for_each = toset(var.local_trust_public_ips) + for_each = toset(var.lza_managed_network ? [] : var.local_trust_public_ips) type = "ingress" from_port = var.FL_SERVER_PORT to_port = var.FL_SERVER_PORT @@ -719,7 +746,8 @@ resource "aws_security_group_rule" "local_trust_fl_server_nlb" { resource "aws_security_group_rule" "k8s_trust_fl_server_nlb" { # The deprecated scalar is marked sensitive for backwards compatibility, # but an address used as a resource key is necessarily disclosed in state. - for_each = toset(concat( + # Emptied on LZA (FLIP#749) like the on-prem rule above: no NLB there. + for_each = toset(var.lza_managed_network ? [] : concat( var.k8s_trust_public_ips, nonsensitive(var.K8S_TRUST_IP) != "" ? [nonsensitive(var.K8S_TRUST_IP)] : [] )) @@ -783,12 +811,12 @@ output "CognitoAppClientId" { } output "FlServerEndpoint" { - description = "FL server DNS endpoint (NLB pass-through)" + description = "FL server DNS endpoint (NLB pass-through; on LZA there is no NLB yet β€” FL inbound is a FLIP#749 WP2 decision)" value = var.flip_nlb_subdomain } output "FlServerRawNlbDns" { - description = "Raw AWS NLB DNS name for FL server debugging" + description = "Raw AWS NLB DNS name for FL server debugging (null on LZA β€” FLIP#749)" value = module.fl_server_nlb.dns_name } diff --git a/deploy/providers/AWS/security.tf b/deploy/providers/AWS/security.tf index 05af45a14..a5744f7d9 100644 --- a/deploy/providers/AWS/security.tf +++ b/deploy/providers/AWS/security.tf @@ -20,17 +20,26 @@ # DeleteSecurityGroup, ModifySecurityGroupRules, # UpdateSecurityGroupRuleDescriptions{Ingress,Egress}, CreateSecurityGroup, # and CreateTags/DeleteTags on SG resources (mitigates tag-evasion). +# +# The whole stack is gated off on the LZA account (FLIP#749): the org security +# baseline there (Control Tower org trail, GuardDuty, Security Hub, AWS Config) +# already owns change monitoring, so FLIP's own CloudTrailβ†’EventBridgeβ†’Lambda +# plumbing would be redundant noise alongside it. ############################ # SNS Topic for SG drift alerts ############################ resource "aws_sns_topic" "sg_drift" { + count = var.lza_managed_network ? 0 : 1 + name = "flip-sg-drift" } resource "aws_sns_topic_subscription" "sg_drift_email" { - topic_arn = aws_sns_topic.sg_drift.arn + count = var.lza_managed_network ? 0 : 1 + + topic_arn = aws_sns_topic.sg_drift[0].arn protocol = "email" endpoint = var.SES_VERIFIED_EMAIL } @@ -40,6 +49,8 @@ resource "aws_sns_topic_subscription" "sg_drift_email" { ############################ resource "aws_sqs_queue" "sg_drift_dlq" { + count = var.lza_managed_network ? 0 : 1 + name = "flip-sg-drift-dlq" message_retention_seconds = 1209600 # 14 days visibility_timeout_seconds = 30 @@ -50,6 +61,8 @@ resource "aws_sqs_queue" "sg_drift_dlq" { ############################ data "archive_file" "sg_drift_lambda" { + count = var.lza_managed_network ? 0 : 1 + type = "zip" output_path = "${path.module}/.terraform-build/sg_drift_lambda.zip" @@ -164,23 +177,25 @@ PYTHON } resource "aws_lambda_function" "sg_drift_filter" { - filename = data.archive_file.sg_drift_lambda.output_path + count = var.lza_managed_network ? 0 : 1 + + filename = data.archive_file.sg_drift_lambda[0].output_path function_name = "flip-sg-drift-filter" - role = aws_iam_role.sg_drift_lambda_role.arn + role = aws_iam_role.sg_drift_lambda_role[0].arn handler = "lambda_function.handler" runtime = "python3.12" architectures = ["arm64"] - source_code_hash = data.archive_file.sg_drift_lambda.output_base64sha256 + source_code_hash = data.archive_file.sg_drift_lambda[0].output_base64sha256 timeout = 10 memory_size = 128 dead_letter_config { - target_arn = aws_sqs_queue.sg_drift_dlq.arn + target_arn = aws_sqs_queue.sg_drift_dlq[0].arn } environment { variables = { - SNS_TOPIC_ARN = aws_sns_topic.sg_drift.arn + SNS_TOPIC_ARN = aws_sns_topic.sg_drift[0].arn } } } @@ -190,11 +205,15 @@ resource "aws_lambda_function" "sg_drift_filter" { ############################ resource "aws_iam_role" "sg_drift_lambda_role" { + count = var.lza_managed_network ? 0 : 1 + name = "flip-sg-drift-lambda-role" - assume_role_policy = data.aws_iam_policy_document.lambda_assume.json + assume_role_policy = data.aws_iam_policy_document.lambda_assume[0].json } data "aws_iam_policy_document" "lambda_assume" { + count = var.lza_managed_network ? 0 : 1 + statement { actions = ["sts:AssumeRole"] principals { @@ -205,6 +224,8 @@ data "aws_iam_policy_document" "lambda_assume" { } data "aws_iam_policy_document" "sg_drift_lambda_policy" { + count = var.lza_managed_network ? 0 : 1 + statement { sid = "DescribeSecurityGroups" actions = ["ec2:DescribeSecurityGroups"] @@ -214,13 +235,13 @@ data "aws_iam_policy_document" "sg_drift_lambda_policy" { statement { sid = "PublishSns" actions = ["sns:Publish"] - resources = [aws_sns_topic.sg_drift.arn] + resources = [aws_sns_topic.sg_drift[0].arn] } statement { sid = "SendToDlq" actions = ["sqs:SendMessage"] - resources = [aws_sqs_queue.sg_drift_dlq.arn] + resources = [aws_sqs_queue.sg_drift_dlq[0].arn] } # Basic Lambda logging β€” scoped to this function's log group. @@ -238,9 +259,11 @@ data "aws_iam_policy_document" "sg_drift_lambda_policy" { } resource "aws_iam_role_policy" "sg_drift_lambda_policy" { + count = var.lza_managed_network ? 0 : 1 + name = "flip-sg-drift-lambda-policy" - role = aws_iam_role.sg_drift_lambda_role.id - policy = data.aws_iam_policy_document.sg_drift_lambda_policy.json + role = aws_iam_role.sg_drift_lambda_role[0].id + policy = data.aws_iam_policy_document.sg_drift_lambda_policy[0].json } ############################ @@ -248,6 +271,8 @@ resource "aws_iam_role_policy" "sg_drift_lambda_policy" { ############################ resource "aws_cloudwatch_event_rule" "sg_drift" { + count = var.lza_managed_network ? 0 : 1 + name = "flip-sg-drift" description = "Capture SG modification events for FlipSG-tagged security groups" @@ -276,17 +301,21 @@ resource "aws_cloudwatch_event_rule" "sg_drift" { } resource "aws_cloudwatch_event_target" "sg_drift" { - rule = aws_cloudwatch_event_rule.sg_drift.name + count = var.lza_managed_network ? 0 : 1 + + rule = aws_cloudwatch_event_rule.sg_drift[0].name target_id = "sg-drift-filter-lambda" - arn = aws_lambda_function.sg_drift_filter.arn + arn = aws_lambda_function.sg_drift_filter[0].arn } resource "aws_lambda_permission" "sg_drift_allow_eventbridge" { + count = var.lza_managed_network ? 0 : 1 + statement_id = "AllowExecutionFromEventBridge" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.sg_drift_filter.function_name + function_name = aws_lambda_function.sg_drift_filter[0].function_name principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.sg_drift.arn + source_arn = aws_cloudwatch_event_rule.sg_drift[0].arn } ############################ @@ -294,6 +323,8 @@ resource "aws_lambda_permission" "sg_drift_allow_eventbridge" { ############################ resource "aws_cloudwatch_metric_alarm" "sg_drift_dlq_depth" { + count = var.lza_managed_network ? 0 : 1 + alarm_name = "flip-sg-drift-dlq-depth" comparison_operator = "GreaterThanThreshold" evaluation_periods = "1" @@ -303,8 +334,67 @@ resource "aws_cloudwatch_metric_alarm" "sg_drift_dlq_depth" { statistic = "Sum" threshold = "0" alarm_description = "SG drift DLQ has undelivered events β€” Lambda failed after EventBridge retry window" - alarm_actions = [aws_sns_topic.sg_drift.arn] + alarm_actions = [aws_sns_topic.sg_drift[0].arn] dimensions = { - QueueName = aws_sqs_queue.sg_drift_dlq.name + QueueName = aws_sqs_queue.sg_drift_dlq[0].name } } + +############################ +# State migration (FLIP#749) +############################ +# +# The counts added above change every address from to [0]; these +# moved blocks keep existing legacy states aligned without a manual +# `terraform state mv`. Safe to remove once every live state file has been +# migrated. + +moved { + from = aws_sns_topic.sg_drift + to = aws_sns_topic.sg_drift[0] +} + +moved { + from = aws_sns_topic_subscription.sg_drift_email + to = aws_sns_topic_subscription.sg_drift_email[0] +} + +moved { + from = aws_sqs_queue.sg_drift_dlq + to = aws_sqs_queue.sg_drift_dlq[0] +} + +moved { + from = aws_lambda_function.sg_drift_filter + to = aws_lambda_function.sg_drift_filter[0] +} + +moved { + from = aws_iam_role.sg_drift_lambda_role + to = aws_iam_role.sg_drift_lambda_role[0] +} + +moved { + from = aws_iam_role_policy.sg_drift_lambda_policy + to = aws_iam_role_policy.sg_drift_lambda_policy[0] +} + +moved { + from = aws_cloudwatch_event_rule.sg_drift + to = aws_cloudwatch_event_rule.sg_drift[0] +} + +moved { + from = aws_cloudwatch_event_target.sg_drift + to = aws_cloudwatch_event_target.sg_drift[0] +} + +moved { + from = aws_lambda_permission.sg_drift_allow_eventbridge + to = aws_lambda_permission.sg_drift_allow_eventbridge[0] +} + +moved { + from = aws_cloudwatch_metric_alarm.sg_drift_dlq_depth + to = aws_cloudwatch_metric_alarm.sg_drift_dlq_depth[0] +} diff --git a/deploy/providers/AWS/variables.tf b/deploy/providers/AWS/variables.tf index a339a0945..684358a2b 100644 --- a/deploy/providers/AWS/variables.tf +++ b/deploy/providers/AWS/variables.tf @@ -143,11 +143,17 @@ variable "flip_fl_image_tag" { } variable "docker_registry" { - description = "Docker image registry prefix (e.g. ghcr.io/londonaicentre/)" + description = "Docker image registry prefix (e.g. ghcr.io/londonaicentre/). On the LZA account point it at the ECR pull-through cache mirror (.dkr.ecr..amazonaws.com/ghcr/londonaicentre/) β€” there is no internet egress to reach GHCR directly (FLIP#749)." type = string default = "ghcr.io/londonaicentre/" } +variable "efs_provision_image" { + description = "Image for the one-shot EFS provisioning task (ecs_efs_provision.tf). Default is the Docker-Hub-hosted amazon/aws-cli, unchanged for legacy envs; on the egress-less LZA account point it at the credential-less ECR Public pull-through cache mirror (.dkr.ecr..amazonaws.com/ecr-public/aws-cli/aws-cli:) via EFS_PROVISION_IMAGE in the env file (FLIP#749)." + type = string + default = "amazon/aws-cli:2.22.35" +} + variable "fl_api_name" { description = "FL API Docker image name (backend-specific: flare-fl-api or flower-fl-api)" type = string From 84a3b48f778c81786b08709092a44f79e18e5843 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 9 Jul 2026 13:34:11 +0100 Subject: [PATCH 04/41] feat(deploy/AWS): support a DNS-less first bring-up on the default CloudFront domain (#749) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New manage_dns toggle (MANAGE_DNS in the env file; unset => true so legacy prod/stag are untouched). false β€” the zone-less first LZA bring-up, before the platform-side DNS migration lands β€” skips the Route53 zone lookup, every record, and both DNS-validated ACM certs: CloudFront serves *.cloudfront.net on the default viewer cert (no aliases), the CF->ALB VPC-origin leg falls back to plain HTTP on the same listener (an ALB HTTPS listener needs an ISSUED cert, which needs DNS validation), and the bucket CORS + Cognito URLs follow the CloudFront default domain via local.ui_origin so uploads, downloads and sign-in work before any DNS exists. Listener keys and the /api/* rule attachment are unchanged in both modes to keep state stable; moved blocks cover the counts added to certs and records. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/Makefile | 7 +++ deploy/providers/AWS/certificate.tf | 31 +++++++++++--- deploy/providers/AWS/cloudfront.tf | 66 +++++++++++++++++++++++------ deploy/providers/AWS/main.tf | 41 ++++++++++++++---- deploy/providers/AWS/services.tf | 26 +++++++----- deploy/providers/AWS/variables.tf | 6 +++ 6 files changed, 141 insertions(+), 36 deletions(-) diff --git a/deploy/providers/AWS/Makefile b/deploy/providers/AWS/Makefile index e76274d02..1da00e706 100644 --- a/deploy/providers/AWS/Makefile +++ b/deploy/providers/AWS/Makefile @@ -203,6 +203,13 @@ DEPLOY_TRUST_EC2 ?= true export TF_VAR_deploy_trust_ec2=$(DEPLOY_TRUST_EC2) export TF_VAR_AWS_REGION=${AWS_REGION} export TF_VAR_VPC_NAME=${VPC_NAME} +# DNS availability (FLIP#749): the LZA account has no Route53 hosted zone until +# the platform-side DNS migration lands, so .env.lza-prod sets MANAGE_DNS=false +# for the first bring-up (default CloudFront domain, no custom certs/records) and +# flips it to true once the zone exists there. Unset (legacy prod/stag) β‡’ true β€” +# behaviour unchanged. +MANAGE_DNS ?= true +export TF_VAR_manage_dns=$(MANAGE_DNS) export TF_VAR_ADMIN_USER_PASSWORD=${ADMIN_USER_PASSWORD} export TF_VAR_SES_VERIFIED_EMAIL=${SES_VERIFIED_EMAIL} export TF_VAR_flip_alb_subdomain=${ALB_SUBDOMAIN} diff --git a/deploy/providers/AWS/certificate.tf b/deploy/providers/AWS/certificate.tf index 53b3ac419..7f25b8289 100644 --- a/deploy/providers/AWS/certificate.tf +++ b/deploy/providers/AWS/certificate.tf @@ -26,9 +26,16 @@ # This is because the domain_validation_options are only known after # the certificate is created, so Terraform cannot plan the DNS records # in a single pass. +# +# The whole chain is skipped when var.manage_dns is false (the zone-less +# first LZA bring-up, FLIP#749): DNS validation is impossible without the +# hosted zone, and an unvalidated cert cannot be attached to the ALB β€” which +# serves plain HTTP on the private CloudFront-VPC-origin leg instead (see the +# listeners comment in main.tf). ############################ resource "aws_acm_certificate" "flip" { + count = var.manage_dns ? 1 : 0 domain_name = var.flip_alb_subdomain validation_method = "DNS" @@ -44,7 +51,7 @@ resource "aws_acm_certificate" "flip" { # DNS validation record resource "aws_route53_record" "cert_validation" { for_each = { - for dvo in tolist(aws_acm_certificate.flip.domain_validation_options) : dvo.domain_name => { + for dvo in var.manage_dns ? tolist(aws_acm_certificate.flip[0].domain_validation_options) : [] : dvo.domain_name => { name = dvo.resource_record_name record = dvo.resource_record_value type = dvo.resource_record_type @@ -56,16 +63,30 @@ resource "aws_route53_record" "cert_validation" { records = [each.value.record] ttl = 60 type = each.value.type - zone_id = data.aws_route53_zone.subdomain.zone_id + zone_id = data.aws_route53_zone.subdomain[0].zone_id } # Certificate validation resource "aws_acm_certificate_validation" "flip" { - certificate_arn = aws_acm_certificate.flip.arn + count = var.manage_dns ? 1 : 0 + certificate_arn = aws_acm_certificate.flip[0].arn validation_record_fqdns = [for record in aws_route53_record.cert_validation : record.fqdn] } output "CertificateArn" { - description = "ACM Certificate ARN (validated)" - value = aws_acm_certificate_validation.flip.certificate_arn + description = "ACM Certificate ARN (validated; null when manage_dns is false β€” FLIP#749)" + value = var.manage_dns ? aws_acm_certificate_validation.flip[0].certificate_arn : null +} + +# State migration for the counts added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_acm_certificate.flip + to = aws_acm_certificate.flip[0] +} + +moved { + from = aws_acm_certificate_validation.flip + to = aws_acm_certificate_validation.flip[0] } diff --git a/deploy/providers/AWS/cloudfront.tf b/deploy/providers/AWS/cloudfront.tf index 69df45c43..32810902e 100644 --- a/deploy/providers/AWS/cloudfront.tf +++ b/deploy/providers/AWS/cloudfront.tf @@ -45,8 +45,14 @@ provider "aws" { ############################ # CloudFront viewer cert (us-east-1) ############################ +# +# Skipped when var.manage_dns is false (the zone-less first LZA bring-up, +# FLIP#749): the viewer cert can't be DNS-validated without the zone, and a +# custom viewer cert is pointless anyway while the distribution has no aliases +# β€” it serves the default *.cloudfront.net domain with the default cert. resource "aws_acm_certificate" "flip_cloudfront" { + count = var.manage_dns ? 1 : 0 provider = aws.us_east_1 domain_name = var.flip_alb_subdomain validation_method = "DNS" @@ -62,7 +68,7 @@ resource "aws_acm_certificate" "flip_cloudfront" { resource "aws_route53_record" "cloudfront_cert_validation" { for_each = { - for dvo in tolist(aws_acm_certificate.flip_cloudfront.domain_validation_options) : dvo.domain_name => { + for dvo in var.manage_dns ? tolist(aws_acm_certificate.flip_cloudfront[0].domain_validation_options) : [] : dvo.domain_name => { name = dvo.resource_record_name record = dvo.resource_record_value type = dvo.resource_record_type @@ -74,15 +80,29 @@ resource "aws_route53_record" "cloudfront_cert_validation" { records = [each.value.record] ttl = 60 type = each.value.type - zone_id = data.aws_route53_zone.subdomain.zone_id + zone_id = data.aws_route53_zone.subdomain[0].zone_id } resource "aws_acm_certificate_validation" "flip_cloudfront" { + count = var.manage_dns ? 1 : 0 provider = aws.us_east_1 - certificate_arn = aws_acm_certificate.flip_cloudfront.arn + certificate_arn = aws_acm_certificate.flip_cloudfront[0].arn validation_record_fqdns = [for record in aws_route53_record.cloudfront_cert_validation : record.fqdn] } +# State migration for the counts added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_acm_certificate.flip_cloudfront + to = aws_acm_certificate.flip_cloudfront[0] +} + +moved { + from = aws_acm_certificate_validation.flip_cloudfront + to = aws_acm_certificate_validation.flip_cloudfront[0] +} + ############################ # CloudFront VPC origin for the (internal) ALB # @@ -97,11 +117,16 @@ resource "aws_cloudfront_vpc_origin" "flip_api" { vpc_origin_endpoint_config { # CloudFront VPC origin names accept only alphanumerics, dashes, and # underscores β€” the subdomain contains dots, so replace them with dashes. - name = "flip-api-vpc-origin-${replace(var.flip_alb_subdomain, ".", "-")}" - arn = module.alb.arn - http_port = 80 + name = "flip-api-vpc-origin-${replace(var.flip_alb_subdomain, ".", "-")}" + arn = module.alb.arn + # Without a hosted zone the ALB cannot carry an ISSUED cert, so the private + # VPC-origin leg falls back to plain HTTP until DNS lands (FLIP#749; see the + # ALB listeners comment in main.tf). The ALB's main listener then serves + # plain HTTP on ALB_HTTPS_PORT, so the HTTP port follows it there. Viewer + # traffic stays HTTPS either way. + http_port = var.manage_dns ? 80 : var.ALB_HTTPS_PORT https_port = 443 - origin_protocol_policy = "https-only" + origin_protocol_policy = var.manage_dns ? "https-only" : "http-only" origin_ssl_protocols { items = ["TLSv1.2"] @@ -434,6 +459,14 @@ resource "aws_cloudfront_function" "spa_rewrite" { locals { cloudfront_policy_caching_optimized = "658327ea-f89d-4fab-a63d-7e88639e58f6" cloudfront_policy_caching_disabled = "4135ea2d-6df8-44a3-9df3-4b5a84be39ad" + + # Browser-facing origin of the UI, consumed by the S3 bucket CORS rules and + # the Cognito URLs in services.tf. With DNS managed it is the canonical + # subdomain (legacy shape, unchanged); on the zone-less first bring-up + # (FLIP#749) it is the CloudFront default domain, so uploads/downloads and + # sign-in keep working before any DNS exists. No dependency cycle: the + # distribution references neither the app buckets' CORS nor Cognito. + ui_origin = var.manage_dns ? "https://${var.flip_alb_subdomain}" : "https://${aws_cloudfront_distribution.flip_ui.domain_name}" } # Custom origin-request policy for /api/*. The managed AllViewer policy @@ -877,9 +910,13 @@ resource "aws_cloudfront_distribution" "flip_ui" { http_version = "http2" default_root_object = "index.html" price_class = "PriceClass_100" - aliases = [var.flip_alb_subdomain] - comment = "flip-ui at ${var.flip_alb_subdomain}" - web_acl_id = aws_wafv2_web_acl.flip_ui_cloudfront.arn + # No aliases without DNS (FLIP#749): CloudFront only allows the default + # viewer cert when no aliases are set, and an alias without a record pointing + # at it is unreachable anyway. The distribution serves *.cloudfront.net until + # MANAGE_DNS flips to true. + aliases = var.manage_dns ? [var.flip_alb_subdomain] : [] + comment = "flip-ui at ${var.flip_alb_subdomain}" + web_acl_id = aws_wafv2_web_acl.flip_ui_cloudfront.arn origin { domain_name = aws_s3_bucket.flip_ui.bucket_regional_domain_name @@ -1002,9 +1039,12 @@ resource "aws_cloudfront_distribution" "flip_ui" { } viewer_certificate { - acm_certificate_arn = aws_acm_certificate_validation.flip_cloudfront.certificate_arn - ssl_support_method = "sni-only" - minimum_protocol_version = "TLSv1.2_2021" + acm_certificate_arn = var.manage_dns ? aws_acm_certificate_validation.flip_cloudfront[0].certificate_arn : null + cloudfront_default_certificate = var.manage_dns ? null : true + ssl_support_method = var.manage_dns ? "sni-only" : null + # AWS forces TLSv1 while the default *.cloudfront.net certificate is in + # use; pinning TLSv1.2_2021 there would just plan perpetual drift. + minimum_protocol_version = var.manage_dns ? "TLSv1.2_2021" : "TLSv1" } tags = { diff --git a/deploy/providers/AWS/main.tf b/deploy/providers/AWS/main.tf index e9ee284ba..30af50372 100644 --- a/deploy/providers/AWS/main.tf +++ b/deploy/providers/AWS/main.tf @@ -480,6 +480,15 @@ module "alb" { security_groups = [module.alb_security_group.security_group.id] enable_deletion_protection = false + # The main listener's protocol depends on DNS availability (FLIP#749): with a + # hosted zone it terminates HTTPS with the DNS-validated ACM cert β€” the + # canonical shape, unchanged for legacy prod/stag. Without one + # (var.manage_dns = false, the zone-less first LZA bring-up) an ISSUED cert + # is impossible, so the same listener (key kept for state stability) serves + # plain HTTP on the same port; that leg only ever carries + # CloudFront-VPC-origin traffic over an AWS-managed ENI inside the VPC + # (viewers still get HTTPS on the default CloudFront domain), and it reverts + # to HTTPS as soon as the zone lands and MANAGE_DNS flips to true. listeners = { # HTTPS default action: return 404. CloudFront is the canonical front door # for user traffic; anything reaching the ALB default action (e.g. direct @@ -488,15 +497,19 @@ module "alb" { # /api/* behaviour and any direct trust access. "https-listener" = { port = var.ALB_HTTPS_PORT - protocol = "HTTPS" - certificate_arn = aws_acm_certificate.flip.arn - ssl_policy = "ELBSecurityPolicy-TLS13-1-3-2021-06" + protocol = var.manage_dns ? "HTTPS" : "HTTP" + certificate_arn = var.manage_dns ? aws_acm_certificate.flip[0].arn : null + ssl_policy = var.manage_dns ? "ELBSecurityPolicy-TLS13-1-3-2021-06" : null fixed_response = { content_type = "text/plain" message_body = "Not Found" status_code = "404" } }, + # On the zone-less bring-up this redirect points at what is temporarily a + # plain-HTTP listener β€” dead config there, but nothing dials port 80 (the + # VPC origin dials ALB_HTTPS_PORT and the ALB is internal), and keeping the + # key avoids a state churn on the flip back to HTTPS. "http-redirect" = { port = var.ALB_HTTP_PORT protocol = "HTTP" @@ -580,12 +593,16 @@ module "fl_server_nlb" { target_groups = {} } +# Skipped when the account has no hosted zone (var.manage_dns = false β€” the +# zone-less first LZA bring-up, FLIP#749): the lookup would hard-fail there. data "aws_route53_zone" "subdomain" { - name = var.flip_alb_subdomain + count = var.manage_dns ? 1 : 0 + name = var.flip_alb_subdomain } resource "aws_route53_record" "alb" { - zone_id = data.aws_route53_zone.subdomain.zone_id + count = var.manage_dns ? 1 : 0 + zone_id = data.aws_route53_zone.subdomain[0].zone_id name = var.flip_alb_subdomain type = "A" @@ -599,6 +616,14 @@ resource "aws_route53_record" "alb" { } } +# State migration for the count added above (FLIP#749): keeps existing legacy +# states aligned without a manual `terraform state mv`. Safe to remove once +# every live state file has been migrated. +moved { + from = aws_route53_record.alb + to = aws_route53_record.alb[0] +} + # Target group for the fl-server-net-1 ECS Fargate service. Registered by # the ECS service via the load_balancer block in ecs_services.tf - we never # attach instance/IP targets here. target_type=ip is required for awsvpc @@ -637,10 +662,10 @@ resource "aws_lb_target_group" "ecs_fl_server_tcp" { deregistration_delay = 30 } -# Gated off with the NLB on LZA (FLIP#749). +# Gated off with the NLB on LZA, and with the zone when DNS is unmanaged (FLIP#749). resource "aws_route53_record" "fl_server_nlb" { - count = var.lza_managed_network ? 0 : 1 - zone_id = data.aws_route53_zone.subdomain.zone_id + count = var.manage_dns && !var.lza_managed_network ? 1 : 0 + zone_id = data.aws_route53_zone.subdomain[0].zone_id name = var.flip_nlb_subdomain type = "A" diff --git a/deploy/providers/AWS/services.tf b/deploy/providers/AWS/services.tf index b745893ca..bb136f01c 100644 --- a/deploy/providers/AWS/services.tf +++ b/deploy/providers/AWS/services.tf @@ -52,8 +52,10 @@ module "flip_model_files_uploads_bucket" { # GET is for model-file downloads: `download_file.py` hands the browser a # presigned GET URL (FLIP#784) instead of proxying bytes through flip-api, # so the bucket needs to accept a direct browser GET. - cors_methods = ["POST", "GET"] - cors_allowed_origins = ["https://${var.flip_alb_subdomain}"] + cors_methods = ["POST", "GET"] + # local.ui_origin (cloudfront.tf) β€” the canonical subdomain, or the default + # CloudFront domain on a zone-less bring-up (FLIP#749). + cors_allowed_origins = [local.ui_origin] kms_key_arn = aws_kms_key.flip_app_key.arn logging_target_bucket = local.access_logs_bucket_name mfa_delete_protection = true @@ -74,7 +76,7 @@ module "flip_fl_results_bucket" { source = "./modules/flip_s3_bucket" bucket_name = var.FLIP_FL_RESULTS_BUCKET_NAME cors_methods = ["GET"] - cors_allowed_origins = ["https://${var.flip_alb_subdomain}"] + cors_allowed_origins = [local.ui_origin] kms_key_arn = aws_kms_key.flip_app_key.arn logging_target_bucket = local.access_logs_bucket_name mfa_delete_protection = true @@ -135,7 +137,7 @@ resource "aws_s3_bucket_cors_configuration" "aicentre_bucket_cors" { cors_rule { allowed_headers = ["*"] allowed_methods = ["PUT", "GET"] - allowed_origins = ["https://${var.flip_alb_subdomain}"] + allowed_origins = [local.ui_origin] expose_headers = [] } } @@ -191,18 +193,22 @@ resource "aws_s3_bucket_logging" "aicentre_bucket" { module "cognito" { source = "./modules/cognito" - user_pool_name = var.flip_user_pool_name - client_name = var.flip_cognito_client - sign_in_hostname = var.flip_alb_subdomain + user_pool_name = var.flip_user_pool_name + client_name = var.flip_cognito_client + # Hostname stamped into invite emails; derived from local.ui_origin + # (cloudfront.tf) so a zone-less bring-up (FLIP#749) sends the reachable + # CloudFront default domain rather than a dead subdomain. Identical to + # var.flip_alb_subdomain whenever DNS is managed (all legacy envs). + sign_in_hostname = trimprefix(local.ui_origin, "https://") admin_email = var.flip_cognito_admin_email researcher_email = var.flip_cognito_researcher_email seed_user_password = var.ADMIN_USER_PASSWORD templates_dir = "${path.module}/templates/cognito" # The UI uses USER_SRP_AUTH (Cognito's native auth flow, not OAuth2 redirect), - # so callback_urls are hygiene only β€” keep the canonical subdomain + localhost + # so callback_urls are hygiene only β€” keep the canonical UI origin + localhost # for dev. - callback_urls = ["https://${var.flip_alb_subdomain}", "https://localhost:443"] - logout_urls = ["https://${var.flip_alb_subdomain}", "https://localhost:443"] + callback_urls = [local.ui_origin, "https://localhost:443"] + logout_urls = [local.ui_origin, "https://localhost:443"] } # State migration: Cognito resources used to live at the root of this stack and diff --git a/deploy/providers/AWS/variables.tf b/deploy/providers/AWS/variables.tf index 684358a2b..a4432832f 100644 --- a/deploy/providers/AWS/variables.tf +++ b/deploy/providers/AWS/variables.tf @@ -347,6 +347,12 @@ variable "FL_SERVER_PORT" { } +variable "manage_dns" { + description = "Whether this account hosts the Route53 zone for flip_alb_subdomain. false (first LZA bring-up, before the zone moves in the platform DNS migration β€” FLIP#749) skips the zone lookup, every Route53 record, and both DNS-validated ACM certs: CloudFront then serves on its default *.cloudfront.net domain with the default viewer certificate (allowed only when no aliases are set), and the CloudFrontβ†’ALB origin leg falls back to plain HTTP over the private VPC-origin ENI, because an ALB HTTPS listener needs an ISSUED certificate and issuance needs DNS validation. Legacy prod/stag keep the default true." + type = bool + default = true +} + variable "flip_alb_subdomain" { description = "Public canonical subdomain for FLIP. Aliased via Route53 to the CloudFront distribution; CloudFront fronts both the SPA (from S3) and the API (/api/* -> ALB). Name is retained for Terraform-state backwards compatibility - see main.tf:492-494." type = string From c2d2a9790527aebc348ac2dfe6293731f4973a40 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 9 Jul 2026 13:38:44 +0100 Subject: [PATCH 05/41] docs(deploy/AWS): document the PROD=lza deployment path (#749) New README section "Deploying to the LZA account (FLIPProduction)": prerequisites (SSO profile, out-of-band state bucket + ECR pull-through cache rules), what PROD=lza selects, platform-managed vs FLIP-managed split, the .env.lza-prod value deltas (flip-lza-* buckets, pull-through registry, MANAGE_DNS), the DNS-less first bring-up on the default CloudFront domain with its limitations, and what is not yet exercised on LZA. CLAUDE.md + AGENTS.md updated in lockstep (profile row, network_lza.tf table entry, deploy-commands pointer); root CLAUDE.md PROD values extended. Signed-off-by: at24_bioeng625-pc --- AGENTS.md | 2 +- CLAUDE.md | 2 +- deploy/providers/AWS/AGENTS.md | 3 + deploy/providers/AWS/CLAUDE.md | 3 + deploy/providers/AWS/README.md | 122 ++++++++++++++++++++++++++++++++- 5 files changed, 128 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index be65f2a0a..c57e35a95 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -368,7 +368,7 @@ After changes, evaluate if docs need updating: - `FL_PROVISIONED_DIR` β€” path to the in-tree provisioned FL artifacts, derived per-backend by `deploy/fl_backend.mk` from `FL_BACKEND`: `fl-services/nvflare/provision/workspace-dev` (nvflare startup kits) or `fl-services/flower/provision/creds` (flower per-net TLS certs + SuperNode keys). Both gitignored. Read only by the dev compose overlays for the cert/workspace volume mounts; override at the CLI for a one-off (`make up FL_PROVISIONED_DIR=...`). FL Makefiles are **per-backend** β€” each `fl-services//Makefile` owns that backend's `build`/`provision`/`up`/`down`/`submit` (flower also `up-secure`); the root Makefile forwards only `build-fl` by `FL_BACKEND`. Each backend's `fl-services//Makefile` also owns its network provisioning (NVFLARE adds `provision`/`provision-2-nets`/`provision-stag`/`provision-prod`/`upload-kits-to-s3`; the project YAMLs, `scripts/`, and gitignored `workspace-{dev,stag,prod}/` output live under `provision/`). Provision with `make -C fl-services/nvflare provision-2-nets` (nvflare) or `make -C fl-services/flower provision NET_NUMBER=` (flower). To run a backend standalone + submit without the full stack: `make -C fl-services/ up` (or `up-secure`) then `make -C fl-services/ submit APP=`. - `FL_APP_BASE_DIR` β€” Local directory holding the base FL application templates (the repo's `fl-apps/` tree), baked into the flip-api image and bind-mounted in dev. flip-api walks `///` to bundle an application (uploading those files into `FL_APP_DESTINATION_BUCKET/`) and reads each backend's manifest from `//required_files.json`. Default `/app/fl-apps`; override to mount operator-provided templates. Replaces the removed `FL_APP_BASE_BUCKET` S3 dependency (FLIP#724): base templates are no longer published to S3 (the `fl-apps-push-s3-*` sync workflows are gone), so a template hotfix now ships by rebuilding + redeploying the flip-api image rather than syncing S3. `fl-apps/` is baked into the image via a BuildKit named build context (`fl_apps=../fl-apps`) since it sits outside flip-api's build context. For the Flower backend, the template pyprojects also steer Flower's **per-run dependency install** (`uv sync` on every app launch; SuperNodes opt in via `--allow-runtime-dependency-installation` in the composes): `[tool.uv.sources]` pins `flip-utils` to the source kept at `/opt/flip-utils` inside the FL images (never PyPI β€” FLIP#767; a flip-utils change ships by rebuilding the FL images, `make build-fl FL_BACKEND=flower`) and torch/torchvision to the cu128 index (PyPI's default cu130 wheels need driver >=580). - `FL_KIT_SLOT_NAMES` β€” JSON list (e.g. `["Trust_1", "Trust_2"]`) of FL kit-slot names for the hub's `fl_kit_slot` pool that `register_trust` claims from; each name must match a provisioned participant kit (in-tree workspace for dev, `s3:///fl-flare-participant-kits//net-/services//` for stag/prod; slot names are global across nets β€” every net carries a kit per name). The pool is seeded at flip-api boot and **reconciled on demand** when a registration finds it exhausted (`resolve_fl_kit_slot_names`, additive β€” never deletes or re-assigns rows); only then does `NoFreeKitSlotError` surface. Single source per env: dev = this env var (a `DevSettings`-only field; restart to change, settings load once); stag/prod = the `/flip/fl_kit_slot_names` SSM parameter (Terraform-rendered from this var β€” the list is plain config, not a secret; deliberately **no env fallback**, so a broken/missing parameter means the pool can't grow, loudly, never masked by stale task-def env). Growing the pool is an env-file edit + `make -C deploy/providers/AWS apply-fl-kit-slots` (targeted plan/apply of just the parameter, plain-text diff) β€” **no restart, no task-definition change**. One-command workflow: `make -C deploy/providers/AWS add-fl-kits N= PROD=stag|true` (N = "ensure N more live slots": activate spares toward N first, mint only the shortfall on every net β†’ additive S3 upload β†’ env edit β†’ parameter apply); full runbook in `fl-services/nvflare/README.md` ("Onboarding a new client onto an existing network"). NVFLARE-only dynamics β€” Flower's SuperNode key labelling reads the list at net startup. -- `PROD` β€” `true` (production), `stag` (staging), unset (development) +- `PROD` β€” `true` (production), `stag` (staging), `lza` (LZA FLIPProduction account, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets only; selects the root `.env.lza-prod` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying to the LZA account"), unset (development) - `AES_KEY_BASE64` β€” encryption key for trust communication - A remote trust operator only needs their kit file (`trust/.env.`) β€” no hub `.env.` needed on trust hosts. See `trust/README.md` for the standalone-operator quick-start. diff --git a/CLAUDE.md b/CLAUDE.md index 15bbed29b..c97db510e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -368,7 +368,7 @@ After changes, evaluate if docs need updating: - `FL_PROVISIONED_DIR` β€” path to the in-tree provisioned FL artifacts, derived per-backend by `deploy/fl_backend.mk` from `FL_BACKEND`: `fl-services/nvflare/provision/workspace-dev` (nvflare startup kits) or `fl-services/flower/provision/creds` (flower per-net TLS certs + SuperNode keys). Both gitignored. Read only by the dev compose overlays for the cert/workspace volume mounts; override at the CLI for a one-off (`make up FL_PROVISIONED_DIR=...`). FL Makefiles are **per-backend** β€” each `fl-services//Makefile` owns that backend's `build`/`provision`/`up`/`down`/`submit` (flower also `up-secure`); the root Makefile forwards only `build-fl` by `FL_BACKEND`. Each backend's `fl-services//Makefile` also owns its network provisioning (NVFLARE adds `provision`/`provision-2-nets`/`provision-stag`/`provision-prod`/`upload-kits-to-s3`; the project YAMLs, `scripts/`, and gitignored `workspace-{dev,stag,prod}/` output live under `provision/`). Provision with `make -C fl-services/nvflare provision-2-nets` (nvflare) or `make -C fl-services/flower provision NET_NUMBER=` (flower). To run a backend standalone + submit without the full stack: `make -C fl-services/ up` (or `up-secure`) then `make -C fl-services/ submit APP=`. - `FL_APP_BASE_DIR` β€” Local directory holding the base FL application templates (the repo's `fl-apps/` tree), baked into the flip-api image and bind-mounted in dev. flip-api walks `///` to bundle an application (uploading those files into `FL_APP_DESTINATION_BUCKET/`) and reads each backend's manifest from `//required_files.json`. Default `/app/fl-apps`; override to mount operator-provided templates. Replaces the removed `FL_APP_BASE_BUCKET` S3 dependency (FLIP#724): base templates are no longer published to S3 (the `fl-apps-push-s3-*` sync workflows are gone), so a template hotfix now ships by rebuilding + redeploying the flip-api image rather than syncing S3. `fl-apps/` is baked into the image via a BuildKit named build context (`fl_apps=../fl-apps`) since it sits outside flip-api's build context. For the Flower backend, the template pyprojects also steer Flower's **per-run dependency install** (`uv sync` on every app launch; SuperNodes opt in via `--allow-runtime-dependency-installation` in the composes): `[tool.uv.sources]` pins `flip-utils` to the source kept at `/opt/flip-utils` inside the FL images (never PyPI β€” FLIP#767; a flip-utils change ships by rebuilding the FL images, `make build-fl FL_BACKEND=flower`) and torch/torchvision to the cu128 index (PyPI's default cu130 wheels need driver >=580). - `FL_KIT_SLOT_NAMES` β€” JSON list (e.g. `["Trust_1", "Trust_2"]`) of FL kit-slot names for the hub's `fl_kit_slot` pool that `register_trust` claims from; each name must match a provisioned participant kit (in-tree workspace for dev, `s3:///fl-flare-participant-kits//net-/services//` for stag/prod; slot names are global across nets β€” every net carries a kit per name). The pool is seeded at flip-api boot and **reconciled on demand** when a registration finds it exhausted (`resolve_fl_kit_slot_names`, additive β€” never deletes or re-assigns rows); only then does `NoFreeKitSlotError` surface. Single source per env: dev = this env var (a `DevSettings`-only field; restart to change, settings load once); stag/prod = the `/flip/fl_kit_slot_names` SSM parameter (Terraform-rendered from this var β€” the list is plain config, not a secret; deliberately **no env fallback**, so a broken/missing parameter means the pool can't grow, loudly, never masked by stale task-def env). Growing the pool is an env-file edit + `make -C deploy/providers/AWS apply-fl-kit-slots` (targeted plan/apply of just the parameter, plain-text diff) β€” **no restart, no task-definition change**. One-command workflow: `make -C deploy/providers/AWS add-fl-kits N= PROD=stag|true` (N = "ensure N more live slots": activate spares toward N first, mint only the shortfall on every net β†’ additive S3 upload β†’ env edit β†’ parameter apply); full runbook in `fl-services/nvflare/README.md` ("Onboarding a new client onto an existing network"). NVFLARE-only dynamics β€” Flower's SuperNode key labelling reads the list at net startup. -- `PROD` β€” `true` (production), `stag` (staging), unset (development) +- `PROD` β€” `true` (production), `stag` (staging), `lza` (LZA FLIPProduction account, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets only; selects the root `.env.lza-prod` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying to the LZA account"), unset (development) - `AES_KEY_BASE64` β€” encryption key for trust communication - A remote trust operator only needs their kit file (`trust/.env.`) β€” no hub `.env.` needed on trust hosts. See `trust/README.md` for the standalone-operator quick-start. diff --git a/deploy/providers/AWS/AGENTS.md b/deploy/providers/AWS/AGENTS.md index 29f61ee34..3a6e04a0f 100644 --- a/deploy/providers/AWS/AGENTS.md +++ b/deploy/providers/AWS/AGENTS.md @@ -5,6 +5,7 @@ | File | Resources | |------|-----------| | `main.tf` | Provider config, VPC, subnets, IGW, NAT, route tables, RDS instance, Secrets Manager, SES | +| `network_lza.tf` | LZA platform-managed network (FLIP#749): VPC/subnet data lookups + the `local.vpc_id` / `local.app_subnet_ids` / `local.data_subnet_ids` locals both paths consume | | `services.tf` | S3 buckets, Cognito | | `rds_proxy.tf` | RDS Proxy + IAM DB auth (proxy, IAM role/policy, SG, `rds-db:connect`) β€” see FLIP#556 | | `ecs.tf` | ECS cluster, capacity providers, ECS CloudWatch log groups (ALB / NLB / target groups / listener rules live in `main.tf`) | @@ -27,6 +28,7 @@ | ------- | ------------- | --------- | | `stag` | Staging | `flipstag` | | `prod` | Production | `flipprod` | +| `FLIPAdminAccess-893493035022` | LZA FLIPProduction (`PROD=lza`, FLIP#749) | `893493035022` | | `FlipDeveloperAccess-080369786334` | Developer access | β€” | ## Key Deploy Commands @@ -34,6 +36,7 @@ ```bash make full-deploy PROD=stag # Full staging deploy make full-deploy PROD=true # Full prod deploy +make init/plan/apply PROD=lza # LZA FLIPProduction (env-gated; full-deploy chains untested there β€” see README "Deploying to the LZA account") make full-deploy-hybrid PROD= [LOCAL_TRUST_IP=] # Hybrid with on-prem trust make full-deploy-hub-only PROD= # Hub only, NO cloud Trust EC2 (all trusts on-prem, e.g. GPU hosts) β€” see README "Hub-only Deployment" make init/plan/apply # Terraform workflow diff --git a/deploy/providers/AWS/CLAUDE.md b/deploy/providers/AWS/CLAUDE.md index 036435036..ca6a6f911 100644 --- a/deploy/providers/AWS/CLAUDE.md +++ b/deploy/providers/AWS/CLAUDE.md @@ -5,6 +5,7 @@ | File | Resources | |------|-----------| | `main.tf` | Provider config, VPC, subnets, IGW, NAT, route tables, RDS instance, Secrets Manager, SES | +| `network_lza.tf` | LZA platform-managed network (FLIP#749): VPC/subnet data lookups + the `local.vpc_id` / `local.app_subnet_ids` / `local.data_subnet_ids` locals both paths consume | | `services.tf` | S3 buckets, Cognito | | `rds_proxy.tf` | RDS Proxy + IAM DB auth (proxy, IAM role/policy, SG, `rds-db:connect`) β€” see FLIP#556 | | `ecs.tf` | ECS cluster, capacity providers, ECS CloudWatch log groups (ALB / NLB / target groups / listener rules live in `main.tf`) | @@ -27,6 +28,7 @@ | ------- | ------------- | --------- | | `stag` | Staging | `flipstag` | | `prod` | Production | `flipprod` | +| `FLIPAdminAccess-893493035022` | LZA FLIPProduction (`PROD=lza`, FLIP#749) | `893493035022` | | `FlipDeveloperAccess-080369786334` | Developer access | β€” | ## Key Deploy Commands @@ -34,6 +36,7 @@ ```bash make full-deploy PROD=stag # Full staging deploy make full-deploy PROD=true # Full prod deploy +make init/plan/apply PROD=lza # LZA FLIPProduction (env-gated; full-deploy chains untested there β€” see README "Deploying to the LZA account") make full-deploy-hybrid PROD= [LOCAL_TRUST_IP=] # Hybrid with on-prem trust make full-deploy-hub-only PROD= # Hub only, NO cloud Trust EC2 (all trusts on-prem, e.g. GPU hosts) β€” see README "Hub-only Deployment" make init/plan/apply # Terraform workflow diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index 5f4c19583..f3b4337c8 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -34,7 +34,7 @@ In both models, trusts poll the Central Hub for tasks over HTTPS β€” all communi 5. **GitHub CLI** installed via [GitHub CLI installation guide](https://cli.github.com/) 6. **SSH key pair** created at `~/.ssh/host-aws` (see [deploy README](../../README.md)) 7. **Environment files** configured: (see [deploy README](../../README.md)) - - `.env.stag` (staging) or `.env.production` (production) in project root + - `.env.stag` (staging), `.env.production` (production), or `.env.lza-prod` (LZA FLIPProduction) in project root - Service-specific `.env` files (see Environment Configuration section) ### Required AWS Permissions @@ -586,10 +586,12 @@ The `PROD` variable determines which environment files are loaded: - `PROD=stag` β†’ Uses the root `.env.stag` - `PROD=true` β†’ Uses the root `.env.production` +- `PROD=lza` β†’ Uses the root `.env.lza-prod` (the LZA FLIPProduction account β€” see + [Deploying to the LZA account](#deploying-to-the-lza-account-flipproduction)) If `PROD` is omitted when running the AWS provider Makefile, it defaults to staging. -The Makefile maps `PROD` onto `TF_VAR_environment` (`prod` when `PROD=true`, otherwise `stag`). Terraform branches on this variable to gate prod-only RDS hardening β€” see [RDS lifecycle](#rds-lifecycle-stag-vs-prod). +The Makefile maps `PROD` onto `TF_VAR_environment` (`prod` when `PROD=true` or `PROD=lza`, otherwise `stag`). Terraform branches on this variable to gate prod-only RDS hardening β€” see [RDS lifecycle](#rds-lifecycle-stag-vs-prod). `PROD=lza` additionally sets the orthogonal `TF_VAR_lza_managed_network=true` platform-managed-network toggle. #### AWS profile aliases @@ -638,6 +640,122 @@ make apply See [`dev/README.md`](./dev/README.md) for the one-time `terraform import` workflow that pulls the manually-created dev Cognito pool into state. +### Deploying to the LZA account (FLIPProduction) + +[FLIP#749](https://github.com/londonaicentre/FLIP/issues/749) migrates the production deployment to the LZA-provisioned +**FLIPProduction** account (`893493035022`, eu-west-2). Until cutover it runs **in parallel** with legacy prod: every +LZA adaptation in this stack is env-gated behind `PROD=lza`, and with `PROD=true`/`PROD=stag` the resolved +configuration is identical to before β€” the legacy environments are never touched by LZA work. + +**What `PROD=lza` selects:** + +| Concern | Value | +| --- | --- | +| Env file | root `.env.lza-prod` (gitignored, like the other env files) | +| Profile guard | `AWS_PROFILE=FLIPAdminAccess-893493035022` (override via `LZA_AWS_PROFILE`) | +| `TF_VAR_environment` | `prod` β€” LZA is a production estate, so all prod-only hardening (RDS deletion protection, final snapshot) stays on | +| `TF_VAR_lza_managed_network` | `true` β€” the platform-managed-network toggle, orthogonal to `environment` (see below) | +| Trust kit suffix | `trust/.env..lza-prod` β€” a separate namespace so legacy prod kits are never overwritten | +| `deploy-centralhub` git ref | `origin/main` (same as legacy prod) | + +**Platform-managed vs FLIP-managed.** The LZA account's network is owned by the accelerator pipeline +([londonaicentre/lza](https://github.com/londonaicentre/lza)) and VPC-layer creation is SCP-denied in-account, so with +`TF_VAR_lza_managed_network=true` Terraform: + +- **skips creating**: the VPC module (VPC/subnets/NAT/IGW/EIPs), the in-account VPC endpoints (interface endpoints are + centralised in the Network account; S3+DynamoDB gateway endpoints are platform-provided), the DHCP options, and the + `/flip/networking/*` SSM params (legacy TGW coupling β€” the LZA TGW attachment is platform-managed); +- **discovers instead**: the `AWSAccelerator-eu-west-2-prod` VPC and its subnets by Name tag (`network_lza.tf`; + override the name via `LZA_VPC_NAME`). Subnet lookups match ALL `-app-*` / `-data-*` hits, so the multi-AZ `-b` + subnets the platform team is adding appear on the next plan without a code change; +- **places by connectivity need**: RDS instances go to the isolated **data** subnets (local routes only β€” nothing + there can reach TGW/endpoints, and RDS doesn't need to); ECS tasks, the internal ALB, the RDS Proxy, EFS mount + targets and the EC2 hosts go to the TGW-routed **app** subnets (they need the central endpoints / image pulls); +- **gates off**: the SG-drift CloudTrailβ†’EventBridgeβ†’Lambda stack (`security.tf` β€” the org baseline of Control Tower + org trail, GuardDuty, Security Hub and Config covers it) and the public FL-server NLB + target group + DNS record + + SG rules (no IGW and VPC Block Public Access make an internet-facing NLB impossible; the FL inbound architecture is + the open #749 WP2 decision). The `fl-server-net-1` / `fl-api-net-1` ECS services themselves still deploy β€” they just + have no inbound FL path yet. + +Everything else (ECS Fargate, RDS + Proxy, Cognito, S3 + CMK, Secrets Manager, SES, EFS, Cloud Map, internal ALB, +CloudFront + WAF + ACM) remains FLIP-managed exactly as on legacy prod. + +**Prerequisites (already provisioned out-of-band in the account, not Terraform-managed here):** + +- TF state bucket `flip-terraform-state-lza` (versioned, SSE-KMS, public access blocked). `make create-backend + PROD=lza` is idempotent against it. +- ECR **pull-through cache rules** β€” the account has no internet egress, so images come from in-account mirrors over + the central `ecr.api`/`ecr.dkr` endpoints: prefix `ghcr/` mirroring `ghcr.io` (upstream auth via a read-only GHCR + PAT in the `ecr-pullthroughcache/ghcr` Secrets Manager secret) and the credential-less `ecr-public/` prefix + mirroring `public.ecr.aws` (used for the EFS-provision utility image). The execution role's + `ecr:BatchImportUpstreamImage`/`ecr:CreateRepository` grant for first-pull imports IS Terraform-managed + (`iam_ecs.tf`, LZA-gated). +- The Identity Center `FLIPAdminAccess` profile in `~/.aws/config` (an `aws configure sso` against the account). + +**`.env.lza-prod`.** Carries the same keys as `.env.production` (start from that shape); the values that MUST differ, +plus the LZA-only keys: + +```bash +# Terraform backend + account +FLIP_TFSTATE_BUCKET_NAME=flip-terraform-state-lza +AWS_REGION=eu-west-2 + +# Registry: the ghcr/ pull-through cache, NOT ghcr.io (no internet egress). +# Composes with the image names exactly like the GHCR prefix does: +# : β†’ .../ghcr/londonaicentre/flip-api: +DOCKER_REGISTRY=893493035022.dkr.ecr.eu-west-2.amazonaws.com/ghcr/londonaicentre/ +# EFS-provision one-shot utility image via the credential-less ecr-public/ cache +EFS_PROVISION_IMAGE=893493035022.dkr.ecr.eu-west-2.amazonaws.com/ecr-public/aws-cli/aws-cli:2.22.35 + +# Bucket names are globally unique and the legacy flipprod-* names stay taken +# while the old account lives β€” the LZA env uses its own flip-lza-* namespace. +FLIP_MODEL_FILES_UPLOADS_BUCKET_NAME=flip-lza-model-files-uploads +FLIP_FL_RESULTS_BUCKET_NAME=flip-lza-fl-results +FLIP_APP_BUNDLES_BUCKET_NAME=flip-lza-app-bundles +AICENTRE_BUCKET_NAME=flip-lza-aicentre +FLIP_UI_BUCKET_NAME=flip-lza-ui + +# No Route53 hosted zone in the account yet (its move is a platform-side DNS +# line item) β€” first bring-up runs on the default CloudFront domain. Flip to +# true (and re-apply) once the zone lands. ALB_SUBDOMAIN/NLB_SUBDOMAIN keep +# their eventual post-cutover values meanwhile (used for resource naming). +MANAGE_DNS=false +ALB_SUBDOMAIN=app.flip.aicentre.co.uk +NLB_SUBDOMAIN=fl.app.flip.aicentre.co.uk + +# Optional: only needed if the platform VPC template is renamed. +# LZA_VPC_NAME=AWSAccelerator-eu-west-2-prod +``` + +Secrets (`AES_KEY_BASE64`, `INTERNAL_SERVICE_KEY*`, `ADMIN_USER_PASSWORD`, …) are minted fresh for the account during +the WP3 bring-up and later replaced by the carried-over legacy **values** in the WP4 data migration (so existing trust +kits and encrypted data stay valid) β€” never reuse the legacy Secrets Manager secret itself. + +**First bring-up without DNS (`MANAGE_DNS=false`).** The zone lookup would hard-fail in a zone-less account, so +`MANAGE_DNS=false` skips it plus every Route53 record and both DNS-validated ACM certs. Consequences, all of which +revert by flipping `MANAGE_DNS=true` + `make plan`/`apply` once the zone lands: + +- CloudFront serves on its default `*.cloudfront.net` domain with the default viewer certificate (no aliases β€” + CloudFront only allows the default cert when no aliases are set). `terraform output CloudfrontDistributionDomain` + prints the URL. +- The CloudFrontβ†’ALB `/api/*` origin leg runs **plain HTTP** over the private VPC-origin ENI (an ALB HTTPS listener + needs an ISSUED cert, and issuance needs DNS validation). Viewer traffic stays HTTPS. Accepted as a bring-up-only + limitation β€” the leg never leaves the VPC-origin path. +- Bucket CORS and the Cognito sign-in hostname/callback URLs follow the CloudFront default domain automatically + (`local.ui_origin`), so uploads, downloads and sign-in work; `make deploy-ui` must generate `window.js` with + `CENTRAL_HUB_API_URL` pointing at the CloudFront domain. +- Trusts polling the hub would need the CloudFront domain as `CENTRAL_HUB_API_URL` β€” fine for WP3 smoke trusts; + the real cutover is DNS-only and happens after the zone migrates. + +**Not yet on LZA** (tracked on #749): + +- `make apply` will fail on the ALB and RDS subnet group until the platform team lands the multi-AZ `-b` subnets + (both need β‰₯2 AZs) β€” the hard WP2 platform blocker. +- No FL inbound path (see the NLB gating above) β€” FL training across trusts waits on the WP2 ingress decision. +- The `full-deploy*` chains, `make status`/`check_status.py`, `update_env.py` and `make destroy` are untested against + `PROD=lza` β€” WP3 exercises the `init`/`plan`/`apply` (+ `deploy-centralhub`/`deploy-ui`) loop first and fixes up the + auxiliary tooling as findings come in. + ### Terraform module layout ``` From 8dcdf5a31f15d9ccd2a8a598655141cf868db829 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Mon, 17 Aug 2026 11:46:38 +0100 Subject: [PATCH 06/41] feat(deploy/AWS): route the Flower register-keys one-shot through the LZA-aware subnet locals (#749) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ecs_flower.tf landed on develop after the LZA branch was cut and still read module.flip_vpc.private_subnets directly β€” an empty list on the LZA account, where the VPC module is emptied by create_vpc = false. local.app_subnet_ids resolves to exactly module.flip_vpc.private_subnets on the legacy path, so this is a no-op for prod/stag. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/ecs_flower.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/providers/AWS/ecs_flower.tf b/deploy/providers/AWS/ecs_flower.tf index 69ff22a3c..07697ecd7 100644 --- a/deploy/providers/AWS/ecs_flower.tf +++ b/deploy/providers/AWS/ecs_flower.tf @@ -118,7 +118,7 @@ resource "null_resource" "flower_register_supernode_keys" { --cluster ${aws_ecs_cluster.flip.name} \ --task-definition ${aws_ecs_task_definition.flower_register_supernode_keys[0].arn} \ --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={subnets=[${join(",", module.flip_vpc.private_subnets)}],securityGroups=[${aws_security_group.ecs_fl_api.id}],assignPublicIp=DISABLED}" \ + --network-configuration "awsvpcConfiguration={subnets=[${join(",", local.app_subnet_ids)}],securityGroups=[${aws_security_group.ecs_fl_api.id}],assignPublicIp=DISABLED}" \ --count 1 \ --region ${var.AWS_REGION} \ --no-cli-pager \ From 71e13abf51e50efa604e8604a710f727d995babf Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Mon, 17 Aug 2026 11:49:31 +0100 Subject: [PATCH 07/41] =?UTF-8?q?docs(deploy/AWS):=20refresh=20the=20LZA?= =?UTF-8?q?=20state=20notes=20=E2=80=94=20multi-AZ=20landed,=20FL=20ingres?= =?UTF-8?q?s=20proven=20(#749)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The multi-AZ -b subnets landed via lza#38 (ALB/RDS unblocked, with the TGW-attachment AZ caveat reported on PR#830), and the FL ingress chain was proven end-to-end through the networking-account edge NLB (FLIP#829). Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/README.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index f3b4337c8..51ec3f09b 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -673,9 +673,10 @@ configuration is identical to before β€” the legacy environments are never touch targets and the EC2 hosts go to the TGW-routed **app** subnets (they need the central endpoints / image pulls); - **gates off**: the SG-drift CloudTrailβ†’EventBridgeβ†’Lambda stack (`security.tf` β€” the org baseline of Control Tower org trail, GuardDuty, Security Hub and Config covers it) and the public FL-server NLB + target group + DNS record + - SG rules (no IGW and VPC Block Public Access make an internet-facing NLB impossible; the FL inbound architecture is - the open #749 WP2 decision). The `fl-server-net-1` / `fl-api-net-1` ECS services themselves still deploy β€” they just - have no inbound FL path yet. + SG rules (no IGW and VPC Block Public Access make an internet-facing NLB impossible; FL ingress instead comes + through the networking account's edge NLB over TGW β€” proven end-to-end in FLIP#829/PR#830, wired up for the real + fl-server as follow-up #749 work). The `fl-server-net-1` / `fl-api-net-1` ECS services themselves still deploy β€” + they just have no inbound FL path yet. Everything else (ECS Fargate, RDS + Proxy, Cognito, S3 + CMK, Secrets Manager, SES, EFS, Cloud Map, internal ALB, CloudFront + WAF + ACM) remains FLIP-managed exactly as on legacy prod. @@ -749,9 +750,14 @@ revert by flipping `MANAGE_DNS=true` + `make plan`/`apply` once the zone lands: **Not yet on LZA** (tracked on #749): -- `make apply` will fail on the ALB and RDS subnet group until the platform team lands the multi-AZ `-b` subnets - (both need β‰₯2 AZs) β€” the hard WP2 platform blocker. -- No FL inbound path (see the NLB gating above) β€” FL training across trusts waits on the WP2 ingress decision. +- The multi-AZ `-b` subnets have landed (lza#38), unblocking the ALB and the RDS subnet group β€” but as of 2026-08-11 + the platform TGW attachment was still AZ-a-only, which **blackholes TGW-routed traffic from the `-b` subnets** + (central endpoints, image pulls; reported on PR#830). Until the attachment spans both AZs, workloads that need the + TGW must stay pinned to `-a` subnets β€” the e2e-lza stack (FLIP#829) shows the pinning pattern. +- No FL inbound path (see the NLB gating above). The ingress architecture itself is settled and proven end-to-end + with dummy services β€” internet β†’ networking-account edge NLB β†’ TGW β†’ central firewall β†’ workload NLB β†’ ECS + (FLIP#829 / PR#830) β€” but wiring the real `fl-server-net-1` behind that chain (internal NLB + target group in this + stack, edge listener registration on the platform side) is follow-up #749 work. - The `full-deploy*` chains, `make status`/`check_status.py`, `update_env.py` and `make destroy` are untested against `PROD=lza` β€” WP3 exercises the `init`/`plan`/`apply` (+ `deploy-centralhub`/`deploy-ui`) loop first and fixes up the auxiliary tooling as findings come in. From db86cfe30ff61ddc5c4239842f01b447313a3f04 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Mon, 17 Aug 2026 11:58:57 +0100 Subject: [PATCH 08/41] refactor(deploy/AWS): short 'lza-prod' profile alias for the LZA guard, matching prod/stag (#749) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the auto-generated FLIPAdminAccess-893493035022 profile name with the same short-alias convention the other envs use (override still via LZA_AWS_PROFILE). Also corrects the stale FlipDeveloperAccess-080369786334 row in the CLAUDE/AGENTS profile table β€” that permission set on the legacy staging account was never a distinct environment; the dev row now documents the dev/ root's profile. Inert for PROD=true/stag: LZA_AWS_PROFILE is only read in the PROD=lza branch of the guard. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/AGENTS.md | 4 ++-- deploy/providers/AWS/CLAUDE.md | 4 ++-- deploy/providers/AWS/Makefile | 6 +++--- deploy/providers/AWS/README.md | 6 ++++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/deploy/providers/AWS/AGENTS.md b/deploy/providers/AWS/AGENTS.md index 3a6e04a0f..0282d1344 100644 --- a/deploy/providers/AWS/AGENTS.md +++ b/deploy/providers/AWS/AGENTS.md @@ -28,8 +28,8 @@ | ------- | ------------- | --------- | | `stag` | Staging | `flipstag` | | `prod` | Production | `flipprod` | -| `FLIPAdminAccess-893493035022` | LZA FLIPProduction (`PROD=lza`, FLIP#749) | `893493035022` | -| `FlipDeveloperAccess-080369786334` | Developer access | β€” | +| `lza-prod` | LZA FLIPProduction (`PROD=lza`, FLIP#749; `FLIPAdminAccess` permission set) | `893493035022` | +| `dev` | Development (the `dev/` root: Cognito + SES; `FlipDeveloperAccess` permission set) | `flipdev` | ## Key Deploy Commands diff --git a/deploy/providers/AWS/CLAUDE.md b/deploy/providers/AWS/CLAUDE.md index ca6a6f911..f6ea18a84 100644 --- a/deploy/providers/AWS/CLAUDE.md +++ b/deploy/providers/AWS/CLAUDE.md @@ -28,8 +28,8 @@ | ------- | ------------- | --------- | | `stag` | Staging | `flipstag` | | `prod` | Production | `flipprod` | -| `FLIPAdminAccess-893493035022` | LZA FLIPProduction (`PROD=lza`, FLIP#749) | `893493035022` | -| `FlipDeveloperAccess-080369786334` | Developer access | β€” | +| `lza-prod` | LZA FLIPProduction (`PROD=lza`, FLIP#749; `FLIPAdminAccess` permission set) | `893493035022` | +| `dev` | Development (the `dev/` root: Cognito + SES; `FlipDeveloperAccess` permission set) | `flipdev` | ## Key Deploy Commands diff --git a/deploy/providers/AWS/Makefile b/deploy/providers/AWS/Makefile index 1da00e706..b84b5a0e4 100644 --- a/deploy/providers/AWS/Makefile +++ b/deploy/providers/AWS/Makefile @@ -61,9 +61,9 @@ $(if $(FL_KIT_DATE),,$(error 🧠 $(if $(filter nvflare,$(FL_BACKEND)),FLARE_KIT PROD_AWS_PROFILE ?= prod STAG_AWS_PROFILE ?= stag # The LZA FLIPProduction account (893493035022) is reached via the Identity -# Center `FLIPAdminAccess` permission set; the default matches the profile name -# `aws configure sso` generates for it (FLIP#749). -LZA_AWS_PROFILE ?= FLIPAdminAccess-893493035022 +# Center `FLIPAdminAccess` permission set; like prod/stag above, the default is +# a short local alias for it in ~/.aws/config (FLIP#749). +LZA_AWS_PROFILE ?= lza-prod # Refuse to operate against an unintended AWS account. Profile name is the safety # guard because Make has no easy way to read the account ID without an `aws` call. diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index 51ec3f09b..631b734b3 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -652,7 +652,7 @@ configuration is identical to before β€” the legacy environments are never touch | Concern | Value | | --- | --- | | Env file | root `.env.lza-prod` (gitignored, like the other env files) | -| Profile guard | `AWS_PROFILE=FLIPAdminAccess-893493035022` (override via `LZA_AWS_PROFILE`) | +| Profile guard | `AWS_PROFILE=lza-prod` β€” a short local alias for the `FLIPAdminAccess` permission set on `893493035022`, per the same convention as `prod`/`stag` (override via `LZA_AWS_PROFILE`) | | `TF_VAR_environment` | `prod` β€” LZA is a production estate, so all prod-only hardening (RDS deletion protection, final snapshot) stays on | | `TF_VAR_lza_managed_network` | `true` β€” the platform-managed-network toggle, orthogonal to `environment` (see below) | | Trust kit suffix | `trust/.env..lza-prod` β€” a separate namespace so legacy prod kits are never overwritten | @@ -691,7 +691,9 @@ CloudFront + WAF + ACM) remains FLIP-managed exactly as on legacy prod. mirroring `public.ecr.aws` (used for the EFS-provision utility image). The execution role's `ecr:BatchImportUpstreamImage`/`ecr:CreateRepository` grant for first-pull imports IS Terraform-managed (`iam_ecs.tf`, LZA-gated). -- The Identity Center `FLIPAdminAccess` profile in `~/.aws/config` (an `aws configure sso` against the account). +- An `lza-prod` profile in `~/.aws/config` for the account's Identity Center `FLIPAdminAccess` permission set (an + `aws configure sso` against the account, then rename the generated profile β€” same short-alias convention as + `prod`/`stag`; override the expected name via `LZA_AWS_PROFILE`). **`.env.lza-prod`.** Carries the same keys as `.env.production` (start from that shape); the values that MUST differ, plus the LZA-only keys: From 5a9343fde109515be875a6f5d8e560157733be7a Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Mon, 17 Aug 2026 12:06:42 +0100 Subject: [PATCH 09/41] =?UTF-8?q?docs(deploy/AWS):=20correct=20the=20multi?= =?UTF-8?q?-AZ=20state=20=E2=80=94=20data-b=20and=20tgw-b=20subnets=20stil?= =?UTF-8?q?l=20missing=20(#749)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified against the live FLIPProduction account: prod-app-b exists (ALB unblocked) but prod-data-b does not (RDS subnet group still fails >=2-AZ) and prod-tgw-b does not (TGW attachment single-subnet, AZ-b blackhole). Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index 631b734b3..3f70100e4 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -752,10 +752,12 @@ revert by flipping `MANAGE_DNS=true` + `make plan`/`apply` once the zone lands: **Not yet on LZA** (tracked on #749): -- The multi-AZ `-b` subnets have landed (lza#38), unblocking the ALB and the RDS subnet group β€” but as of 2026-08-11 - the platform TGW attachment was still AZ-a-only, which **blackholes TGW-routed traffic from the `-b` subnets** - (central endpoints, image pulls; reported on PR#830). Until the attachment spans both AZs, workloads that need the - TGW must stay pinned to `-a` subnets β€” the e2e-lza stack (FLIP#829) shows the pinning pattern. +- Multi-AZ is only partially landed (verified against the live account 2026-08-17): `prod-app-b` exists (lza#38), so + the **ALB is unblocked**, but there is **no `prod-data-b`** β€” the RDS subnet group still fails its β‰₯2-AZ + requirement β€” and **no `prod-tgw-b`**, so the TGW attachment rides the lone `tgw-a` subnet and **TGW-routed + traffic from `-b` subnets blackholes** (central endpoints, image pulls; reported on PR#830). Until `data-b`/`tgw-b` + land and the attachment spans both AZs, workloads that need the TGW must stay pinned to `-a` subnets β€” the e2e-lza + stack (FLIP#829) shows the pinning pattern. - No FL inbound path (see the NLB gating above). The ingress architecture itself is settled and proven end-to-end with dummy services β€” internet β†’ networking-account edge NLB β†’ TGW β†’ central firewall β†’ workload NLB β†’ ECS (FLIP#829 / PR#830) β€” but wiring the real `fl-server-net-1` behind that chain (internal NLB + target group in this From 512e19e7c3313800f69368fdc1fa1bfada5379a5 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Mon, 17 Aug 2026 12:22:45 +0100 Subject: [PATCH 10/41] fix(deploy/AWS): gate the endpoint-SG trust-egress rule off on LZA (#749) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #876 trust-EC2 egress allowlist (landed after this branch was cut) references aws_security_group.vpc_endpoints[0] whenever enable_ecs_endpoints is true β€” but on LZA that SG is count-gated off (endpoints are centralised in the Network account), so the first real 'terraform plan PROD=lza' failed with an empty-tuple index. The rule's gate now mirrors the SG's own count. Legacy is untouched: the predicate only changes where lza_managed_network is true. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/main.tf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deploy/providers/AWS/main.tf b/deploy/providers/AWS/main.tf index 30af50372..2ae676b21 100644 --- a/deploy/providers/AWS/main.tf +++ b/deploy/providers/AWS/main.tf @@ -189,8 +189,10 @@ locals { # private IPs), so it is documentation of intent rather than a control. When # var.enable_ecs_endpoints is false the endpoints don't exist and this rule is omitted entirely: # the 443 floor already carries that traffic, and re-stating it as a public rule here would - # collide with the floor on the same tuple. - var.enable_ecs_endpoints ? [ + # collide with the floor on the same tuple. Same on LZA (the gate mirrors the SG's own count in + # vpc_endpoints.tf): endpoints are centralised in the Network account there, so no local + # endpoint SG exists to reference. + var.enable_ecs_endpoints && !var.lza_managed_network ? [ merge(local.trust_egress_rule_defaults, { port = 443 source_security_group_id = aws_security_group.vpc_endpoints[0].id From a492b9b98345cd0dbf3bbf62ab1f4225904aacaf Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Mon, 17 Aug 2026 12:22:45 +0100 Subject: [PATCH 11/41] feat(deploy/AWS): overridable log-bucket names for cross-account name collisions (#749) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The access-logs and CloudFront-logs buckets derive their names from flip_alb_subdomain β€” which the LZA env deliberately keeps at its post-cutover value, so the derived names are still owned by legacy prod (S3 names are global; head-bucket returns 403 from FLIPProduction). New optional ACCESS_LOGS_BUCKET_NAME / CF_LOGS_BUCKET_NAME env overrides move them into the flip-lza-* namespace; empty (the default everywhere else) keeps the derived names, so legacy remains byte-identical (make -n re-proven for PROD=true/stag). Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/Makefile | 3 +++ deploy/providers/AWS/README.md | 6 ++++++ deploy/providers/AWS/cloudfront.tf | 4 +++- deploy/providers/AWS/s3_logging.tf | 5 ++++- deploy/providers/AWS/variables.tf | 12 ++++++++++++ 5 files changed, 28 insertions(+), 2 deletions(-) diff --git a/deploy/providers/AWS/Makefile b/deploy/providers/AWS/Makefile index b84b5a0e4..b952c1221 100644 --- a/deploy/providers/AWS/Makefile +++ b/deploy/providers/AWS/Makefile @@ -117,6 +117,9 @@ export TF_VAR_FLIP_MODEL_FILES_UPLOADS_BUCKET_NAME=${FLIP_MODEL_FILES_UPLOADS_BU export TF_VAR_FLIP_FL_RESULTS_BUCKET_NAME=${FLIP_FL_RESULTS_BUCKET_NAME} export TF_VAR_FLIP_APP_BUNDLES_BUCKET_NAME=${FLIP_APP_BUNDLES_BUCKET_NAME} export TF_VAR_AICENTRE_BUCKET_NAME=${AICENTRE_BUCKET_NAME} +# Optional log-bucket name overrides (empty = derived from flip_alb_subdomain; see variables.tf) +export TF_VAR_ACCESS_LOGS_BUCKET_NAME=${ACCESS_LOGS_BUCKET_NAME} +export TF_VAR_CF_LOGS_BUCKET_NAME=${CF_LOGS_BUCKET_NAME} # A copied-but-unedited .env leaves `` placeholders in place. Those are non-empty, so an # emptiness check passes them straight through to Terraform, which fails much later with an opaque # AWS bucket-name error. Treat a placeholder as unset β€” same idea as XNAT_CREDENTIAL_PLACEHOLDER in diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index 3f70100e4..9b91092f7 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -717,6 +717,12 @@ FLIP_FL_RESULTS_BUCKET_NAME=flip-lza-fl-results FLIP_APP_BUNDLES_BUCKET_NAME=flip-lza-app-bundles AICENTRE_BUCKET_NAME=flip-lza-aicentre FLIP_UI_BUCKET_NAME=flip-lza-ui +# The two log buckets default to subdomain-derived names +# (flip-access-logs-/flip-cf-logs-) β€” but ALB_SUBDOMAIN keeps its +# post-cutover value here, so those derived names are still owned by legacy +# prod. Bucket names are global: override them into the flip-lza-* namespace. +ACCESS_LOGS_BUCKET_NAME=flip-lza-access-logs +CF_LOGS_BUCKET_NAME=flip-lza-cf-logs # No Route53 hosted zone in the account yet (its move is a platform-side DNS # line item) β€” first bring-up runs on the default CloudFront domain. Flip to diff --git a/deploy/providers/AWS/cloudfront.tf b/deploy/providers/AWS/cloudfront.tf index 32810902e..7b2a46cec 100644 --- a/deploy/providers/AWS/cloudfront.tf +++ b/deploy/providers/AWS/cloudfront.tf @@ -179,7 +179,9 @@ resource "aws_security_group_rule" "alb_ingress_https_from_cloudfront" { ############################ resource "aws_s3_bucket" "cloudfront_logs" { - bucket = "flip-cf-logs-${var.flip_alb_subdomain}" + # Derived-name-with-override, same rationale as access_logs_bucket_name in + # s3_logging.tf (global bucket names vs the shared subdomain, FLIP#749). + bucket = var.CF_LOGS_BUCKET_NAME != "" ? var.CF_LOGS_BUCKET_NAME : "flip-cf-logs-${var.flip_alb_subdomain}" tags = { Name = "flip-cloudfront-logs" diff --git a/deploy/providers/AWS/s3_logging.tf b/deploy/providers/AWS/s3_logging.tf index 9042c7ddc..8d2970ef2 100644 --- a/deploy/providers/AWS/s3_logging.tf +++ b/deploy/providers/AWS/s3_logging.tf @@ -29,7 +29,10 @@ locals { # argument" on any environment where this bucket does not yet exist. The # module callers pass this local; first-apply ordering is still guaranteed # by their `depends_on = [aws_s3_bucket_acl.flip_access_logs]`. - access_logs_bucket_name = "flip-access-logs-${var.flip_alb_subdomain}" + # Name is derived from the subdomain unless overridden β€” the override exists + # because bucket names are global and the LZA env shares the eventual + # subdomain with legacy prod, which still owns the derived name (FLIP#749). + access_logs_bucket_name = var.ACCESS_LOGS_BUCKET_NAME != "" ? var.ACCESS_LOGS_BUCKET_NAME : "flip-access-logs-${var.flip_alb_subdomain}" } resource "aws_s3_bucket" "flip_access_logs" { diff --git a/deploy/providers/AWS/variables.tf b/deploy/providers/AWS/variables.tf index a4432832f..88af0f51b 100644 --- a/deploy/providers/AWS/variables.tf +++ b/deploy/providers/AWS/variables.tf @@ -242,6 +242,18 @@ variable "enable_service_discovery" { default = true } +variable "ACCESS_LOGS_BUCKET_NAME" { + description = "Override for the S3 server-access-logs bucket name; empty derives flip-access-logs-. Bucket names are global, so set this where the derived name is already owned by another account β€” e.g. the LZA env, whose flip_alb_subdomain deliberately keeps its post-cutover value while legacy prod still owns the derived name (FLIP#749)." + type = string + default = "" +} + +variable "CF_LOGS_BUCKET_NAME" { + description = "Override for the CloudFront standard-logs bucket name; empty derives flip-cf-logs-. Same global-name rationale as ACCESS_LOGS_BUCKET_NAME (FLIP#749)." + type = string + default = "" +} + variable "FLIP_MODEL_FILES_UPLOADS_BUCKET_NAME" { description = "Globally-unique S3 bucket name for researcher-uploaded model files (browser presigned-PUT surface today; narrows to presigned POST once PR #438 lands). Required, no default β€” must be set per environment in the matching .env.*." type = string From fbf6c275b534f161a0d4d5cc6042233c30415143 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Wed, 19 Aug 2026 23:42:48 +0100 Subject: [PATCH 12/41] feat(deploy/AWS): port the LZA FL/web ingress from the e2e harness (#749 WP3) The workload-account half of the two-tier ingress, proven end-to-end by the FLIP#829 e2e stack and now provided by the real stack so it can supersede the dummy services: - fl_ingress_lza.tf: internal FL NLB (LZA counterpart of the gated-off internet-facing module.fl_server_nlb) with static per-subnet IPs assigned via subnet_mapping -- the stable targets the networking-account edge NLB registers once, no sync machinery; backend-keyed Fargate TG; SG admitting FL_SERVER_PORT from the VPC and the networking ingress CIDRs; the ALB ingress rule for the CloudFront relay path (legacy admits CloudFront via the SG-drift stack, gated off on LZA); /flip/networking/* SSM handoff params (fl_nlb_private_ips filtered to TGW-reachable AZs, fl_port, alb_dns_name for the relay target-sync Lambda, web_port). - ecs_services.tf: fl-server-net-1 now always attaches a load_balancer -- the legacy TG or the LZA one -- and keeps its health-check grace period on both paths. - variables.tf: networking_ingress_cidrs (default [], stack applies standalone) + lza_fl_nlb_host_num (default 251; deliberately distinct from the e2e harness's 250 while both stacks share the app subnets). Everything is count/create-gated on lza_managed_network with list arguments emptied on legacy, so the legacy prod/stag plan is unchanged. terraform validate green. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/AGENTS.md | 1 + deploy/providers/AWS/CLAUDE.md | 1 + deploy/providers/AWS/ecs_services.tf | 28 ++- deploy/providers/AWS/fl_ingress_lza.tf | 226 +++++++++++++++++++++++++ deploy/providers/AWS/variables.tf | 14 +- 5 files changed, 253 insertions(+), 17 deletions(-) create mode 100644 deploy/providers/AWS/fl_ingress_lza.tf diff --git a/deploy/providers/AWS/AGENTS.md b/deploy/providers/AWS/AGENTS.md index 0282d1344..8753cdeec 100644 --- a/deploy/providers/AWS/AGENTS.md +++ b/deploy/providers/AWS/AGENTS.md @@ -6,6 +6,7 @@ |------|-----------| | `main.tf` | Provider config, VPC, subnets, IGW, NAT, route tables, RDS instance, Secrets Manager, SES | | `network_lza.tf` | LZA platform-managed network (FLIP#749): VPC/subnet data lookups + the `local.vpc_id` / `local.app_subnet_ids` / `local.data_subnet_ids` locals both paths consume | +| `fl_ingress_lza.tf` | LZA-only ingress (FLIP#749 WP3, ported from the #829 e2e harness): internal FL NLB with static per-subnet IPs + its TG/SG, the ALB ingress rule for the networking-account relay path, and the `/flip/networking/*` edge-handoff SSM params | | `services.tf` | S3 buckets, Cognito | | `rds_proxy.tf` | RDS Proxy + IAM DB auth (proxy, IAM role/policy, SG, `rds-db:connect`) β€” see FLIP#556 | | `ecs.tf` | ECS cluster, capacity providers, ECS CloudWatch log groups (ALB / NLB / target groups / listener rules live in `main.tf`) | diff --git a/deploy/providers/AWS/CLAUDE.md b/deploy/providers/AWS/CLAUDE.md index f6ea18a84..995008e85 100644 --- a/deploy/providers/AWS/CLAUDE.md +++ b/deploy/providers/AWS/CLAUDE.md @@ -6,6 +6,7 @@ |------|-----------| | `main.tf` | Provider config, VPC, subnets, IGW, NAT, route tables, RDS instance, Secrets Manager, SES | | `network_lza.tf` | LZA platform-managed network (FLIP#749): VPC/subnet data lookups + the `local.vpc_id` / `local.app_subnet_ids` / `local.data_subnet_ids` locals both paths consume | +| `fl_ingress_lza.tf` | LZA-only ingress (FLIP#749 WP3, ported from the #829 e2e harness): internal FL NLB with static per-subnet IPs + its TG/SG, the ALB ingress rule for the networking-account relay path, and the `/flip/networking/*` edge-handoff SSM params | | `services.tf` | S3 buckets, Cognito | | `rds_proxy.tf` | RDS Proxy + IAM DB auth (proxy, IAM role/policy, SG, `rds-db:connect`) β€” see FLIP#556 | | `ecs.tf` | ECS cluster, capacity providers, ECS CloudWatch log groups (ALB / NLB / target groups / listener rules live in `main.tf`) | diff --git a/deploy/providers/AWS/ecs_services.tf b/deploy/providers/AWS/ecs_services.tf index a7e9c2bec..c5fad17db 100644 --- a/deploy/providers/AWS/ecs_services.tf +++ b/deploy/providers/AWS/ecs_services.tf @@ -173,24 +173,20 @@ resource "aws_ecs_service" "fl_server_net_1" { } # Register the running task's ENI IP with the NLB target group so FL - # clients reaching fl..flip.aicentre.co.uk:8002 hit the Fargate task - # over gRPC. NLB on TCP forwards the gRPC stream untouched. - # Skipped on LZA (FLIP#749): there is no NLB there yet β€” the service still - # deploys, it just has no inbound FL path until the WP2 ingress decision. - dynamic "load_balancer" { - for_each = var.lza_managed_network ? [] : [1] - content { - target_group_arn = aws_lb_target_group.ecs_fl_server_tcp[0].arn - container_name = "fl-server-net-1" - # Backend-dependent container port (Flower: SuperLink Fleet 9092); the - # NLB listener port trusts connect to stays var.FL_SERVER_PORT. - container_port = local.fl_server_container_port - } + # clients hit the Fargate task over gRPC. NLB on TCP forwards the gRPC + # stream untouched. Legacy: the internet-facing module.fl_server_nlb's TG, + # reached at fl..flip.aicentre.co.uk:8002. LZA (FLIP#749): the + # internal NLB's TG (fl_ingress_lza.tf), reached via the networking + # account's edge NLB over the TGW. + load_balancer { + target_group_arn = var.lza_managed_network ? aws_lb_target_group.ecs_fl_server_tcp_lza[0].arn : aws_lb_target_group.ecs_fl_server_tcp[0].arn + container_name = "fl-server-net-1" + # Backend-dependent container port (Flower: SuperLink Fleet 9092); the + # NLB listener port trusts connect to stays var.FL_SERVER_PORT. + container_port = local.fl_server_container_port } - # The grace period is only valid on services with a load balancer β€” ECS - # rejects it outright when the block above is skipped on LZA. - health_check_grace_period_seconds = var.lza_managed_network ? null : 120 + health_check_grace_period_seconds = 120 deployment_minimum_healthy_percent = 100 deployment_maximum_percent = 200 diff --git a/deploy/providers/AWS/fl_ingress_lza.tf b/deploy/providers/AWS/fl_ingress_lza.tf new file mode 100644 index 000000000..5aed0b51b --- /dev/null +++ b/deploy/providers/AWS/fl_ingress_lza.tf @@ -0,0 +1,226 @@ +# Copyright (c) 2026 Guy's and St Thomas' NHS Foundation Trust & King's College London +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# LZA-only ingress plumbing (FLIP#749 WP3), the workload-account half of the +# two-tier design proven end-to-end by the e2e harness (FLIP#829/#830), from +# which this file is ported: +# +# FL: trust FL client β†’ internet-facing edge NLB :{FL_SERVER_PORT} +# (networking account) β†’ central firewall β†’ TGW β†’ the INTERNAL NLB +# below β†’ fl-server ECS task. The legacy internet-facing +# module.fl_server_nlb is impossible in-account (no IGW + VPC BPA) and +# stays gated off; this internal NLB exists as the STATIC-IP anchor β€” +# its per-subnet IPs are assigned (not discovered) via subnet_mapping, +# so the networking account registers them once, with no sync +# machinery. +# Web: CloudFront + WAF β†’ relay NLB (networking account, target-synced to +# this stack's ALB IPs) β†’ firewall β†’ TGW β†’ module.alb. The ALB itself +# is unchanged; the only LZA addition is the ingress rule below β€” +# legacy admits CloudFront via the SG-drift stack, which is gated off +# here. +# +# Everything in this file is inert on legacy prod/stag (count/create-gated on +# var.lza_managed_network, list arguments emptied) β€” the legacy plan is +# unchanged. + +# Per-subnet detail for the static IP assignment below. Read at plan time +# (the subnets pre-exist), unlike an ENI-discovery data source, which would +# defer to apply and break for_each on the first plan. +data "aws_subnet" "lza_app" { + for_each = var.lza_managed_network ? toset(data.aws_subnets.lza_app[0].ids) : toset([]) + id = each.value +} + +# The prod VPC's TGW attachment β€” read to filter the PUBLISHED FL NLB IPs to +# AZs whose return path exists (AWS drops TGW traffic in AZs where the +# attachment has no ENI). Since lza#45 the attachment spans both AZs, so this +# publishes everything today; the filter keeps a future third AZ from being +# registered before its attachment subnet lands. +data "aws_ec2_transit_gateway_vpc_attachment" "lza" { + count = var.lza_managed_network ? 1 : 0 + + filter { + name = "vpc-id" + values = [data.aws_vpc.lza[0].id] + } + + filter { + name = "state" + values = ["available"] + } +} + +data "aws_subnet" "lza_tgw_attachment" { + for_each = var.lza_managed_network ? toset(data.aws_ec2_transit_gateway_vpc_attachment.lza[0].subnet_ids) : toset([]) + id = each.value +} + +locals { + # The FL NLB's private IPs are ASSIGNED via subnet_mapping rather than + # discovered from its ENIs afterwards: known at plan time, stable by + # construction, and publishable to the networking account before apply. + lza_fl_nlb_private_ips = var.lza_managed_network ? [ + for id in local.app_subnet_ids : cidrhost(data.aws_subnet.lza_app[id].cidr_block, var.lza_fl_nlb_host_num) + ] : [] + + lza_tgw_attachment_azs = var.lza_managed_network ? distinct([ + for s in values(data.aws_subnet.lza_tgw_attachment) : s.availability_zone + ]) : [] + + lza_fl_nlb_published_ips = var.lza_managed_network ? [ + for id in local.app_subnet_ids : + cidrhost(data.aws_subnet.lza_app[id].cidr_block, var.lza_fl_nlb_host_num) + if contains(local.lza_tgw_attachment_azs, data.aws_subnet.lza_app[id].availability_zone) + ] : [] +} + +module "fl_internal_nlb_security_group" { + count = var.lza_managed_network ? 1 : 0 + source = "./modules/secgroup" + name = "flip-fl-internal-nlb-sg" + vpc_id = local.vpc_id + description = "Internal FL NLB - FL TCP from the networking-account edge path and in-VPC callers" + ingress_rules = [ + { + # VPC CIDR: in-VPC verification (probe curls, fl-api). The networking + # ingress CIDRs are the edge-NLB β†’ firewall β†’ TGW path. Trust VPN + # CIDRs join this list when the VPN lands. + port = var.FL_SERVER_PORT + description = "FL TCP from the networking-account edge NLB path and VPC-internal callers" + cidr_blocks = concat([local.vpc_cidr_block], var.networking_ingress_cidrs) + } + ] +} + +resource "aws_ec2_tag" "fl_internal_nlb_security_group_flip_sg" { + count = var.lza_managed_network ? 1 : 0 + resource_id = module.fl_internal_nlb_security_group[0].security_group.id + key = "FlipSG" + value = "true" +} + +module "fl_server_internal_nlb" { + source = "terraform-aws-modules/alb/aws" + # LZA counterpart of the legacy module.fl_server_nlb above β€” created on + # exactly the opposite gate, so precisely one FL NLB exists per environment. + create = var.lza_managed_network + name = "flip-fl-internal-nlb" + load_balancer_type = "network" + vpc_id = local.vpc_id + internal = true + + # Static private IP per subnet (see local.lza_fl_nlb_private_ips above). + subnet_mapping = var.lza_managed_network ? [ + for id in local.app_subnet_ids : { + subnet_id = id + private_ipv4_address = cidrhost(data.aws_subnet.lza_app[id].cidr_block, var.lza_fl_nlb_host_num) + } + ] : [] + + create_security_group = false + security_groups = var.lza_managed_network ? [module.fl_internal_nlb_security_group[0].security_group.id] : [] + enable_deletion_protection = false + + # Standalone Fargate TG below, not the module's target_groups map β€” same + # rationale as aws_lb_target_group.ecs_fl_server_tcp in main.tf. + listeners = { + "fl-server-tcp-listener" = { + port = var.FL_SERVER_PORT + protocol = "TCP" + forward = { + target_group_arn = var.lza_managed_network ? aws_lb_target_group.ecs_fl_server_tcp_lza[0].arn : null + } + } + } + + target_groups = {} +} + +# LZA counterpart of aws_lb_target_group.ecs_fl_server_tcp (main.tf) β€” +# identical semantics: backend-keyed name (port is ForceNew, see the comment +# there), container port per backend, registered by the ECS service's +# load_balancer block in ecs_services.tf, never by Terraform. +resource "aws_lb_target_group" "ecs_fl_server_tcp_lza" { + count = var.lza_managed_network ? 1 : 0 + name = var.fl_backend == "flower" ? "ecs-fl-server-flwr-lza" : "ecs-fl-server-lza" + port = local.fl_server_container_port + protocol = "TCP" + target_type = "ip" + vpc_id = local.vpc_id + + lifecycle { + create_before_destroy = true + } + + health_check { + enabled = true + protocol = "TCP" + port = "traffic-port" + healthy_threshold = 3 + unhealthy_threshold = 3 + interval = 30 + } + + deregistration_delay = 30 +} + +# Web leg: admit the networking-account relay (CloudFront VPC origin β†’ +# relay NLB β†’ firewall β†’ TGW) onto the ALB's main listener. Legacy admits +# CloudFront through the SG-drift stack instead; empty ingress CIDRs create +# nothing, so the stack still applies standalone. +resource "aws_security_group_rule" "alb_ingress_web_from_networking" { + count = var.lza_managed_network && length(var.networking_ingress_cidrs) > 0 ? 1 : 0 + type = "ingress" + description = "Web from the networking-account relay NLB path (CloudFront VPC origin)" + protocol = "tcp" + from_port = var.ALB_HTTPS_PORT + to_port = var.ALB_HTTPS_PORT + cidr_blocks = var.networking_ingress_cidrs + security_group_id = module.alb_security_group.security_group.id +} + +# SSM handoff parameters β€” the contract with the networking account's edge +# configuration (aicentre-lza-iac), replacing the e2e harness's +# /flip-e2e/networking/ prefix with the real one. +resource "aws_ssm_parameter" "lza_fl_nlb_private_ips" { + count = var.lza_managed_network ? 1 : 0 + name = "/flip/networking/fl_nlb_private_ips" + description = "Internal FL NLB static private IPs (comma-separated, assigned via subnet_mapping so they are stable by construction; TGW-reachable AZs only) - registered as IP targets on the networking account's edge NLB FL listener" + type = "StringList" + value = join(",", local.lza_fl_nlb_published_ips) +} + +resource "aws_ssm_parameter" "lza_fl_port" { + count = var.lza_managed_network ? 1 : 0 + name = "/flip/networking/fl_port" + description = "Workload-side FL ingress port (internal NLB listener) - consumed by the networking account's edge configuration" + type = "String" + value = tostring(var.FL_SERVER_PORT) +} + +resource "aws_ssm_parameter" "lza_alb_dns_name" { + count = var.lza_managed_network ? 1 : 0 + name = "/flip/networking/alb_dns_name" + description = "Internal web ALB DNS name - the networking account's target-sync Lambda resolves this on a cadence to keep the web relay NLB targets current (ALB IPs rotate)" + type = "String" + value = module.alb.dns_name +} + +resource "aws_ssm_parameter" "lza_web_port" { + count = var.lza_managed_network ? 1 : 0 + name = "/flip/networking/web_port" + description = "Workload-side web ingress port (the ALB's main listener; plain HTTP on the zone-less bring-up) - the relay NLB's target port on the networking side" + type = "String" + value = tostring(var.ALB_HTTPS_PORT) +} diff --git a/deploy/providers/AWS/variables.tf b/deploy/providers/AWS/variables.tf index 88af0f51b..e1dfa1810 100644 --- a/deploy/providers/AWS/variables.tf +++ b/deploy/providers/AWS/variables.tf @@ -31,7 +31,7 @@ variable "VPC_NAME" { } variable "lza_managed_network" { - description = "Run against the LZA-provisioned (platform-managed) network instead of creating one (FLIP#749). true skips the VPC module (and with it NAT/IGW/EIPs), the VPC endpoints, the DHCP options, and the /flip/networking/* SSM params, and discovers the AWSAccelerator VPC + subnets by Name tag instead (see network_lza.tf). Orthogonal to var.environment: the LZA account is prod-grade, so it deploys with environment=prod AND this flag. Set via PROD=lza in the Makefile." + description = "Run against the LZA-provisioned (platform-managed) network instead of creating one (FLIP#749). true skips the VPC module (and with it NAT/IGW/EIPs), the VPC endpoints, the DHCP options, and the legacy /flip/networking/* SSM params (fl_ingress_lza.tf publishes the LZA edge-handoff set under that prefix instead), and discovers the AWSAccelerator VPC + subnets by Name tag instead (see network_lza.tf). Orthogonal to var.environment: the LZA account is prod-grade, so it deploys with environment=prod AND this flag. Set via PROD=lza in the Makefile." type = bool default = false } @@ -42,6 +42,18 @@ variable "lza_vpc_name" { default = "AWSAccelerator-eu-west-2-prod" } +variable "networking_ingress_cidrs" { + description = "CIDRs of the networking account's ingress-VPC subnets (the edge NLB and CloudFront relay path over the TGW), admitted onto the LZA FL NLB and the ALB's main listener (fl_ingress_lza.tf). Empty by default so the stack applies standalone; the value comes from the networking account. LZA-only β€” legacy prod/stag never reads it." + type = list(string) + default = [] +} + +variable "lza_fl_nlb_host_num" { + description = "Host number (cidrhost index) assigned to the internal FL NLB's static private IP in each app subnet via subnet_mapping (fl_ingress_lza.tf) β€” e.g. 251 in 10.12.0.0/24 gives 10.12.0.251. Assigned (not discovered) so the IPs are known at plan time and stable by construction; keep it high, clear of DHCP-assigned task/ALB ENIs, and DIFFERENT from the e2e harness's fl_nlb_host_num (250) while both stacks share the app subnets (FLIP#829)." + type = number + default = 251 +} + variable "max_azs" { type = number } From d4cc04672cbb02dcf921ea04d14a31a4fba26729 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 01:46:42 +0100 Subject: [PATCH 13/41] feat(deploy/AWS): gate the workload CloudFront off on LZA (#749 WP3) The GRCLOUDFRONTVPCORIGIN SCP denies cloudfront:CreateVpcOrigin in LZA workload accounts -- deliberately: a workload VPC origin dials the ALB inside the VPC, bypassing the TGW + central firewall. On LZA the networking account's edge distribution (aicentre-lza-iac) is the front door: it serves the UI from aws_s3_bucket.flip_ui via cross-account OAC (the bucket policies now grant var.lza_web_edge_distribution_arn there) and relays /api/* through the firewall to the ALB, admitted by fl_ingress_lza.tf's ingress rule instead of the managed-SG rule here. Gated off on lza_managed_network: the VPC origin, the managed-SG lookup + ALB rule, and the distribution (moved blocks keep legacy states aligned). local.ui_origin resolves to var.lza_web_edge_domain on LZA. The WAF/OAC/function/response-policy components stay standing unused there to keep legacy churn minimal. Legacy prod/stag plans unchanged. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/cloudfront.tf | 51 ++++++++++++++++++++++++++---- deploy/providers/AWS/main.tf | 4 +-- deploy/providers/AWS/variables.tf | 12 +++++++ 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/deploy/providers/AWS/cloudfront.tf b/deploy/providers/AWS/cloudfront.tf index 7b2a46cec..c49a5c788 100644 --- a/deploy/providers/AWS/cloudfront.tf +++ b/deploy/providers/AWS/cloudfront.tf @@ -114,6 +114,14 @@ moved { ############################ resource "aws_cloudfront_vpc_origin" "flip_api" { + # Gated off on LZA (FLIP#749): the GRCLOUDFRONTVPCORIGIN SCP denies + # cloudfront:CreateVpcOrigin in workload accounts -- deliberately, since a + # VPC origin reaches the ALB inside the VPC and bypasses the TGW + central + # firewall. On LZA the networking account's edge distribution is the front + # door (aicentre-lza-iac); fl_ingress_lza.tf admits its relay path onto the + # ALB instead of the SG rule below. + count = var.lza_managed_network ? 0 : 1 + vpc_origin_endpoint_config { # CloudFront VPC origin names accept only alphanumerics, dashes, and # underscores β€” the subdomain contains dots, so replace them with dashes. @@ -148,6 +156,7 @@ resource "aws_cloudfront_vpc_origin" "flip_api" { # checks against the service-managed SG (or the CloudFront managed prefix # list), not the ENI source IP. data "aws_security_group" "cloudfront_vpcorigins_service" { + count = var.lza_managed_network ? 0 : 1 name = "CloudFront-VPCOrigins-Service-SG" vpc_id = local.vpc_id @@ -160,13 +169,32 @@ data "aws_security_group" "cloudfront_vpcorigins_service" { # lookup needs the VPC origin. Attaching the rule outside the module keeps the # chain linear. resource "aws_security_group_rule" "alb_ingress_https_from_cloudfront" { + count = var.lza_managed_network ? 0 : 1 description = "HTTPS from the CloudFront-VPCOrigins-Service-SG (Option 2 in AWS VPC origins docs)" type = "ingress" from_port = var.ALB_HTTPS_PORT to_port = var.ALB_HTTPS_PORT protocol = "tcp" security_group_id = module.alb_security_group.security_group.id - source_security_group_id = data.aws_security_group.cloudfront_vpcorigins_service.id + source_security_group_id = data.aws_security_group.cloudfront_vpcorigins_service[0].id +} + +# State migration for the counts added above (FLIP#749 WP3): keeps existing +# legacy states aligned without a manual `terraform state mv`. Safe to remove +# once every live state file has been migrated. +moved { + from = aws_cloudfront_vpc_origin.flip_api + to = aws_cloudfront_vpc_origin.flip_api[0] +} + +moved { + from = aws_security_group_rule.alb_ingress_https_from_cloudfront + to = aws_security_group_rule.alb_ingress_https_from_cloudfront[0] +} + +moved { + from = aws_cloudfront_distribution.flip_ui + to = aws_cloudfront_distribution.flip_ui[0] } ############################ @@ -398,7 +426,7 @@ resource "aws_s3_bucket_policy" "demo_assets" { Resource = "${data.aws_s3_bucket.demo_assets[0].arn}/ark_demo/assets/*" Condition = { StringEquals = { - "AWS:SourceArn" = aws_cloudfront_distribution.flip_ui.arn + "AWS:SourceArn" = var.lza_managed_network ? var.lza_web_edge_distribution_arn : aws_cloudfront_distribution.flip_ui[0].arn } } }] @@ -468,7 +496,9 @@ locals { # (FLIP#749) it is the CloudFront default domain, so uploads/downloads and # sign-in keep working before any DNS exists. No dependency cycle: the # distribution references neither the app buckets' CORS nor Cognito. - ui_origin = var.manage_dns ? "https://${var.flip_alb_subdomain}" : "https://${aws_cloudfront_distribution.flip_ui.domain_name}" + # On LZA the workload distribution is gated off -- the UI origin is the + # networking account's edge distribution (FLIP#749 WP3). + ui_origin = var.manage_dns ? "https://${var.flip_alb_subdomain}" : var.lza_managed_network ? "https://${var.lza_web_edge_domain}" : "https://${aws_cloudfront_distribution.flip_ui[0].domain_name}" } # Custom origin-request policy for /api/*. The managed AllViewer policy @@ -907,6 +937,13 @@ resource "aws_cloudfront_response_headers_policy" "flip_api" { } resource "aws_cloudfront_distribution" "flip_ui" { + # Gated off on LZA with the VPC origin above: the networking account's edge + # distribution serves the UI (cross-account OAC on aws_s3_bucket.flip_ui) + # and relays /api/* -- see aicentre-lza-iac. The WAF/OAC/function/response + # policies below stay standing unused on LZA to keep this diff and the + # legacy state churn minimal; the edge carries its own WAF. + count = var.lza_managed_network ? 0 : 1 + enabled = true is_ipv6_enabled = true http_version = "http2" @@ -936,7 +973,7 @@ resource "aws_cloudfront_distribution" "flip_ui" { origin_id = "alb-api-origin" vpc_origin_config { - vpc_origin_id = aws_cloudfront_vpc_origin.flip_api.id + vpc_origin_id = aws_cloudfront_vpc_origin.flip_api[0].id } } @@ -1068,7 +1105,7 @@ resource "aws_s3_bucket_policy" "flip_ui" { Resource = "${aws_s3_bucket.flip_ui.arn}/*" Condition = { StringEquals = { - "AWS:SourceArn" = aws_cloudfront_distribution.flip_ui.arn + "AWS:SourceArn" = var.lza_managed_network ? var.lza_web_edge_distribution_arn : aws_cloudfront_distribution.flip_ui[0].arn } } }, @@ -1109,12 +1146,12 @@ resource "aws_s3_bucket_logging" "flip_ui" { output "CloudfrontDistributionId" { description = "CloudFront distribution ID for flip-ui (used by make deploy-ui for cache invalidation)" - value = aws_cloudfront_distribution.flip_ui.id + value = one(aws_cloudfront_distribution.flip_ui[*].id) } output "CloudfrontDistributionDomain" { description = "CloudFront distribution CloudFront-assigned domain (*.cloudfront.net). Use for pre-cutover smoke tests." - value = aws_cloudfront_distribution.flip_ui.domain_name + value = one(aws_cloudfront_distribution.flip_ui[*].domain_name) } output "FlipUiBucketName" { diff --git a/deploy/providers/AWS/main.tf b/deploy/providers/AWS/main.tf index 2ae676b21..0701dd6d4 100644 --- a/deploy/providers/AWS/main.tf +++ b/deploy/providers/AWS/main.tf @@ -612,8 +612,8 @@ resource "aws_route53_record" "alb" { # (Resource is still named "alb" for TF-state backwards compatibility; a # rename would recreate the record. The alias target is now CloudFront.) alias { - name = aws_cloudfront_distribution.flip_ui.domain_name - zone_id = aws_cloudfront_distribution.flip_ui.hosted_zone_id + name = aws_cloudfront_distribution.flip_ui[0].domain_name + zone_id = aws_cloudfront_distribution.flip_ui[0].hosted_zone_id evaluate_target_health = false } } diff --git a/deploy/providers/AWS/variables.tf b/deploy/providers/AWS/variables.tf index e1dfa1810..606a71463 100644 --- a/deploy/providers/AWS/variables.tf +++ b/deploy/providers/AWS/variables.tf @@ -54,6 +54,18 @@ variable "lza_fl_nlb_host_num" { default = 251 } +variable "lza_web_edge_domain" { + description = "Domain name of the networking account's edge CloudFront distribution -- the user-facing front door on LZA (FLIP#749 WP3; the workload distribution is gated off there). Used for local.ui_origin (CORS / UI origin URL). LZA-only." + type = string + default = "" +} + +variable "lza_web_edge_distribution_arn" { + description = "ARN of the networking account's edge CloudFront distribution, granted s3:GetObject on the flip-ui (and demo-assets) buckets via cross-account OAC on LZA. LZA-only -- legacy grants the in-account distribution instead." + type = string + default = "" +} + variable "max_azs" { type = number } From 1fa57391b9469411eb78248b345a5c7e4887b88f Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 09:54:39 +0100 Subject: [PATCH 14/41] fix(flip-api): never re-resolve the venv at boot (UV_NO_SYNC) The image bakes the venv at build time (uv sync --locked), but every 'uv run' in the entrypoint re-checked the project at boot and rebuilt its editable install -- fetching the setuptools build backend from PyPI each start. Legacy tolerated the wasted fetch; the egress-less LZA account (FLIP#749) cannot reach PyPI at all, so the Alembic step timed out and crash-looped the service (311 failed tasks overnight). UV_NO_SYNC makes the baked venv authoritative on every path; dependency changes already ship by image rebuild (BUILD=true in dev), so the contract is unchanged. Signed-off-by: at24_bioeng625-pc --- flip-api/entrypoint.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/flip-api/entrypoint.sh b/flip-api/entrypoint.sh index 016cdd798..0674c7048 100755 --- a/flip-api/entrypoint.sh +++ b/flip-api/entrypoint.sh @@ -17,6 +17,15 @@ DEBUG=${DEBUG:-false} ENV=${ENV:-development} +# The venv is baked at image build time (uv sync --locked in the Dockerfile). +# Never re-resolve it at boot: uv would otherwise try to rebuild the project's +# editable install, which fetches its build backend (setuptools) from PyPI -- +# on the egress-less LZA account (FLIP#749) that hangs and kills every boot, +# and on legacy it silently re-downloads setuptools each start. Dependency +# changes ship by rebuilding the image (BUILD=true in dev), never at boot. +UV_NO_SYNC=1 +export UV_NO_SYNC + echo "πŸš€ Starting flip-api entrypoint script..." echo "🌍 Environment: $ENV" echo "πŸ› Debug mode: $DEBUG" From 4f914e5f5e60cf641fc1fdb1eb3c4a46d352d12d Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 09:55:59 +0100 Subject: [PATCH 15/41] feat(deploy/AWS): skip the CloudFront invalidation in deploy-ui on LZA No workload distribution exists there (gated off; the networking account's edge distribution serves the bucket) and no cross-account invalidation permission is wanted -- by design (FLIP#749): the hashed bundles are immutable and index.html/window.js already carry no-cache Cache-Control, so a UI deploy is visible on the next index.html fetch with no invalidation and zero networking-account involvement. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deploy/providers/AWS/Makefile b/deploy/providers/AWS/Makefile index b952c1221..5bf2eb717 100644 --- a/deploy/providers/AWS/Makefile +++ b/deploy/providers/AWS/Makefile @@ -807,12 +807,21 @@ deploy-ui: ## Build flip-ui from the working tree and deploy it to S3 + invalida $(_AWS_ENV) aws s3 cp ../../../flip-ui/dist/js/window.js s3://$$UI_BUCKET/js/window.js \ --cache-control "no-cache, no-store, must-revalidate" \ --content-type "application/javascript; charset=utf-8" + @# On LZA there is no workload distribution to invalidate (cloudfront.tf gates + @# it off; the networking account's edge distribution serves the bucket) and no + @# cross-account invalidation permission -- BY DESIGN (FLIP#749): the hashed + @# bundles are immutable and index.html/window.js carry no-cache metadata, so a + @# deploy is visible on the next index.html fetch with no invalidation at all. +ifeq ($(PROD),lza) + @echo "🧹 Skipping CloudFront invalidation on LZA (edge distribution honours the Cache-Control metadata)." +else @echo "🧹 Invalidating CloudFront..." @DIST_ID=$$($(_AWS_ENV) terraform output -raw CloudfrontDistributionId); \ [ -n "$$DIST_ID" ] || { echo "❌ Terraform output CloudfrontDistributionId is empty"; exit 1; }; \ $(_AWS_ENV) aws cloudfront create-invalidation \ --distribution-id "$$DIST_ID" \ --paths "/*" >/dev/null +endif @echo "βœ… deploy-ui complete." .PHONY: deploy-ark-demo From 52bfb5c0fa60bd186331ac8e19fbd162dfe5d473 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 10:12:46 +0100 Subject: [PATCH 16/41] fix(flip-api): put src/ on PYTHONPATH now the boot-time sync is gone UV_NO_SYNC exposed that nothing ever baked the flip_api package into the venv: the Dockerfile syncs dependencies before COPY src/, and the boot-time 'uv run' re-sync (the PyPI fetch the previous commit removed) was what installed the project editable on every start. PYTHONPATH makes the src layout importable directly -- one mechanism for prod images and dev bind-mounts alike, with no boot-time resolution. Signed-off-by: at24_bioeng625-pc --- flip-api/entrypoint.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flip-api/entrypoint.sh b/flip-api/entrypoint.sh index 0674c7048..eb78d314e 100755 --- a/flip-api/entrypoint.sh +++ b/flip-api/entrypoint.sh @@ -26,6 +26,13 @@ ENV=${ENV:-development} UV_NO_SYNC=1 export UV_NO_SYNC +# With the boot-time sync gone, nothing installs the flip_api package into the +# venv (the Dockerfile syncs dependencies before src/ exists, and dev +# bind-mounts src/ over the image). Put the src layout on the import path +# directly -- one mechanism for prod images and dev mounts alike. +PYTHONPATH="/app/src${PYTHONPATH:+:$PYTHONPATH}" +export PYTHONPATH + echo "πŸš€ Starting flip-api entrypoint script..." echo "🌍 Environment: $ENV" echo "πŸ› Debug mode: $DEBUG" From f05c8e62bcff3cf52875a9c0a1c7288f54116165 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 10:36:28 +0100 Subject: [PATCH 17/41] fix(deploy/AWS): classic hosted UI on LZA -- Managed Login blocks PrivateLink Cognito refuses interface-endpoint (PrivateLink) API access to user pools with Managed Login (v2) configured, and the egress-less LZA account's only route to cognito-idp is the central interface endpoint -- every boot-time Cognito call failed with InvalidParameterException and flip-api could not start. The hosted-UI generation is cosmetic for FLIP (the UI signs in via the SDK), so the LZA mode runs the classic hosted UI (v1); the single-account mode keeps Managed Login (v2), unchanged. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/modules/cognito/main.tf | 2 +- deploy/providers/AWS/modules/cognito/variables.tf | 6 ++++++ deploy/providers/AWS/services.tf | 15 ++++++++++----- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/deploy/providers/AWS/modules/cognito/main.tf b/deploy/providers/AWS/modules/cognito/main.tf index ed76212e5..83234a036 100644 --- a/deploy/providers/AWS/modules/cognito/main.tf +++ b/deploy/providers/AWS/modules/cognito/main.tf @@ -113,7 +113,7 @@ resource "random_string" "cognito_domain" { resource "aws_cognito_user_pool_domain" "main" { domain = random_string.cognito_domain.result user_pool_id = aws_cognito_user_pool.flip_user_pool.id - managed_login_version = 2 + managed_login_version = var.managed_login_version lifecycle { prevent_destroy = true diff --git a/deploy/providers/AWS/modules/cognito/variables.tf b/deploy/providers/AWS/modules/cognito/variables.tf index a965cdcb5..3538188e5 100644 --- a/deploy/providers/AWS/modules/cognito/variables.tf +++ b/deploy/providers/AWS/modules/cognito/variables.tf @@ -70,3 +70,9 @@ variable "mfa_configuration" { error_message = "mfa_configuration must be one of: OFF, OPTIONAL, ON." } } + +variable "managed_login_version" { + description = "Hosted-UI generation for the user pool domain: 2 = Managed Login (branding designer), 1 = classic hosted UI. Cognito refuses PrivateLink (interface-endpoint) API access to pools with Managed Login configured, so the LZA mode -- whose only route to cognito-idp is the central interface endpoint -- must run 1 (FLIP#749). FLIP's own UI performs SDK-based sign-in, so the hosted-UI generation is cosmetic." + type = number + default = 2 +} diff --git a/deploy/providers/AWS/services.tf b/deploy/providers/AWS/services.tf index bb136f01c..52c5496c8 100644 --- a/deploy/providers/AWS/services.tf +++ b/deploy/providers/AWS/services.tf @@ -199,11 +199,16 @@ module "cognito" { # (cloudfront.tf) so a zone-less bring-up (FLIP#749) sends the reachable # CloudFront default domain rather than a dead subdomain. Identical to # var.flip_alb_subdomain whenever DNS is managed (all legacy envs). - sign_in_hostname = trimprefix(local.ui_origin, "https://") - admin_email = var.flip_cognito_admin_email - researcher_email = var.flip_cognito_researcher_email - seed_user_password = var.ADMIN_USER_PASSWORD - templates_dir = "${path.module}/templates/cognito" + sign_in_hostname = trimprefix(local.ui_origin, "https://") + # Managed Login (v2) blocks PrivateLink access to cognito-idp, and the LZA + # account's ONLY path to Cognito is the central interface endpoint -- flip-api + # cannot boot against a v2 pool there (FLIP#749). Cosmetic either way: the + # FLIP UI signs in via the SDK, not the hosted UI. + managed_login_version = var.lza_managed_network ? 1 : 2 + admin_email = var.flip_cognito_admin_email + researcher_email = var.flip_cognito_researcher_email + seed_user_password = var.ADMIN_USER_PASSWORD + templates_dir = "${path.module}/templates/cognito" # The UI uses USER_SRP_AUTH (Cognito's native auth flow, not OAuth2 redirect), # so callback_urls are hygiene only β€” keep the canonical UI origin + localhost # for dev. From 6aa2e5ea97cde4f95918dfd9a25bdbd6a8498fae Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 11:01:42 +0100 Subject: [PATCH 18/41] fix(deploy/AWS): pin the LZA user pool to the LITE tier The v1-domain downgrade wasn't enough: new pools default to the ESSENTIALS tier, and the tier itself is what Cognito's PrivateLink gate reads as 'ManagedLogin configured' -- flip-api's startup DescribeUserPoolClient kept failing through the interface endpoint even with the classic hosted UI. LITE matches what the grandfathered legacy pools effectively run and covers every flow FLIP uses (SDK sign-in, TOTP MFA, admin APIs). Null on legacy: the argument is never sent, so existing pools keep their tier and the legacy plan is unchanged. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/modules/cognito/main.tf | 8 ++++++++ deploy/providers/AWS/modules/cognito/variables.tf | 6 ++++++ deploy/providers/AWS/services.tf | 1 + 3 files changed, 15 insertions(+) diff --git a/deploy/providers/AWS/modules/cognito/main.tf b/deploy/providers/AWS/modules/cognito/main.tf index 83234a036..89508ea1e 100644 --- a/deploy/providers/AWS/modules/cognito/main.tf +++ b/deploy/providers/AWS/modules/cognito/main.tf @@ -15,6 +15,14 @@ # FLIP Cognito user pool + client + domain + seed users. resource "aws_cognito_user_pool" "flip_user_pool" { + # Feature tier. Null = never sent (legacy pools keep their grandfathered + # tier). The LZA mode pins LITE: ESSENTIALS -- the default for new pools -- + # counts as "ManagedLogin configured", which Cognito refuses to serve over + # PrivateLink, and the interface endpoint is that account's only route to + # cognito-idp (FLIP#749). FLIP uses classic flows (SDK sign-in, TOTP MFA), + # all LITE features. + user_pool_tier = var.user_pool_tier + name = var.user_pool_name username_attributes = ["email"] auto_verified_attributes = ["email"] diff --git a/deploy/providers/AWS/modules/cognito/variables.tf b/deploy/providers/AWS/modules/cognito/variables.tf index 3538188e5..67c6c380a 100644 --- a/deploy/providers/AWS/modules/cognito/variables.tf +++ b/deploy/providers/AWS/modules/cognito/variables.tf @@ -76,3 +76,9 @@ variable "managed_login_version" { type = number default = 2 } + +variable "user_pool_tier" { + description = "Cognito feature tier for the pool (LITE | ESSENTIALS | PLUS). Null omits the argument so existing pools keep their current tier. See the resource comment for why the LZA mode requires LITE." + type = string + default = null +} diff --git a/deploy/providers/AWS/services.tf b/deploy/providers/AWS/services.tf index 52c5496c8..085de9d3c 100644 --- a/deploy/providers/AWS/services.tf +++ b/deploy/providers/AWS/services.tf @@ -205,6 +205,7 @@ module "cognito" { # cannot boot against a v2 pool there (FLIP#749). Cosmetic either way: the # FLIP UI signs in via the SDK, not the hosted UI. managed_login_version = var.lza_managed_network ? 1 : 2 + user_pool_tier = var.lza_managed_network ? "LITE" : null admin_email = var.flip_cognito_admin_email researcher_email = var.flip_cognito_researcher_email seed_user_password = var.ADMIN_USER_PASSWORD From 0f41212e08fc1705b5936210fa4847ba10eeac4b Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 11:35:21 +0100 Subject: [PATCH 19/41] fix(deploy/AWS): no hosted-UI domain on LZA -- it latches the PrivateLink block Empirical result (2026-08-20): a user pool domain that has ever had Managed Login keeps the pool in 'ManagedLogin configured' through an in-place downgrade to v1 -- ListUsers via the interface endpoint kept returning the ManagedLogin refusal with the domain present at LITE+v1 and returned 200 the moment the domain was deleted. FLIP never uses the hosted UI (SDK sign-in), so the LZA mode simply creates no domain; legacy keeps domain + Managed Login v2, unchanged (moved block covers the count). Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/modules/cognito/main.tf | 8 ++++++++ .../providers/AWS/modules/cognito/outputs.tf | 2 +- .../providers/AWS/modules/cognito/variables.tf | 6 ++++++ deploy/providers/AWS/services.tf | 18 ++++++++++++------ 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/deploy/providers/AWS/modules/cognito/main.tf b/deploy/providers/AWS/modules/cognito/main.tf index 89508ea1e..f4ee8c041 100644 --- a/deploy/providers/AWS/modules/cognito/main.tf +++ b/deploy/providers/AWS/modules/cognito/main.tf @@ -119,6 +119,14 @@ resource "random_string" "cognito_domain" { } resource "aws_cognito_user_pool_domain" "main" { + # Gated off on LZA (FLIP#749): FLIP never uses the hosted UI (the app signs + # in via the SDK), and a domain object born with Managed Login keeps its + # "ManagedLogin configured" state through an in-place downgrade to v1 -- + # Cognito then refuses PrivateLink API access to the whole pool, which is + # the LZA account's only route to cognito-idp. Proven empirically 2026-08-20: + # ListUsers through the interface endpoint returned the ManagedLogin error + # with the domain present (any version) and 200 once it was deleted. + count = var.create_hosted_ui_domain ? 1 : 0 domain = random_string.cognito_domain.result user_pool_id = aws_cognito_user_pool.flip_user_pool.id managed_login_version = var.managed_login_version diff --git a/deploy/providers/AWS/modules/cognito/outputs.tf b/deploy/providers/AWS/modules/cognito/outputs.tf index 7231abc2a..1605cb9cf 100644 --- a/deploy/providers/AWS/modules/cognito/outputs.tf +++ b/deploy/providers/AWS/modules/cognito/outputs.tf @@ -29,5 +29,5 @@ output "app_client_id" { output "domain" { description = "Cognito hosted-UI subdomain (the random 8-char prefix, not the full URL)" - value = aws_cognito_user_pool_domain.main.domain + value = one(aws_cognito_user_pool_domain.main[*].domain) } diff --git a/deploy/providers/AWS/modules/cognito/variables.tf b/deploy/providers/AWS/modules/cognito/variables.tf index 67c6c380a..8954d86a3 100644 --- a/deploy/providers/AWS/modules/cognito/variables.tf +++ b/deploy/providers/AWS/modules/cognito/variables.tf @@ -82,3 +82,9 @@ variable "user_pool_tier" { type = string default = null } + +variable "create_hosted_ui_domain" { + description = "Whether to create the hosted-UI user pool domain. False on LZA -- see the resource comment: a domain that has ever had Managed Login blocks PrivateLink access to the pool, and FLIP's flows never use the hosted UI." + type = bool + default = true +} diff --git a/deploy/providers/AWS/services.tf b/deploy/providers/AWS/services.tf index 085de9d3c..6ff1c683e 100644 --- a/deploy/providers/AWS/services.tf +++ b/deploy/providers/AWS/services.tf @@ -204,12 +204,13 @@ module "cognito" { # account's ONLY path to Cognito is the central interface endpoint -- flip-api # cannot boot against a v2 pool there (FLIP#749). Cosmetic either way: the # FLIP UI signs in via the SDK, not the hosted UI. - managed_login_version = var.lza_managed_network ? 1 : 2 - user_pool_tier = var.lza_managed_network ? "LITE" : null - admin_email = var.flip_cognito_admin_email - researcher_email = var.flip_cognito_researcher_email - seed_user_password = var.ADMIN_USER_PASSWORD - templates_dir = "${path.module}/templates/cognito" + managed_login_version = var.lza_managed_network ? 1 : 2 + user_pool_tier = var.lza_managed_network ? "LITE" : null + create_hosted_ui_domain = !var.lza_managed_network + admin_email = var.flip_cognito_admin_email + researcher_email = var.flip_cognito_researcher_email + seed_user_password = var.ADMIN_USER_PASSWORD + templates_dir = "${path.module}/templates/cognito" # The UI uses USER_SRP_AUTH (Cognito's native auth flow, not OAuth2 redirect), # so callback_urls are hygiene only β€” keep the canonical UI origin + localhost # for dev. @@ -238,6 +239,11 @@ moved { to = module.cognito.aws_cognito_user_pool_domain.main } +moved { + from = module.cognito.aws_cognito_user_pool_domain.main + to = module.cognito.aws_cognito_user_pool_domain.main[0] +} + moved { from = aws_cognito_user_pool_client.client to = module.cognito.aws_cognito_user_pool_client.client From 16c0edc95cd86fc24aa213bf97c78794615be9b5 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 11:39:27 +0100 Subject: [PATCH 20/41] fix(deploy/AWS): admit the VPC CIDR on the LZA ALB web rule The FL NLB SG already concats the VPC CIDR with the networking ingress CIDRs; the ALB rule only admitted the relay path, so in-VPC callers (probe verification, internal consumers) timed out against a healthy ALB. Same shape on both now, and the rule no longer vanishes when the ingress CIDR list is empty. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/fl_ingress_lza.tf | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/deploy/providers/AWS/fl_ingress_lza.tf b/deploy/providers/AWS/fl_ingress_lza.tf index 5aed0b51b..21a70e4f3 100644 --- a/deploy/providers/AWS/fl_ingress_lza.tf +++ b/deploy/providers/AWS/fl_ingress_lza.tf @@ -180,13 +180,15 @@ resource "aws_lb_target_group" "ecs_fl_server_tcp_lza" { # CloudFront through the SG-drift stack instead; empty ingress CIDRs create # nothing, so the stack still applies standalone. resource "aws_security_group_rule" "alb_ingress_web_from_networking" { - count = var.lza_managed_network && length(var.networking_ingress_cidrs) > 0 ? 1 : 0 - type = "ingress" - description = "Web from the networking-account relay NLB path (CloudFront VPC origin)" - protocol = "tcp" - from_port = var.ALB_HTTPS_PORT - to_port = var.ALB_HTTPS_PORT - cidr_blocks = var.networking_ingress_cidrs + count = var.lza_managed_network ? 1 : 0 + type = "ingress" + description = "Web from the networking-account relay NLB path (CloudFront VPC origin) and VPC-internal callers" + protocol = "tcp" + from_port = var.ALB_HTTPS_PORT + to_port = var.ALB_HTTPS_PORT + # VPC CIDR: in-VPC verification (probe curls) and internal callers -- the + # same shape as the FL NLB SG above. The networking CIDRs are the relay path. + cidr_blocks = concat([local.vpc_cidr_block], var.networking_ingress_cidrs) security_group_id = module.alb_security_group.security_group.id } From f647cd49b3424ffad1a1c9ba1c41031dcc44076c Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 12:50:20 +0100 Subject: [PATCH 21/41] fix(deploy/AWS): admit the internal FL NLB onto the fl-server task SG The legacy nlb-grpc admission rule references the legacy NLB's SG and is gated off on LZA, leaving the fl-server task with no ingress on its FL port -- the internal NLB's health checks could never pass and ECS churned unhealthy targets. Mirror the legacy pattern against the internal NLB's SG, LZA-gated like the rest of fl_ingress_lza.tf. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/fl_ingress_lza.tf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/deploy/providers/AWS/fl_ingress_lza.tf b/deploy/providers/AWS/fl_ingress_lza.tf index 21a70e4f3..a183b619f 100644 --- a/deploy/providers/AWS/fl_ingress_lza.tf +++ b/deploy/providers/AWS/fl_ingress_lza.tf @@ -226,3 +226,21 @@ resource "aws_ssm_parameter" "lza_web_port" { type = "String" value = tostring(var.ALB_HTTPS_PORT) } + +# Task-side admission for the internal NLB: the legacy +# ecs_fl_server_ingress_nlb_grpc rule (ecs_sg.tf) references the legacy NLB's +# SG and is gated off on LZA, which left the fl-server task admitting nothing +# on its FL port -- the NLB's health checks parked the target at unhealthy and +# ECS churned replacements forever. +resource "aws_security_group_rule" "ecs_fl_server_ingress_internal_nlb" { + count = var.lza_managed_network ? 1 : 0 + type = "ingress" + description = "gRPC from the internal FL NLB (edge-relayed FL clients + health checks)" + # Backend-dependent container port (Flower: SuperLink Fleet 9092) -- the + # NLB forwards its FL_SERVER_PORT listener here. + from_port = local.fl_server_container_port + to_port = local.fl_server_container_port + protocol = "tcp" + source_security_group_id = module.fl_internal_nlb_security_group[0].security_group.id + security_group_id = aws_security_group.ecs_fl_server.id +} From 2dcbaa117165b49c72bcd966f92d8663cd45e456 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 18:05:58 +0100 Subject: [PATCH 22/41] fix(lza): pin fl-server-net-1 to the internal FL NLB static IPs for fl-api The NVFLARE admin kit targets the bare host fl-server-net-1 (a SAN on the server cert). Legacy resolves it via the flip.local DHCP search domain (dhcp.tf), which the LZA-managed VPC cannot carry - Fargate offers no search domains - so fl-api's admin session died at boot on LZA (ClientConnectorDNSError), the app failed its lifespan startup, and the hub saw connection-refused on every net-status call (FL nets card never loaded). extraHosts pins the name to the internal FL NLB's deterministic subnet_mapping IPs; the merge is empty on legacy so its container definition stays byte-identical. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/ecs_tasks.tf | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/deploy/providers/AWS/ecs_tasks.tf b/deploy/providers/AWS/ecs_tasks.tf index f3527a208..cfd1c1f01 100644 --- a/deploy/providers/AWS/ecs_tasks.tf +++ b/deploy/providers/AWS/ecs_tasks.tf @@ -150,7 +150,7 @@ resource "aws_ecs_task_definition" "fl_api_net_1" { task_role_arn = aws_iam_role.ecs_fl_api_task.arn container_definitions = jsonencode([ - { + merge({ name = "fl-api-net-1" image = local.fl_api_image cpu = 1024 @@ -229,7 +229,19 @@ resource "aws_ecs_task_definition" "fl_api_net_1" { retries = 3 startPeriod = 120 } - } + }, + # The NVFLARE admin kit (provisioned from net-1_project_prod.yml) targets the + # bare host `fl-server-net-1` β€” a SAN on the server cert. On legacy that name + # resolves via the flip.local DHCP search domain (dhcp.tf), which the + # LZA-managed VPC cannot carry, so pin it to the internal FL NLB's static IPs + # (deterministic by subnet_mapping host_num). Empty merge on legacy keeps its + # container definition byte-identical. + var.lza_managed_network ? { + extraHosts = [ + for ip in local.lza_fl_nlb_private_ips : { hostname = "fl-server-net-1", ipAddress = ip } + ] + } : {} + ) ]) dynamic "volume" { From fe6d935cf08061ba61cac123bf33339df2413b81 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 18:10:43 +0100 Subject: [PATCH 23/41] Revert "fix(lza): pin fl-server-net-1 to the internal FL NLB static IPs for fl-api" This reverts commit 2dcbaa117165b49c72bcd966f92d8663cd45e456. --- deploy/providers/AWS/ecs_tasks.tf | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/deploy/providers/AWS/ecs_tasks.tf b/deploy/providers/AWS/ecs_tasks.tf index cfd1c1f01..f3527a208 100644 --- a/deploy/providers/AWS/ecs_tasks.tf +++ b/deploy/providers/AWS/ecs_tasks.tf @@ -150,7 +150,7 @@ resource "aws_ecs_task_definition" "fl_api_net_1" { task_role_arn = aws_iam_role.ecs_fl_api_task.arn container_definitions = jsonencode([ - merge({ + { name = "fl-api-net-1" image = local.fl_api_image cpu = 1024 @@ -229,19 +229,7 @@ resource "aws_ecs_task_definition" "fl_api_net_1" { retries = 3 startPeriod = 120 } - }, - # The NVFLARE admin kit (provisioned from net-1_project_prod.yml) targets the - # bare host `fl-server-net-1` β€” a SAN on the server cert. On legacy that name - # resolves via the flip.local DHCP search domain (dhcp.tf), which the - # LZA-managed VPC cannot carry, so pin it to the internal FL NLB's static IPs - # (deterministic by subnet_mapping host_num). Empty merge on legacy keeps its - # container definition byte-identical. - var.lza_managed_network ? { - extraHosts = [ - for ip in local.lza_fl_nlb_private_ips : { hostname = "fl-server-net-1", ipAddress = ip } - ] - } : {} - ) + } ]) dynamic "volume" { From 646a3f8ef846bc0ddb9d05dd8bd8adac0f9ba720 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 18:11:22 +0100 Subject: [PATCH 24/41] fix(lza): resolve the admin kit's bare fl-server-net-1 via a private hosted zone Fargate's awsvpc network mode rejects extraHosts (the first attempt, reverted), so publish the bare name the NVFLARE admin kit targets as a single-label private hosted zone on the workload VPC, apex A records pointing at the internal FL NLB's deterministic static IPs. Legacy keeps resolving it via the flip.local DHCP search domain; the zone is LZA-gated. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/fl_ingress_lza.tf | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/deploy/providers/AWS/fl_ingress_lza.tf b/deploy/providers/AWS/fl_ingress_lza.tf index a183b619f..44d83e635 100644 --- a/deploy/providers/AWS/fl_ingress_lza.tf +++ b/deploy/providers/AWS/fl_ingress_lza.tf @@ -244,3 +244,30 @@ resource "aws_security_group_rule" "ecs_fl_server_ingress_internal_nlb" { source_security_group_id = module.fl_internal_nlb_security_group[0].security_group.id security_group_id = aws_security_group.ecs_fl_server.id } + +# The NVFLARE admin kit (provisioned from net-1_project_prod.yml) targets the +# bare host `fl-server-net-1` β€” a SAN on the server cert alongside the public +# FQDN. Legacy resolves it through the flip.local DHCP search domain +# (dhcp.tf), which the LZA-managed VPC cannot carry, and Fargate's awsvpc +# network mode rejects extraHosts β€” so publish the bare name as a +# single-label private hosted zone whose apex A records are the internal FL +# NLB's deterministic static IPs. A bare-name lookup exhausts the VPC search +# domains and then matches this zone's apex, landing on the NLB β†’ fl-server. +resource "aws_route53_zone" "fl_server_bare_name" { + count = var.lza_managed_network ? 1 : 0 + name = "fl-server-net-1" + comment = "LZA: resolve the NVFLARE admin kit's bare fl-server host inside the workload VPC (FLIP#749)" + + vpc { + vpc_id = local.vpc_id + } +} + +resource "aws_route53_record" "fl_server_bare_name_apex" { + count = var.lza_managed_network ? 1 : 0 + zone_id = aws_route53_zone.fl_server_bare_name[0].zone_id + name = "fl-server-net-1" + type = "A" + ttl = 60 + records = local.lza_fl_nlb_private_ips +} From 07163fa4556248d5402247d4c8c5f6f7d4ab3f5d Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Thu, 20 Aug 2026 20:18:09 +0100 Subject: [PATCH 25/41] fix(lza): codify the accelerator-managed ALB access logging The LZA guardrail enables ALB access logging to the LogArchive account's central bucket out-of-band, so every plan proposed reverting enabled=true. Mirror it behind var.lza_elb_access_logs_bucket (empty default keeps the access_logs block omitted and legacy plans byte-identical). Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/main.tf | 8 ++++++++ deploy/providers/AWS/variables.tf | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/deploy/providers/AWS/main.tf b/deploy/providers/AWS/main.tf index 0701dd6d4..bad3f225a 100644 --- a/deploy/providers/AWS/main.tf +++ b/deploy/providers/AWS/main.tf @@ -482,6 +482,14 @@ module "alb" { security_groups = [module.alb_security_group.security_group.id] enable_deletion_protection = false + # LZA (FLIP#749): the accelerator guardrail enables access logging to the + # LogArchive bucket out-of-band; mirror it so Terraform stops reverting it. + access_logs = var.lza_elb_access_logs_bucket != "" ? { + enabled = true + bucket = var.lza_elb_access_logs_bucket + prefix = "${data.aws_caller_identity.current.account_id}/elb-flip-alb" + } : {} + # The main listener's protocol depends on DNS availability (FLIP#749): with a # hosted zone it terminates HTTPS with the DNS-validated ACM cert β€” the # canonical shape, unchanged for legacy prod/stag. Without one diff --git a/deploy/providers/AWS/variables.tf b/deploy/providers/AWS/variables.tf index 606a71463..f04d3aa06 100644 --- a/deploy/providers/AWS/variables.tf +++ b/deploy/providers/AWS/variables.tf @@ -459,3 +459,16 @@ variable "ecs_exec_enabled" { type = bool default = false } + +variable "lza_elb_access_logs_bucket" { + description = <<-EOT + Name of the LZA LogArchive account's central ELB access-logs bucket. On the + LZA estate the accelerator's guardrail auto-enables ALB access logging to + this bucket out-of-band; setting it here codifies that state so plans stop + proposing to disable platform-managed logging. Empty (the default, and the + only valid value on legacy) omits the access_logs block entirely, keeping + legacy plans byte-identical. + EOT + type = string + default = "" +} From f53e81ce18de3a883ed130c5eb3bed5476a39515 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Fri, 21 Aug 2026 10:33:19 +0100 Subject: [PATCH 26/41] fix(deploy/AWS): access_logs 'off' is null, not {} -- empty map broke every legacy plan The ALB module types access_logs as an object with a required bucket attribute (default null), so the {} passed on the legacy branch of the conditional failed type conversion and errored the whole prod/stag plan. Caught by a live stag plan from this branch; with null it plans clean -- only develop-ahead drift and the out-of-band ECS Exec enables remain, and the sg-drift moved blocks land as pure renames. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/main.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/providers/AWS/main.tf b/deploy/providers/AWS/main.tf index bad3f225a..0ef2fc39d 100644 --- a/deploy/providers/AWS/main.tf +++ b/deploy/providers/AWS/main.tf @@ -484,11 +484,13 @@ module "alb" { # LZA (FLIP#749): the accelerator guardrail enables access logging to the # LogArchive bucket out-of-band; mirror it so Terraform stops reverting it. + # The module's access_logs object requires `bucket`, so "no logging" must be + # null (its own default) β€” an empty map fails type conversion on legacy. access_logs = var.lza_elb_access_logs_bucket != "" ? { enabled = true bucket = var.lza_elb_access_logs_bucket prefix = "${data.aws_caller_identity.current.account_id}/elb-flip-alb" - } : {} + } : null # The main listener's protocol depends on DNS availability (FLIP#749): with a # hosted zone it terminates HTTPS with the DNS-validated ACM cert β€” the From 0d6af849ca8c65b56011dd49e74fdae8d0d9b95e Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Fri, 21 Aug 2026 11:09:47 +0100 Subject: [PATCH 27/41] docs(deploy/AWS): document the two-phase LZA edge wiring; refresh WP1-era state notes The lza_web_edge_domain/_arn variables deliberately carry no required-when-LZA validation: the networking-account edge stack is built from this stack's /flip/networking/* handoff outputs, so the first workload apply necessarily precedes the edge distribution -- empty values mean a grants-nobody bucket policy (fail-closed) and a placeholder ui_origin until the wiring re-apply. Also refresh the LZA section lines superseded by the WP3 commits on this branch (FL ingress is wired, the workload CloudFront is gated off in favour of the edge distribution). Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/README.md | 43 +++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index 9b91092f7..44b854509 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -672,14 +672,29 @@ configuration is identical to before β€” the legacy environments are never touch there can reach TGW/endpoints, and RDS doesn't need to); ECS tasks, the internal ALB, the RDS Proxy, EFS mount targets and the EC2 hosts go to the TGW-routed **app** subnets (they need the central endpoints / image pulls); - **gates off**: the SG-drift CloudTrailβ†’EventBridgeβ†’Lambda stack (`security.tf` β€” the org baseline of Control Tower - org trail, GuardDuty, Security Hub and Config covers it) and the public FL-server NLB + target group + DNS record + - SG rules (no IGW and VPC Block Public Access make an internet-facing NLB impossible; FL ingress instead comes - through the networking account's edge NLB over TGW β€” proven end-to-end in FLIP#829/PR#830, wired up for the real - fl-server as follow-up #749 work). The `fl-server-net-1` / `fl-api-net-1` ECS services themselves still deploy β€” - they just have no inbound FL path yet. - -Everything else (ECS Fargate, RDS + Proxy, Cognito, S3 + CMK, Secrets Manager, SES, EFS, Cloud Map, internal ALB, -CloudFront + WAF + ACM) remains FLIP-managed exactly as on legacy prod. + org trail, GuardDuty, Security Hub and Config covers it), the public FL-server NLB + target group + DNS record + + SG rules (no IGW and VPC Block Public Access make an internet-facing NLB impossible), and the workload CloudFront + distribution + its VPC origin (the `GRCLOUDFRONTVPCORIGIN` SCP denies VPC origins by design β€” a VPC origin dials + the ALB inside the VPC, bypassing the TGW + central firewall). Ingress instead rides the networking account's + two-tier edge (proven end-to-end in FLIP#829/PR#830 and now serving the real stack): the edge CloudFront serves + the UI bucket via cross-account OAC and relays `/api/*` to the internal ALB, and the edge NLB forwards FL traffic + over TGW to the internal FL NLB in `fl_ingress_lza.tf` (static per-subnet IPs the edge registers once as targets), + which fronts `fl-server-net-1`. + +Everything else (ECS Fargate, RDS + Proxy, Cognito, S3 + CMK, Secrets Manager, SES, EFS, Cloud Map, internal ALB) +remains FLIP-managed exactly as on legacy prod; the legacy WAF/OAC/CloudFront-function components stay standing +unused on LZA to keep legacy churn minimal. + +**Edge wiring is two-phase β€” by construction, not configuration.** The networking account's edge stack +([aicentre-lza-iac](https://github.com/londonaicentre/aicentre-lza-iac)) is built *from* this stack's outputs: the +first workload `apply` publishes the `/flip/networking/*` SSM handoff params (FL NLB private IPs + port, ALB DNS +name, web port) that the edge NLB and relay consume, so the workload account necessarily applies before the edge +distribution exists. On that first apply `TF_VAR_lza_web_edge_domain` and `TF_VAR_lza_web_edge_distribution_arn` +are still empty: the UI-bucket policy then grants no principal (fail-closed β€” the edge simply cannot read the +bucket yet) and `local.ui_origin` is a placeholder. Once the edge stack is up, set both values in `.env.lza-prod` +(the edge distribution's default domain and its ARN) and re-apply to grant the cross-account OAC read and point +bucket CORS + Cognito URLs at the edge domain. This ordering is why the two variables deliberately carry no +"required-when-LZA" validation β€” it would hard-fail the legitimate first apply. **Prerequisites (already provisioned out-of-band in the account, not Terraform-managed here):** @@ -744,12 +759,12 @@ kits and encrypted data stay valid) β€” never reuse the legacy Secrets Manager s `MANAGE_DNS=false` skips it plus every Route53 record and both DNS-validated ACM certs. Consequences, all of which revert by flipping `MANAGE_DNS=true` + `make plan`/`apply` once the zone lands: -- CloudFront serves on its default `*.cloudfront.net` domain with the default viewer certificate (no aliases β€” - CloudFront only allows the default cert when no aliases are set). `terraform output CloudfrontDistributionDomain` - prints the URL. -- The CloudFrontβ†’ALB `/api/*` origin leg runs **plain HTTP** over the private VPC-origin ENI (an ALB HTTPS listener - needs an ISSUED cert, and issuance needs DNS validation). Viewer traffic stays HTTPS. Accepted as a bring-up-only - limitation β€” the leg never leaves the VPC-origin path. +- The user-facing URL is the networking account's edge distribution on its default `*.cloudfront.net` domain + (`TF_VAR_lza_web_edge_domain`) β€” the workload distribution is gated off on LZA, so `terraform output + CloudfrontDistributionDomain` is null there by design. +- The edgeβ†’ALB `/api/*` relay leg terminates on a **plain-HTTP** ALB listener (an ALB HTTPS listener needs an + ISSUED cert, and issuance needs DNS validation). Viewer traffic stays HTTPS at the edge, and the relay leg never + leaves the TGW + central-firewall path. Accepted as a bring-up-only limitation. - Bucket CORS and the Cognito sign-in hostname/callback URLs follow the CloudFront default domain automatically (`local.ui_origin`), so uploads, downloads and sign-in work; `make deploy-ui` must generate `window.js` with `CENTRAL_HUB_API_URL` pointing at the CloudFront domain. From 7cd6b5b312df09d60c9a42138c61878c68e03c9d Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Fri, 21 Aug 2026 11:50:11 +0100 Subject: [PATCH 28/41] fix(fl-api-flower): drive the flwr CLI from the baked venv, not uvx uvx resolves its tool environment from PyPI at call time, so on the egress-less LZA account (FLIP#749) every flwr invocation -- run, list, stop, federation list -- timed out against pypi.org: submissions errored the model and status/reconcile went dark. flwr is already a direct dependency of this service's venv (on PATH in the image), so invoke that binary directly; no network at call time on any environment. Signed-off-by: at24_bioeng625-pc --- fl-services/flower/fl-api-flower/fl_api/app.py | 13 ++++++++----- fl-services/flower/fl-api-flower/tests/conftest.py | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/fl-services/flower/fl-api-flower/fl_api/app.py b/fl-services/flower/fl-api-flower/fl_api/app.py index 6488a72d2..f9b9f4463 100644 --- a/fl-services/flower/fl-api-flower/fl_api/app.py +++ b/fl-services/flower/fl-api-flower/fl_api/app.py @@ -122,6 +122,9 @@ def _extract_json_from_stdout(stdout: str) -> dict[str, Any]: def _run_flwr_command(command: list[str], cwd: Path, action_name: str) -> subprocess.CompletedProcess[str]: + # The flwr CLI is this service's own baked venv binary (flwr is a direct dependency), + # deliberately NOT `uvx flwr`: uvx resolves its tool environment from PyPI at call + # time, which the egress-less LZA account cannot reach (FLIP#749). try: return subprocess.run( command, @@ -182,7 +185,7 @@ def _validate_tutorial_folder(tutorial_name: str) -> Path: def _get_federation_nodes(src_root: Path) -> list[dict[str, Any]]: """Query the SuperLink Control API for connected SuperNode statuses.""" - command = ["uvx", "flwr", "federation", "list", "--federation", "@none/default", "local", "--format", "json"] + command = ["flwr", "federation", "list", "--federation", "@none/default", "local", "--format", "json"] result = _run_flwr_command(command, src_root, "federation list") payload = _parse_flwr_payload(result, "federation list") @@ -310,7 +313,7 @@ def check_client_status( @app.get("/list_jobs", include_in_schema=False) # alias, hide from docs def list_runs() -> list[JobMetadata]: src_root = _get_src_root() - command = ["uvx", "flwr", "list", "local", "--format", "json"] + command = ["flwr", "list", "local", "--format", "json"] result = _run_flwr_command(command, src_root, "list") payload = _parse_flwr_payload(result, "list") @@ -336,7 +339,7 @@ def _submit_from_job_dir(job_dir: Path, label: str) -> str: run_config_path: str | None = None try: - command = ["uvx", "flwr", "run", ".", "local", "--format", "json"] + command = ["flwr", "run", ".", "local", "--format", "json"] if config_toml_path.is_file(): logger.info("Using config.toml overrides from %s for job submission.", job_dir) # flip-job-dir points the evaluation ServerApp at the app directory, @@ -418,7 +421,7 @@ def _find_terminal_run(src_root: Path, run_id: str) -> JobMetadata | None: Used to make ``DELETE /abort_run`` idempotent: a failed ``flwr stop`` followed by a ``flwr list`` showing the run in a terminal state is treated as a successful no-op. """ - list_command = ["uvx", "flwr", "list", "local", "--format", "json"] + list_command = ["flwr", "list", "local", "--format", "json"] list_result = _run_flwr_command(list_command, src_root, "list") if list_result.returncode != 0: return None @@ -449,7 +452,7 @@ def abort_run(run_id: int) -> JobMetadata: # the string form. src_root = _get_src_root() run_id_str = str(run_id) - command = ["uvx", "flwr", "stop", run_id_str, "local", "--format", "json"] + command = ["flwr", "stop", run_id_str, "local", "--format", "json"] result = _run_flwr_command(command, src_root, "stop") if result.returncode == 0: diff --git a/fl-services/flower/fl-api-flower/tests/conftest.py b/fl-services/flower/fl-api-flower/tests/conftest.py index 6f1259232..b6e7bfc44 100644 --- a/fl-services/flower/fl-api-flower/tests/conftest.py +++ b/fl-services/flower/fl-api-flower/tests/conftest.py @@ -60,9 +60,9 @@ def _raise(*_args, **_kwargs): if by_command is not None: # by_command: {flwr_subcommand: {"returncode": int, "stdout": str, "stderr": str}} - # e.g. {"stop": {...}, "list": {...}}. command is ["uvx", "flwr", "", ...]. + # e.g. {"stop": {...}, "list": {...}}. command is ["flwr", "", ...]. def _dispatch(command, *_args, **_kwargs): - subcommand = command[2] if len(command) > 2 else "" + subcommand = command[1] if len(command) > 1 else "" spec = by_command.get(subcommand, {}) return subprocess.CompletedProcess( args=command, From eb12a6505591f50d32aa849a0ba6218229df3029 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Fri, 21 Aug 2026 12:57:27 +0100 Subject: [PATCH 29/41] feat(deploy/AWS): disable Flower runtime dependency installs on LZA The SuperLink's per-run 'uv sync' resolves the app env from package indexes at run time -- unreachable from the egress-less account, so every ServerApp died on its first fetch (hatchling, flip-utils' build backend; Flower exit code 608). Flower's documented restricted-network mode is to preinstall app dependencies and disable runtime installation; fl-base already bakes the full ML env, so LZA sets FLWR_DISABLE_RUNTIME_DEPENDENCY_INSTALLATION=1 on the SuperLink task. Researcher package additions there ship via image rebuild, not pyproject. Legacy envs keep runtime installs (merge with {} keeps the map identical). Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/locals.tf | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/deploy/providers/AWS/locals.tf b/deploy/providers/AWS/locals.tf index d48e0961b..a430cb26c 100644 --- a/deploy/providers/AWS/locals.tf +++ b/deploy/providers/AWS/locals.tf @@ -180,7 +180,7 @@ locals { # Flower SuperLink (compose.production.flower.yml fl-server-net-1). TLS + # SuperNode-auth flags travel as the container command (ecs_tasks.tf), not # env. INTERNAL_SERVICE_KEY is injected via the secrets block. - fl_server_flower = { + fl_server_flower = merge({ LOCAL_DEV = "false" NET_ID = "net-1" MIN_CLIENTS = tostring(var.MIN_CLIENTS) @@ -188,7 +188,13 @@ locals { UPLOADED_FEDERATED_DATA_BUCKET = local.uploaded_federated_data_uri FLIP_API_INTERNAL_URL = "http://${local.service_discovery_names.flip_api}:${local.api_container_port}/api" INTERNAL_SERVICE_KEY_HEADER = var.INTERNAL_SERVICE_KEY_HEADER - } + }, var.lza_managed_network ? { + # Sealed egress (FLIP#749): the SuperLink's per-run `uv sync` cannot reach any + # package index, so the ServerApp runs in the image's preinstalled environment + # instead β€” Flower's documented restricted-network mode (exit-code 608 docs). + # Researcher package additions ship via image rebuild, not pyproject. + FLWR_DISABLE_RUNTIME_DEPENDENCY_INSTALLATION = "1" + } : {}) # Flower fl-api (compose.production.flower.yml fl-api-net-1). SuperLink # addresses use the Cloud Map name β€” the provisioned server cert must # carry it as a SAN (FLOWER_EXTRA_SERVER_SANS at provision time). From 994692478e2f40cffc4d308726c8c196a60ed654 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Mon, 24 Aug 2026 11:53:20 +0100 Subject: [PATCH 30/41] docs(deploy/AWS): frame the two deployment modes; refresh LZA state notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Present the AWS Terraform root as offering two permanently supported deployment modes rather than a migration in progress: the self-contained single-account shape (PROD=stag/true, FLIP creates its own VPC and edge) and the platform-managed shape (PROD=lza) for a Landing Zone Accelerator estate. Both docs previously implied the first was on its way out. Correct two stale caveats in the LZA section that the same PR contradicts: - multi-AZ has landed platform-side (both AZs carry app/data/tgw subnets), so the RDS two-AZ constraint and the -b blackhole no longer apply, and no FLIP-side change was needed since the subnet lookups glob Name tags; - FL ingress is wired β€” fl_ingress_lza.tf creates the internal NLB, its target group fronting fl-server-net-1, and the /flip/networking/* SSM handoff the networking account's edge consumes. The auxiliary-tooling caveat (full-deploy chains, check_status.py, update_env.py, destroy) is unverified against PROD=lza and stays. Also drop the workload account ID from tracked files β€” the account is identified by its `lza-prod` profile alias, matching how prod/stag are referenced β€” and rename the section anchor accordingly. Signed-off-by: at24_bioeng625-pc --- AGENTS.md | 2 +- CLAUDE.md | 2 +- deploy/README.md | 11 +++-- deploy/providers/AWS/AGENTS.md | 4 +- deploy/providers/AWS/CLAUDE.md | 4 +- deploy/providers/AWS/Makefile | 12 +++--- deploy/providers/AWS/README.md | 65 +++++++++++++++++------------ deploy/providers/AWS/network_lza.tf | 6 +-- 8 files changed, 61 insertions(+), 45 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 05760ea78..c1e7a519d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -373,7 +373,7 @@ After changes, evaluate if docs need updating: - `FL_PROVISIONED_DIR` β€” path to the in-tree provisioned FL artifacts, derived per-backend by `deploy/fl_backend.mk` from `FL_BACKEND`: `fl-services/nvflare/provision/workspace-dev` (nvflare startup kits) or `fl-services/flower/provision/creds` (flower per-net TLS certs + SuperNode keys). Both gitignored. Read only by the dev compose overlays for the cert/workspace volume mounts; override at the CLI for a one-off (`make up FL_PROVISIONED_DIR=...`). FL Makefiles are **per-backend** β€” each `fl-services//Makefile` owns that backend's `build`/`provision`/`up`/`down`/`submit` (flower also `up-secure`); the root Makefile forwards only `build-fl` by `FL_BACKEND`. Each backend's `fl-services//Makefile` also owns its network provisioning (NVFLARE adds `provision`/`provision-2-nets`/`provision-stag`/`provision-prod`/`upload-kits-to-s3`; the project YAMLs, `scripts/`, and gitignored `workspace-{dev,stag,prod}/` output live under `provision/`). Provision with `make -C fl-services/nvflare provision-2-nets` (nvflare) or `make -C fl-services/flower provision NET_NUMBER=` (flower). To run a backend standalone + submit without the full stack: `make -C fl-services/ up` (or `up-secure`) then `make -C fl-services/ submit APP=`. - `FL_APP_BASE_DIR` β€” Local directory holding the base FL application templates (the repo's `fl-apps/` tree), baked into the flip-api image and bind-mounted in dev. flip-api walks `///` to bundle an application (uploading those files into `FL_APP_DESTINATION_BUCKET/`) and reads each backend's manifest from `//required_files.json`. Default `/app/fl-apps`; override to mount operator-provided templates. Replaces the removed `FL_APP_BASE_BUCKET` S3 dependency (FLIP#724): base templates are no longer published to S3 (the `fl-apps-push-s3-*` sync workflows are gone), so a template hotfix now ships by rebuilding + redeploying the flip-api image rather than syncing S3. `fl-apps/` is baked into the image via a BuildKit named build context (`fl_apps=../fl-apps`) since it sits outside flip-api's build context. For the Flower backend, the template pyprojects also steer Flower's **per-run dependency install** (`uv sync` on every app launch; SuperNodes opt in via `--allow-runtime-dependency-installation` in the composes): `[tool.uv.sources]` pins `flip-utils` to the source kept at `/opt/flip-utils` inside the FL images (never PyPI β€” FLIP#767; a flip-utils change ships by rebuilding the FL images, `make build-fl FL_BACKEND=flower`) and torch/torchvision to the cu128 index (PyPI's default cu130 wheels need driver >=580). - `FL_KIT_SLOT_NAMES` β€” JSON list (e.g. `["Trust_1", "Trust_2"]`) of FL kit-slot names for the hub's `fl_kit_slot` pool that `register_trust` claims from; each name must match a provisioned participant kit (in-tree workspace for dev, `s3:///fl-flare-participant-kits//net-/services//` for stag/prod; slot names are global across nets β€” every net carries a kit per name). The pool is seeded at flip-api boot and **reconciled on demand** when a registration finds it exhausted (`resolve_fl_kit_slot_names`, additive β€” never deletes or re-assigns rows); only then does `NoFreeKitSlotError` surface. Single source per env: dev = this env var (a `DevSettings`-only field; restart to change, settings load once); stag/prod = the `/flip/fl_kit_slot_names` SSM parameter (Terraform-rendered from this var β€” the list is plain config, not a secret; deliberately **no env fallback**, so a broken/missing parameter means the pool can't grow, loudly, never masked by stale task-def env). Growing the pool is an env-file edit + `make -C deploy/providers/AWS apply-fl-kit-slots` (targeted plan/apply of just the parameter, plain-text diff) β€” **no restart, no task-definition change**. One-command workflow: `make -C deploy/providers/AWS add-fl-kits N= PROD=stag|true` (N = "ensure N more live slots": activate spares toward N first, mint only the shortfall on every net β†’ additive S3 upload β†’ env edit β†’ parameter apply); full runbook in `fl-services/nvflare/README.md` ("Onboarding a new client onto an existing network"). NVFLARE-only dynamics β€” Flower's SuperNode key labelling reads the list at net startup. -- `PROD` β€” `true` (production), `stag` (staging), `lza` (LZA FLIPProduction account, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets only; selects the root `.env.lza-prod` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying to the LZA account"), unset (development) +- `PROD` β€” `true` (production), `stag` (staging), `lza` (an AWS Landing Zone Accelerator estate, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets only; selects the root `.env.lza-prod` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying onto an LZA estate"), unset (development) - `AES_KEY_BASE64` β€” encryption key for trust communication - A remote trust operator only needs their kit file (`trust/.env.`) β€” no hub `.env.` needed on trust hosts. See `trust/README.md` for the standalone-operator quick-start. diff --git a/CLAUDE.md b/CLAUDE.md index 7b26f5b90..6f03241eb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -373,7 +373,7 @@ After changes, evaluate if docs need updating: - `FL_PROVISIONED_DIR` β€” path to the in-tree provisioned FL artifacts, derived per-backend by `deploy/fl_backend.mk` from `FL_BACKEND`: `fl-services/nvflare/provision/workspace-dev` (nvflare startup kits) or `fl-services/flower/provision/creds` (flower per-net TLS certs + SuperNode keys). Both gitignored. Read only by the dev compose overlays for the cert/workspace volume mounts; override at the CLI for a one-off (`make up FL_PROVISIONED_DIR=...`). FL Makefiles are **per-backend** β€” each `fl-services//Makefile` owns that backend's `build`/`provision`/`up`/`down`/`submit` (flower also `up-secure`); the root Makefile forwards only `build-fl` by `FL_BACKEND`. Each backend's `fl-services//Makefile` also owns its network provisioning (NVFLARE adds `provision`/`provision-2-nets`/`provision-stag`/`provision-prod`/`upload-kits-to-s3`; the project YAMLs, `scripts/`, and gitignored `workspace-{dev,stag,prod}/` output live under `provision/`). Provision with `make -C fl-services/nvflare provision-2-nets` (nvflare) or `make -C fl-services/flower provision NET_NUMBER=` (flower). To run a backend standalone + submit without the full stack: `make -C fl-services/ up` (or `up-secure`) then `make -C fl-services/ submit APP=`. - `FL_APP_BASE_DIR` β€” Local directory holding the base FL application templates (the repo's `fl-apps/` tree), baked into the flip-api image and bind-mounted in dev. flip-api walks `///` to bundle an application (uploading those files into `FL_APP_DESTINATION_BUCKET/`) and reads each backend's manifest from `//required_files.json`. Default `/app/fl-apps`; override to mount operator-provided templates. Replaces the removed `FL_APP_BASE_BUCKET` S3 dependency (FLIP#724): base templates are no longer published to S3 (the `fl-apps-push-s3-*` sync workflows are gone), so a template hotfix now ships by rebuilding + redeploying the flip-api image rather than syncing S3. `fl-apps/` is baked into the image via a BuildKit named build context (`fl_apps=../fl-apps`) since it sits outside flip-api's build context. For the Flower backend, the template pyprojects also steer Flower's **per-run dependency install** (`uv sync` on every app launch; SuperNodes opt in via `--allow-runtime-dependency-installation` in the composes): `[tool.uv.sources]` pins `flip-utils` to the source kept at `/opt/flip-utils` inside the FL images (never PyPI β€” FLIP#767; a flip-utils change ships by rebuilding the FL images, `make build-fl FL_BACKEND=flower`) and torch/torchvision to the cu128 index (PyPI's default cu130 wheels need driver >=580). - `FL_KIT_SLOT_NAMES` β€” JSON list (e.g. `["Trust_1", "Trust_2"]`) of FL kit-slot names for the hub's `fl_kit_slot` pool that `register_trust` claims from; each name must match a provisioned participant kit (in-tree workspace for dev, `s3:///fl-flare-participant-kits//net-/services//` for stag/prod; slot names are global across nets β€” every net carries a kit per name). The pool is seeded at flip-api boot and **reconciled on demand** when a registration finds it exhausted (`resolve_fl_kit_slot_names`, additive β€” never deletes or re-assigns rows); only then does `NoFreeKitSlotError` surface. Single source per env: dev = this env var (a `DevSettings`-only field; restart to change, settings load once); stag/prod = the `/flip/fl_kit_slot_names` SSM parameter (Terraform-rendered from this var β€” the list is plain config, not a secret; deliberately **no env fallback**, so a broken/missing parameter means the pool can't grow, loudly, never masked by stale task-def env). Growing the pool is an env-file edit + `make -C deploy/providers/AWS apply-fl-kit-slots` (targeted plan/apply of just the parameter, plain-text diff) β€” **no restart, no task-definition change**. One-command workflow: `make -C deploy/providers/AWS add-fl-kits N= PROD=stag|true` (N = "ensure N more live slots": activate spares toward N first, mint only the shortfall on every net β†’ additive S3 upload β†’ env edit β†’ parameter apply); full runbook in `fl-services/nvflare/README.md` ("Onboarding a new client onto an existing network"). NVFLARE-only dynamics β€” Flower's SuperNode key labelling reads the list at net startup. -- `PROD` β€” `true` (production), `stag` (staging), `lza` (LZA FLIPProduction account, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets only; selects the root `.env.lza-prod` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying to the LZA account"), unset (development) +- `PROD` β€” `true` (production), `stag` (staging), `lza` (an AWS Landing Zone Accelerator estate, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets only; selects the root `.env.lza-prod` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying onto an LZA estate"), unset (development) - `AES_KEY_BASE64` β€” encryption key for trust communication - A remote trust operator only needs their kit file (`trust/.env.`) β€” no hub `.env.` needed on trust hosts. See `trust/README.md` for the standalone-operator quick-start. diff --git a/deploy/README.md b/deploy/README.md index 273011624..da696de76 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -233,8 +233,12 @@ This runbook is for the case where **you** have lost access to your TOTP device The Central Hub has **one supported production deployment**: ECS Fargate via the Terraform root in [`deploy/providers/AWS/`](providers/AWS/README.md). The task definitions in `ecs_tasks.tf` (env maps in -`locals.tf`) are the **canonical definition of production container config**. Deploying into an AWS -LZA-governed account is an env-gated **mode** of that same root, not a separate path +`locals.tf`) are the **canonical definition of production container config**. That root offers **two +deployment modes**, both permanently supported: the default **self-contained single-account** shape +(`PROD=stag`/`PROD=true`), where FLIP creates its own VPC and edge; and the **platform-managed** shape +(`PROD=lza`) for an AWS Landing Zone Accelerator estate, where the network and edge are owned by the +accelerator. The second is an env-gated mode of the same root, not a separate path β€” see +[Deploying onto an LZA estate](providers/AWS/README.md#deploying-onto-an-lza-estate-prodlza) ([FLIP#749](https://github.com/londonaicentre/FLIP/issues/749)). The ECS FL task definitions serve **both FL backends** ([FLIP#566](https://github.com/londonaicentre/FLIP/issues/566)): `FL_BACKEND` in the env file switches the same task families between NVFLARE and Flower (SuperLink ports/command/creds β€” Flower @@ -248,7 +252,8 @@ Cloud Map + public FL hostnames). > `compose.production*.yml` files remain maintained **only** as the local prod-image harness > (`make up PROD=stag|true` β€” the baked images, no dev mounts). When changing production config, change > Terraform first and update the compose files only as far as the local harness needs. Remaining hub-EC2 -> material is removed once the LZA migration's legacy decommission lands (FLIP#749 WP6). +> material is removed once the AI Centre's own legacy account is decommissioned (FLIP#749 WP6) β€” that +> retires one *account*, not the self-contained deployment mode, which stays supported. ### Trusts diff --git a/deploy/providers/AWS/AGENTS.md b/deploy/providers/AWS/AGENTS.md index 8753cdeec..dc846eaf0 100644 --- a/deploy/providers/AWS/AGENTS.md +++ b/deploy/providers/AWS/AGENTS.md @@ -29,7 +29,7 @@ | ------- | ------------- | --------- | | `stag` | Staging | `flipstag` | | `prod` | Production | `flipprod` | -| `lza-prod` | LZA FLIPProduction (`PROD=lza`, FLIP#749; `FLIPAdminAccess` permission set) | `893493035022` | +| `lza-prod` | LZA estate (`PROD=lza`, FLIP#749; `FLIPAdminAccess` permission set) | `FLIPProduction` | | `dev` | Development (the `dev/` root: Cognito + SES; `FlipDeveloperAccess` permission set) | `flipdev` | ## Key Deploy Commands @@ -37,7 +37,7 @@ ```bash make full-deploy PROD=stag # Full staging deploy make full-deploy PROD=true # Full prod deploy -make init/plan/apply PROD=lza # LZA FLIPProduction (env-gated; full-deploy chains untested there β€” see README "Deploying to the LZA account") +make init/plan/apply PROD=lza # LZA estate, platform-managed network (env-gated; full-deploy chains untested there β€” see README "Deploying onto an LZA estate") make full-deploy-hybrid PROD= [LOCAL_TRUST_IP=] # Hybrid with on-prem trust make full-deploy-hub-only PROD= # Hub only, NO cloud Trust EC2 (all trusts on-prem, e.g. GPU hosts) β€” see README "Hub-only Deployment" make init/plan/apply # Terraform workflow diff --git a/deploy/providers/AWS/CLAUDE.md b/deploy/providers/AWS/CLAUDE.md index 995008e85..2fcc570fd 100644 --- a/deploy/providers/AWS/CLAUDE.md +++ b/deploy/providers/AWS/CLAUDE.md @@ -29,7 +29,7 @@ | ------- | ------------- | --------- | | `stag` | Staging | `flipstag` | | `prod` | Production | `flipprod` | -| `lza-prod` | LZA FLIPProduction (`PROD=lza`, FLIP#749; `FLIPAdminAccess` permission set) | `893493035022` | +| `lza-prod` | LZA estate (`PROD=lza`, FLIP#749; `FLIPAdminAccess` permission set) | `FLIPProduction` | | `dev` | Development (the `dev/` root: Cognito + SES; `FlipDeveloperAccess` permission set) | `flipdev` | ## Key Deploy Commands @@ -37,7 +37,7 @@ ```bash make full-deploy PROD=stag # Full staging deploy make full-deploy PROD=true # Full prod deploy -make init/plan/apply PROD=lza # LZA FLIPProduction (env-gated; full-deploy chains untested there β€” see README "Deploying to the LZA account") +make init/plan/apply PROD=lza # LZA estate, platform-managed network (env-gated; full-deploy chains untested there β€” see README "Deploying onto an LZA estate") make full-deploy-hybrid PROD= [LOCAL_TRUST_IP=] # Hybrid with on-prem trust make full-deploy-hub-only PROD= # Hub only, NO cloud Trust EC2 (all trusts on-prem, e.g. GPU hosts) β€” see README "Hub-only Deployment" make init/plan/apply # Terraform workflow diff --git a/deploy/providers/AWS/Makefile b/deploy/providers/AWS/Makefile index 5bf2eb717..8c590c437 100644 --- a/deploy/providers/AWS/Makefile +++ b/deploy/providers/AWS/Makefile @@ -13,10 +13,10 @@ ifeq ($(PROD),true) MAIN_ENV_FILE=../../../.env.production else ifeq ($(PROD),lza) -# LZA FLIPProduction account (FLIP#749): a prod-grade environment on the -# platform-managed (AWS Accelerator) network, running in parallel with legacy -# prod until cutover. Gitignored like the other env files β€” see the -# "Deploying to the LZA account" section in README.md for the required values. +# LZA estate (FLIP#749): a prod-grade environment on a platform-managed (AWS +# Landing Zone Accelerator) network, the second of the two supported deployment +# modes. Gitignored like the other env files β€” see the "Deploying onto an LZA +# estate" section in README.md for the required values. MAIN_ENV_FILE=../../../.env.lza-prod else # Sometimes PROD may not be set, default to staging environment in that case @@ -60,8 +60,8 @@ $(if $(FL_KIT_DATE),,$(error 🧠 $(if $(filter nvflare,$(FL_BACKEND)),FLARE_KIT # the env file or on the command line if your local profile names differ. PROD_AWS_PROFILE ?= prod STAG_AWS_PROFILE ?= stag -# The LZA FLIPProduction account (893493035022) is reached via the Identity -# Center `FLIPAdminAccess` permission set; like prod/stag above, the default is +# The LZA workload account is reached via the Identity Center +# `FLIPAdminAccess` permission set; like prod/stag above, the default is # a short local alias for it in ~/.aws/config (FLIP#749). LZA_AWS_PROFILE ?= lza-prod diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index 44b854509..c0f03d65a 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -586,8 +586,8 @@ The `PROD` variable determines which environment files are loaded: - `PROD=stag` β†’ Uses the root `.env.stag` - `PROD=true` β†’ Uses the root `.env.production` -- `PROD=lza` β†’ Uses the root `.env.lza-prod` (the LZA FLIPProduction account β€” see - [Deploying to the LZA account](#deploying-to-the-lza-account-flipproduction)) +- `PROD=lza` β†’ Uses the root `.env.lza-prod` (an LZA-governed estate β€” see + [Deploying onto an LZA estate](#deploying-onto-an-lza-estate-prodlza)) If `PROD` is omitted when running the AWS provider Makefile, it defaults to staging. @@ -640,19 +640,31 @@ make apply See [`dev/README.md`](./dev/README.md) for the one-time `terraform import` workflow that pulls the manually-created dev Cognito pool into state. -### Deploying to the LZA account (FLIPProduction) +### Deploying onto an LZA estate (PROD=lza) -[FLIP#749](https://github.com/londonaicentre/FLIP/issues/749) migrates the production deployment to the LZA-provisioned -**FLIPProduction** account (`893493035022`, eu-west-2). Until cutover it runs **in parallel** with legacy prod: every -LZA adaptation in this stack is env-gated behind `PROD=lza`, and with `PROD=true`/`PROD=stag` the resolved -configuration is identical to before β€” the legacy environments are never touched by LZA work. +This Terraform root supports **two deployment modes**, both permanently: + +| Mode | Selected by | Network | Ingress | +| --- | --- | --- | --- | +| **Self-contained** (default) | `PROD=stag` / `PROD=true` | FLIP creates its own VPC, subnets, IGW, NAT | In-account CloudFront + public FL NLB | +| **Platform-managed (LZA)** | `PROD=lza` | Discovered from the accelerator-provisioned VPC; FLIP creates none of it | Shared networking account's two-tier edge, over the Transit Gateway | + +Self-contained single-account is the supported open-source deployment shape and is not going away. The LZA mode +([FLIP#749](https://github.com/londonaicentre/FLIP/issues/749)) is the multi-account shape for estates running AWS's +[Landing Zone Accelerator](https://aws.amazon.com/solutions/implementations/landing-zone-accelerator-on-aws/), where +the network, guardrails and edge are owned by the accelerator pipeline rather than by FLIP. + +The two are one root module, not a fork: every LZA adaptation is gated behind `var.lza_managed_network` (set from +`PROD=lza`), so with `PROD=true`/`PROD=stag` the resolved configuration is identical to before β€” the self-contained +environments are never touched by LZA work. The AI Centre's own LZA target is the **FLIPProduction** workload +account in `eu-west-2`, reached through the `lza-prod` profile alias and running alongside legacy prod. **What `PROD=lza` selects:** | Concern | Value | | --- | --- | | Env file | root `.env.lza-prod` (gitignored, like the other env files) | -| Profile guard | `AWS_PROFILE=lza-prod` β€” a short local alias for the `FLIPAdminAccess` permission set on `893493035022`, per the same convention as `prod`/`stag` (override via `LZA_AWS_PROFILE`) | +| Profile guard | `AWS_PROFILE=lza-prod` β€” a short local alias for the workload account's `FLIPAdminAccess` permission set, per the same convention as `prod`/`stag` (override via `LZA_AWS_PROFILE`) | | `TF_VAR_environment` | `prod` β€” LZA is a production estate, so all prod-only hardening (RDS deletion protection, final snapshot) stays on | | `TF_VAR_lza_managed_network` | `true` β€” the platform-managed-network toggle, orthogonal to `environment` (see below) | | Trust kit suffix | `trust/.env..lza-prod` β€” a separate namespace so legacy prod kits are never overwritten | @@ -666,8 +678,8 @@ configuration is identical to before β€” the legacy environments are never touch centralised in the Network account; S3+DynamoDB gateway endpoints are platform-provided), the DHCP options, and the `/flip/networking/*` SSM params (legacy TGW coupling β€” the LZA TGW attachment is platform-managed); - **discovers instead**: the `AWSAccelerator-eu-west-2-prod` VPC and its subnets by Name tag (`network_lza.tf`; - override the name via `LZA_VPC_NAME`). Subnet lookups match ALL `-app-*` / `-data-*` hits, so the multi-AZ `-b` - subnets the platform team is adding appear on the next plan without a code change; + override the name via `LZA_VPC_NAME`). Subnet lookups match ALL `-app-*` / `-data-*` hits, so subnets the platform + team adds later β€” as the second AZ's were β€” appear on the next plan with no code change; - **places by connectivity need**: RDS instances go to the isolated **data** subnets (local routes only β€” nothing there can reach TGW/endpoints, and RDS doesn't need to); ECS tasks, the internal ALB, the RDS Proxy, EFS mount targets and the EC2 hosts go to the TGW-routed **app** subnets (they need the central endpoints / image pulls); @@ -721,9 +733,9 @@ AWS_REGION=eu-west-2 # Registry: the ghcr/ pull-through cache, NOT ghcr.io (no internet egress). # Composes with the image names exactly like the GHCR prefix does: # : β†’ .../ghcr/londonaicentre/flip-api: -DOCKER_REGISTRY=893493035022.dkr.ecr.eu-west-2.amazonaws.com/ghcr/londonaicentre/ +DOCKER_REGISTRY=.dkr.ecr.eu-west-2.amazonaws.com/ghcr/londonaicentre/ # EFS-provision one-shot utility image via the credential-less ecr-public/ cache -EFS_PROVISION_IMAGE=893493035022.dkr.ecr.eu-west-2.amazonaws.com/ecr-public/aws-cli/aws-cli:2.22.35 +EFS_PROVISION_IMAGE=.dkr.ecr.eu-west-2.amazonaws.com/ecr-public/aws-cli/aws-cli:2.22.35 # Bucket names are globally unique and the legacy flipprod-* names stay taken # while the old account lives β€” the LZA env uses its own flip-lza-* namespace. @@ -771,21 +783,20 @@ revert by flipping `MANAGE_DNS=true` + `make plan`/`apply` once the zone lands: - Trusts polling the hub would need the CloudFront domain as `CENTRAL_HUB_API_URL` β€” fine for WP3 smoke trusts; the real cutover is DNS-only and happens after the zone migrates. -**Not yet on LZA** (tracked on #749): - -- Multi-AZ is only partially landed (verified against the live account 2026-08-17): `prod-app-b` exists (lza#38), so - the **ALB is unblocked**, but there is **no `prod-data-b`** β€” the RDS subnet group still fails its β‰₯2-AZ - requirement β€” and **no `prod-tgw-b`**, so the TGW attachment rides the lone `tgw-a` subnet and **TGW-routed - traffic from `-b` subnets blackholes** (central endpoints, image pulls; reported on PR#830). Until `data-b`/`tgw-b` - land and the attachment spans both AZs, workloads that need the TGW must stay pinned to `-a` subnets β€” the e2e-lza - stack (FLIP#829) shows the pinning pattern. -- No FL inbound path (see the NLB gating above). The ingress architecture itself is settled and proven end-to-end - with dummy services β€” internet β†’ networking-account edge NLB β†’ TGW β†’ central firewall β†’ workload NLB β†’ ECS - (FLIP#829 / PR#830) β€” but wiring the real `fl-server-net-1` behind that chain (internal NLB + target group in this - stack, edge listener registration on the platform side) is follow-up #749 work. -- The `full-deploy*` chains, `make status`/`check_status.py`, `update_env.py` and `make destroy` are untested against - `PROD=lza` β€” WP3 exercises the `init`/`plan`/`apply` (+ `deploy-centralhub`/`deploy-ui`) loop first and fixes up the - auxiliary tooling as findings come in. +**State of the LZA path** (tracked on #749): + +- **Multi-AZ has landed platform-side** (verified against the live account 2026-08-20): both AZs now carry an + `-app-*`, a `-data-*` and a `-tgw-*` subnet, which clears the earlier single-AZ constraints β€” the RDS subnet + group's two-AZ requirement is satisfiable, and the TGW attachment no longer rides a lone `tgw-a` subnet with + `-b`-originated traffic blackholing (the condition reported on PR#830). No FLIP-side change was needed: the + subnet lookups glob every matching Name tag, so the second AZ arrived on an ordinary plan. +- **FL ingress is wired.** `fl_ingress_lza.tf` creates the internal NLB (static per-subnet IPs, so the edge can + register them once as targets), its target group fronting `fl-server-net-1`, and the `/flip/networking/*` SSM + handoff the networking account consumes. The full chain β€” internet β†’ edge NLB β†’ TGW β†’ central firewall β†’ + internal NLB β†’ ECS β€” was proven with dummy services first (FLIP#829 / PR#830). +- **Still untested against `PROD=lza`:** the `full-deploy*` chains, `make status`/`check_status.py`, `update_env.py` + and `make destroy`. WP3 exercises the `init`/`plan`/`apply` (+ `deploy-centralhub`/`deploy-ui`) loop first and + fixes up the auxiliary tooling as findings come in. ### Terraform module layout diff --git a/deploy/providers/AWS/network_lza.tf b/deploy/providers/AWS/network_lza.tf index 6ebf40988..d9539df37 100644 --- a/deploy/providers/AWS/network_lza.tf +++ b/deploy/providers/AWS/network_lza.tf @@ -16,9 +16,9 @@ # Platform-managed network (LZA) β€” FLIP#749 ############################ # -# On the LZA FLIPProduction account (893493035022) the VPC layer is provisioned -# by the AWS Accelerator pipeline (londonaicentre/lza) and the GRNETSEC2 SCP -# denies CreateVpc / CreateSubnet / CreateInternetGateway / AllocateAddress / +# On an LZA-governed workload account the VPC layer is provisioned by the AWS +# Accelerator pipeline (londonaicentre/lza) and the GRNETSEC2 SCP denies +# CreateVpc / CreateSubnet / CreateInternetGateway / AllocateAddress / # CreateVpcEndpoint to everything but that pipeline (GRTGWVPN makes even # route-table edits pipeline-only), so this stack cannot create its own network # there. When var.lza_managed_network is set the VPC module is skipped From 1eae09390686573fd03090df1725cb471ed65e90 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Mon, 24 Aug 2026 11:57:33 +0100 Subject: [PATCH 31/41] docs(deploy/AWS): confirm the multi-AZ state against the live account MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-checked today rather than resting on the 2026-08-20 notes: all six platform subnets exist (app/data/tgw in both AZs), flip-db-subnet-group spans eu-west-2a and eu-west-2b, and the TGW attachment rides tgw-a and tgw-b β€” so the note can state the RDS group and the attachment as facts rather than as a requirement that is merely satisfiable. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index c0f03d65a..157f35f37 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -785,11 +785,11 @@ revert by flipping `MANAGE_DNS=true` + `make plan`/`apply` once the zone lands: **State of the LZA path** (tracked on #749): -- **Multi-AZ has landed platform-side** (verified against the live account 2026-08-20): both AZs now carry an - `-app-*`, a `-data-*` and a `-tgw-*` subnet, which clears the earlier single-AZ constraints β€” the RDS subnet - group's two-AZ requirement is satisfiable, and the TGW attachment no longer rides a lone `tgw-a` subnet with - `-b`-originated traffic blackholing (the condition reported on PR#830). No FLIP-side change was needed: the - subnet lookups glob every matching Name tag, so the second AZ arrived on an ordinary plan. +- **Multi-AZ has landed platform-side** (verified against the live account 2026-08-24): both AZs carry an + `-app-*`, a `-data-*` and a `-tgw-*` subnet, which clears the earlier single-AZ constraints β€” the live + `flip-db-subnet-group` spans both AZs, and the TGW attachment now rides `tgw-a` **and** `tgw-b`, so traffic + originating in the `-b` subnets no longer blackholes (the condition reported on PR#830). No FLIP-side change + was needed: the subnet lookups glob every matching Name tag, so the second AZ arrived on an ordinary plan. - **FL ingress is wired.** `fl_ingress_lza.tf` creates the internal NLB (static per-subnet IPs, so the edge can register them once as targets), its target group fronting `fl-server-net-1`, and the `/flip/networking/*` SSM handoff the networking account consumes. The full chain β€” internet β†’ edge NLB β†’ TGW β†’ central firewall β†’ From a74c09112d451fcc177a98187651a4884c3f4692 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Mon, 24 Aug 2026 12:00:59 +0100 Subject: [PATCH 32/41] docs: cover both Central Hub deployment modes in the published guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Sphinx deployment guide described only the self-contained shape β€” own VPC with public and private subnets, in-account CloudFront, a public FL NLB, .env.stag/.env.production β€” every line of which is false under PROD=lza. A reader of the published docs had no way to know a second mode exists. Adds a "Deployment modes" section contrasting the two, marks the architecture section as describing the self-contained shape and summarises what moves under a platform-managed one, and threads PROD=lza through the prerequisites, the environment-file mapping and the key generation note. A note records that full-deploy, status and destroy are the self-contained chain and are not yet exercised on PROD=lza. Operator detail deliberately stays in deploy/providers/AWS/README.md; this page says the mode exists, what it implies, and where to read further. Signed-off-by: at24_bioeng625-pc --- .../source/deploy-flip/deploy-central-hub.rst | 86 ++++++++++++++++--- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/docs/source/deploy-flip/deploy-central-hub.rst b/docs/source/deploy-flip/deploy-central-hub.rst index 305b5ece8..e35a7c2d5 100644 --- a/docs/source/deploy-flip/deploy-central-hub.rst +++ b/docs/source/deploy-flip/deploy-central-hub.rst @@ -14,11 +14,55 @@ and :doc:`deploy-flip-node-in-tre`. :local: :depth: 2 +**************** +Deployment modes +**************** + +The Terraform root in ``deploy/providers/AWS/`` supports **two deployment modes**, both +permanently supported. They share one root module β€” the second is selected by an environment +flag, not a fork of the code. + +.. list-table:: + :header-rows: 1 + :widths: 20 18 31 31 + + * - Mode + - Selected by + - Network + - Ingress + * - **Self-contained** (default) + - ``PROD=stag`` / ``PROD=true`` + - FLIP creates its own VPC, subnets, internet gateway and NAT + - In-account CloudFront for the UI and ``/api/*``; a public NLB for FL traffic + * - **Platform-managed** + - ``PROD=lza`` + - Discovered from a VPC provisioned by the AWS `Landing Zone Accelerator + `_; + FLIP creates no network resources + - A shared networking account's edge, reached over a Transit Gateway; no public load + balancer in the workload account + +**Self-contained** is the default, and the shape the rest of this page assumes. Everything +FLIP needs lives in one AWS account, which is the simplest way to stand up a Central Hub. + +**Platform-managed** suits an AWS estate already governed by the Landing Zone Accelerator, +where a platform team owns the network, the guardrails and a shared edge, and workload +accounts are denied VPC-layer creation. FLIP then discovers the network it is given instead +of building one, and ingress arrives through the estate's own edge. Deploying this way means +coordinating with whoever operates that estate β€” the account needs its VPC, Transit Gateway +attachment, egress path and edge listeners in place before FLIP applies. The operator-facing +detail (what the flag gates on and off, the environment file, and the handoff to the +networking account) is in the "Deploying onto an LZA estate" section of +`deploy/providers/AWS/README.md +`_. + +Where a command below differs between the two, the difference is called out inline. + ************ Architecture ************ -The Central Hub stack runs in a custom VPC with public and private subnets across two AZs: +In the self-contained mode the Central Hub stack runs in a custom VPC with public and private subnets across two AZs: - **flip-ui** β€” static assets served from S3 behind CloudFront at the canonical subdomain (``stag.flip.aicentre.co.uk`` / ``app.flip.aicentre.co.uk``). CloudFront also forwards ``/api/*`` to the ALB. - **flip-api** β€” the central application API. @@ -28,6 +72,11 @@ The Central Hub stack runs in a custom VPC with public and private subnets acros - **SES** β€” transactional email (invites, password reset, access requests). - **Secrets Manager** β€” AES key, database password, internal service key hash. +Under the platform-managed mode the same services run unchanged; what differs is where they +sit and how traffic reaches them. The VPC and subnets are the accelerator's, there is no +in-account CloudFront distribution and no public NLB, and both web and FL traffic arrive from +the shared networking account over the Transit Gateway. + Operator access is via AWS Systems Manager (SSM) Session Manager β€” port 22 is **not** open on any security group. @@ -41,13 +90,13 @@ Prerequisites 4. **GitHub CLI** β€” needed to authenticate against GitHub Container Registry for image pulls. 5. **SSH key pair** at ``~/.ssh/host-aws`` β€” uploaded to AWS and used as the identity file for the SSM ProxyCommand-based SSH config. -6. **Environment file** β€” ``.env.stag`` (staging) or ``.env.production`` - (production) in the project root. +6. **Environment file** β€” ``.env.stag`` (staging), ``.env.production`` + (production), or ``.env.lza-prod`` (platform-managed) in the project root. 7. **AWS Session Manager plugin** β€” required for ``ssh flip`` and ``make forward-trust``. -AWS profile aliases (``prod``, ``stag``, ``dev``) should be configured in -``~/.aws/config`` so the Makefile guards can verify the active profile against -the chosen environment. +AWS profile aliases (``prod``, ``stag``, ``dev``, and ``lza-prod`` for a +platform-managed deployment) should be configured in ``~/.aws/config`` so the +Makefile guards can verify the active profile against the chosen environment. ************************ Required IAM permissions @@ -113,9 +162,21 @@ This runs, in order: 13. ``status`` β€” comprehensive health checks. The ``PROD`` variable selects the environment file (``stag`` β†’ ``.env.stag``, -``true`` β†’ ``.env.production``) and is mapped onto ``TF_VAR_environment`` -(``stag`` or ``prod``) so Terraform can gate prod-only RDS hardening (deletion -protection, final snapshot). +``true`` β†’ ``.env.production``, ``lza`` β†’ ``.env.lza-prod``) and is mapped onto +``TF_VAR_environment`` β€” ``stag`` for staging, ``prod`` for both ``true`` and +``lza`` β€” so Terraform can gate prod-only RDS hardening (deletion protection, +final snapshot). ``PROD=lza`` additionally sets the platform-managed-network +flag, which is orthogonal to the environment name: a platform-managed +deployment is a production estate and keeps every prod-only control. + +.. note:: + + ``full-deploy`` is the self-contained chain. Some of its steps β€” the cloud + trust EC2, the SSH and Ansible provisioning β€” do not apply to a + platform-managed deployment, and the chain, along with ``make status`` and + ``make destroy``, is not yet exercised there. On ``PROD=lza`` run ``init`` / + ``plan`` / ``apply`` followed by ``deploy-centralhub`` and ``deploy-ui``; see + the AWS README section linked under `Deployment modes`_. Subsequent UI-only deploys do not need Terraform: @@ -135,7 +196,7 @@ For debugging or selective steps: .. code-block:: shell - export PROD=stag # or: export PROD=true + export PROD=stag # or: export PROD=true, or PROD=lza (platform-managed) make github-login make aws-login @@ -173,8 +234,9 @@ for full details): trust API key and written into the trust's kit file. ``make generate-internal-service-key`` populates the active env file -(``.env.stag`` or ``.env.production``) and preserves any keys that already -exist; ``make register-trusts`` writes the per-trust keys into the kit files. +(``.env.stag``, ``.env.production`` or ``.env.lza-prod``) and preserves any keys +that already exist; ``make register-trusts`` writes the per-trust keys into the +kit files. *********************** Applying schema changes From ef6456e4046cd5c4dbc331cbafa3cc4eecd3df5e Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Wed, 26 Aug 2026 22:30:40 +0100 Subject: [PATCH 33/41] =?UTF-8?q?feat(deploy/AWS):=20PROD=3Dlza-stag=20?= =?UTF-8?q?=E2=80=94=20staging=20on=20a=20platform-managed=20estate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generalises the LZA mode from a single environment into a family. The two axes were already orthogonal (environment vs lza_managed_network), so no Terraform changes are needed β€” lza-stag composes environment=stag with the platform-managed network: - env file .env.lza-stag (gitignored), profile guard lza-stag (LZA_STAG_AWS_PROFILE), kit suffix .lza-stag, origin/develop deploys, and the prod-only RDS hardening off, all matching legacy stag semantics; - LZA_VPC_NAME is mandatory on lza-stag: the variables.tf default names the prod VPC template, and there is no defaultable staging name, so the Makefile refuses to run rather than letting the data lookup fail with an opaque "no results"; - the deploy-ui invalidation skip now covers both LZA values. Docs follow: the two-mode table lists both values, a comparison table contrasts lza vs lza-stag (including the one-account-per-environment rule β€” the stack's resource names and the /flip/networking/* handoff params are fixed per account by design), and the Sphinx guide mentions the staging value in the same breath as lza. Verified: guards exercised for wrong-profile, missing-LZA_VPC_NAME and missing-env-file; derived values probed for all four PROD values (lza-stag β†’ environment=stag, lza_managed_network=true, kit_suffix lza-stag, origin/develop); `make -n plan` byte-identical to HEAD for PROD=stag and PROD=lza; terraform fmt and the docs build clean. Signed-off-by: at24_bioeng625-pc --- .gitignore | 1 + AGENTS.md | 2 +- CLAUDE.md | 2 +- deploy/README.md | 4 +- deploy/providers/AWS/AGENTS.md | 5 +- deploy/providers/AWS/CLAUDE.md | 5 +- deploy/providers/AWS/Makefile | 56 +++++++++++++------ deploy/providers/AWS/README.md | 29 ++++++++-- .../source/deploy-flip/deploy-central-hub.rst | 33 ++++++----- 9 files changed, 91 insertions(+), 46 deletions(-) diff --git a/.gitignore b/.gitignore index f2baec7a8..9bdd03973 100644 --- a/.gitignore +++ b/.gitignore @@ -338,6 +338,7 @@ flip-fl-api/transfer/*/* .env.stag .env.production .env.lza-prod +.env.lza-stag # Per-trust operator kits β€” one file per trust, named by trust CODE and # environment (e.g. trust/.env.KCH.development, trust/.env..production). # Minted by the hub via register-trust / POST /admin/trusts; contain plaintext diff --git a/AGENTS.md b/AGENTS.md index c1e7a519d..398c955cc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -373,7 +373,7 @@ After changes, evaluate if docs need updating: - `FL_PROVISIONED_DIR` β€” path to the in-tree provisioned FL artifacts, derived per-backend by `deploy/fl_backend.mk` from `FL_BACKEND`: `fl-services/nvflare/provision/workspace-dev` (nvflare startup kits) or `fl-services/flower/provision/creds` (flower per-net TLS certs + SuperNode keys). Both gitignored. Read only by the dev compose overlays for the cert/workspace volume mounts; override at the CLI for a one-off (`make up FL_PROVISIONED_DIR=...`). FL Makefiles are **per-backend** β€” each `fl-services//Makefile` owns that backend's `build`/`provision`/`up`/`down`/`submit` (flower also `up-secure`); the root Makefile forwards only `build-fl` by `FL_BACKEND`. Each backend's `fl-services//Makefile` also owns its network provisioning (NVFLARE adds `provision`/`provision-2-nets`/`provision-stag`/`provision-prod`/`upload-kits-to-s3`; the project YAMLs, `scripts/`, and gitignored `workspace-{dev,stag,prod}/` output live under `provision/`). Provision with `make -C fl-services/nvflare provision-2-nets` (nvflare) or `make -C fl-services/flower provision NET_NUMBER=` (flower). To run a backend standalone + submit without the full stack: `make -C fl-services/ up` (or `up-secure`) then `make -C fl-services/ submit APP=`. - `FL_APP_BASE_DIR` β€” Local directory holding the base FL application templates (the repo's `fl-apps/` tree), baked into the flip-api image and bind-mounted in dev. flip-api walks `///` to bundle an application (uploading those files into `FL_APP_DESTINATION_BUCKET/`) and reads each backend's manifest from `//required_files.json`. Default `/app/fl-apps`; override to mount operator-provided templates. Replaces the removed `FL_APP_BASE_BUCKET` S3 dependency (FLIP#724): base templates are no longer published to S3 (the `fl-apps-push-s3-*` sync workflows are gone), so a template hotfix now ships by rebuilding + redeploying the flip-api image rather than syncing S3. `fl-apps/` is baked into the image via a BuildKit named build context (`fl_apps=../fl-apps`) since it sits outside flip-api's build context. For the Flower backend, the template pyprojects also steer Flower's **per-run dependency install** (`uv sync` on every app launch; SuperNodes opt in via `--allow-runtime-dependency-installation` in the composes): `[tool.uv.sources]` pins `flip-utils` to the source kept at `/opt/flip-utils` inside the FL images (never PyPI β€” FLIP#767; a flip-utils change ships by rebuilding the FL images, `make build-fl FL_BACKEND=flower`) and torch/torchvision to the cu128 index (PyPI's default cu130 wheels need driver >=580). - `FL_KIT_SLOT_NAMES` β€” JSON list (e.g. `["Trust_1", "Trust_2"]`) of FL kit-slot names for the hub's `fl_kit_slot` pool that `register_trust` claims from; each name must match a provisioned participant kit (in-tree workspace for dev, `s3:///fl-flare-participant-kits//net-/services//` for stag/prod; slot names are global across nets β€” every net carries a kit per name). The pool is seeded at flip-api boot and **reconciled on demand** when a registration finds it exhausted (`resolve_fl_kit_slot_names`, additive β€” never deletes or re-assigns rows); only then does `NoFreeKitSlotError` surface. Single source per env: dev = this env var (a `DevSettings`-only field; restart to change, settings load once); stag/prod = the `/flip/fl_kit_slot_names` SSM parameter (Terraform-rendered from this var β€” the list is plain config, not a secret; deliberately **no env fallback**, so a broken/missing parameter means the pool can't grow, loudly, never masked by stale task-def env). Growing the pool is an env-file edit + `make -C deploy/providers/AWS apply-fl-kit-slots` (targeted plan/apply of just the parameter, plain-text diff) β€” **no restart, no task-definition change**. One-command workflow: `make -C deploy/providers/AWS add-fl-kits N= PROD=stag|true` (N = "ensure N more live slots": activate spares toward N first, mint only the shortfall on every net β†’ additive S3 upload β†’ env edit β†’ parameter apply); full runbook in `fl-services/nvflare/README.md` ("Onboarding a new client onto an existing network"). NVFLARE-only dynamics β€” Flower's SuperNode key labelling reads the list at net startup. -- `PROD` β€” `true` (production), `stag` (staging), `lza` (an AWS Landing Zone Accelerator estate, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets only; selects the root `.env.lza-prod` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying onto an LZA estate"), unset (development) +- `PROD` β€” `true` (production), `stag` (staging), `lza` / `lza-stag` (production / staging on an AWS Landing Zone Accelerator estate, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets only; select the root `.env.lza-prod` / `.env.lza-stag` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying onto an LZA estate"), unset (development) - `AES_KEY_BASE64` β€” encryption key for trust communication - A remote trust operator only needs their kit file (`trust/.env.`) β€” no hub `.env.` needed on trust hosts. See `trust/README.md` for the standalone-operator quick-start. diff --git a/CLAUDE.md b/CLAUDE.md index 6f03241eb..09eb74070 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -373,7 +373,7 @@ After changes, evaluate if docs need updating: - `FL_PROVISIONED_DIR` β€” path to the in-tree provisioned FL artifacts, derived per-backend by `deploy/fl_backend.mk` from `FL_BACKEND`: `fl-services/nvflare/provision/workspace-dev` (nvflare startup kits) or `fl-services/flower/provision/creds` (flower per-net TLS certs + SuperNode keys). Both gitignored. Read only by the dev compose overlays for the cert/workspace volume mounts; override at the CLI for a one-off (`make up FL_PROVISIONED_DIR=...`). FL Makefiles are **per-backend** β€” each `fl-services//Makefile` owns that backend's `build`/`provision`/`up`/`down`/`submit` (flower also `up-secure`); the root Makefile forwards only `build-fl` by `FL_BACKEND`. Each backend's `fl-services//Makefile` also owns its network provisioning (NVFLARE adds `provision`/`provision-2-nets`/`provision-stag`/`provision-prod`/`upload-kits-to-s3`; the project YAMLs, `scripts/`, and gitignored `workspace-{dev,stag,prod}/` output live under `provision/`). Provision with `make -C fl-services/nvflare provision-2-nets` (nvflare) or `make -C fl-services/flower provision NET_NUMBER=` (flower). To run a backend standalone + submit without the full stack: `make -C fl-services/ up` (or `up-secure`) then `make -C fl-services/ submit APP=`. - `FL_APP_BASE_DIR` β€” Local directory holding the base FL application templates (the repo's `fl-apps/` tree), baked into the flip-api image and bind-mounted in dev. flip-api walks `///` to bundle an application (uploading those files into `FL_APP_DESTINATION_BUCKET/`) and reads each backend's manifest from `//required_files.json`. Default `/app/fl-apps`; override to mount operator-provided templates. Replaces the removed `FL_APP_BASE_BUCKET` S3 dependency (FLIP#724): base templates are no longer published to S3 (the `fl-apps-push-s3-*` sync workflows are gone), so a template hotfix now ships by rebuilding + redeploying the flip-api image rather than syncing S3. `fl-apps/` is baked into the image via a BuildKit named build context (`fl_apps=../fl-apps`) since it sits outside flip-api's build context. For the Flower backend, the template pyprojects also steer Flower's **per-run dependency install** (`uv sync` on every app launch; SuperNodes opt in via `--allow-runtime-dependency-installation` in the composes): `[tool.uv.sources]` pins `flip-utils` to the source kept at `/opt/flip-utils` inside the FL images (never PyPI β€” FLIP#767; a flip-utils change ships by rebuilding the FL images, `make build-fl FL_BACKEND=flower`) and torch/torchvision to the cu128 index (PyPI's default cu130 wheels need driver >=580). - `FL_KIT_SLOT_NAMES` β€” JSON list (e.g. `["Trust_1", "Trust_2"]`) of FL kit-slot names for the hub's `fl_kit_slot` pool that `register_trust` claims from; each name must match a provisioned participant kit (in-tree workspace for dev, `s3:///fl-flare-participant-kits//net-/services//` for stag/prod; slot names are global across nets β€” every net carries a kit per name). The pool is seeded at flip-api boot and **reconciled on demand** when a registration finds it exhausted (`resolve_fl_kit_slot_names`, additive β€” never deletes or re-assigns rows); only then does `NoFreeKitSlotError` surface. Single source per env: dev = this env var (a `DevSettings`-only field; restart to change, settings load once); stag/prod = the `/flip/fl_kit_slot_names` SSM parameter (Terraform-rendered from this var β€” the list is plain config, not a secret; deliberately **no env fallback**, so a broken/missing parameter means the pool can't grow, loudly, never masked by stale task-def env). Growing the pool is an env-file edit + `make -C deploy/providers/AWS apply-fl-kit-slots` (targeted plan/apply of just the parameter, plain-text diff) β€” **no restart, no task-definition change**. One-command workflow: `make -C deploy/providers/AWS add-fl-kits N= PROD=stag|true` (N = "ensure N more live slots": activate spares toward N first, mint only the shortfall on every net β†’ additive S3 upload β†’ env edit β†’ parameter apply); full runbook in `fl-services/nvflare/README.md` ("Onboarding a new client onto an existing network"). NVFLARE-only dynamics β€” Flower's SuperNode key labelling reads the list at net startup. -- `PROD` β€” `true` (production), `stag` (staging), `lza` (an AWS Landing Zone Accelerator estate, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets only; selects the root `.env.lza-prod` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying onto an LZA estate"), unset (development) +- `PROD` β€” `true` (production), `stag` (staging), `lza` / `lza-stag` (production / staging on an AWS Landing Zone Accelerator estate, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets only; select the root `.env.lza-prod` / `.env.lza-stag` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying onto an LZA estate"), unset (development) - `AES_KEY_BASE64` β€” encryption key for trust communication - A remote trust operator only needs their kit file (`trust/.env.`) β€” no hub `.env.` needed on trust hosts. See `trust/README.md` for the standalone-operator quick-start. diff --git a/deploy/README.md b/deploy/README.md index da696de76..d94413df2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -236,8 +236,8 @@ The Central Hub has **one supported production deployment**: ECS Fargate via the `locals.tf`) are the **canonical definition of production container config**. That root offers **two deployment modes**, both permanently supported: the default **self-contained single-account** shape (`PROD=stag`/`PROD=true`), where FLIP creates its own VPC and edge; and the **platform-managed** shape -(`PROD=lza`) for an AWS Landing Zone Accelerator estate, where the network and edge are owned by the -accelerator. The second is an env-gated mode of the same root, not a separate path β€” see +(`PROD=lza`, or `PROD=lza-stag` for a staging estate) for an AWS Landing Zone Accelerator estate, where the +network and edge are owned by the accelerator. The second is an env-gated mode of the same root, not a separate path β€” see [Deploying onto an LZA estate](providers/AWS/README.md#deploying-onto-an-lza-estate-prodlza) ([FLIP#749](https://github.com/londonaicentre/FLIP/issues/749)). The ECS FL task definitions serve **both FL backends** ([FLIP#566](https://github.com/londonaicentre/FLIP/issues/566)): `FL_BACKEND` in the env file diff --git a/deploy/providers/AWS/AGENTS.md b/deploy/providers/AWS/AGENTS.md index dc846eaf0..3e209498b 100644 --- a/deploy/providers/AWS/AGENTS.md +++ b/deploy/providers/AWS/AGENTS.md @@ -29,7 +29,8 @@ | ------- | ------------- | --------- | | `stag` | Staging | `flipstag` | | `prod` | Production | `flipprod` | -| `lza-prod` | LZA estate (`PROD=lza`, FLIP#749; `FLIPAdminAccess` permission set) | `FLIPProduction` | +| `lza-prod` | LZA estate, production (`PROD=lza`, FLIP#749; `FLIPAdminAccess` permission set) | `FLIPProduction` | +| `lza-stag` | LZA estate, staging (`PROD=lza-stag`; `FLIPAdminAccess` permission set) | staging workload account (provisioning tracked on FLIP#749) | | `dev` | Development (the `dev/` root: Cognito + SES; `FlipDeveloperAccess` permission set) | `flipdev` | ## Key Deploy Commands @@ -37,7 +38,7 @@ ```bash make full-deploy PROD=stag # Full staging deploy make full-deploy PROD=true # Full prod deploy -make init/plan/apply PROD=lza # LZA estate, platform-managed network (env-gated; full-deploy chains untested there β€” see README "Deploying onto an LZA estate") +make init/plan/apply PROD=lza # LZA estate, platform-managed network (env-gated; full-deploy chains untested there β€” see README "Deploying onto an LZA estate"). PROD=lza-stag = staging semantics on the same mode (requires LZA_VPC_NAME in .env.lza-stag) make full-deploy-hybrid PROD= [LOCAL_TRUST_IP=] # Hybrid with on-prem trust make full-deploy-hub-only PROD= # Hub only, NO cloud Trust EC2 (all trusts on-prem, e.g. GPU hosts) β€” see README "Hub-only Deployment" make init/plan/apply # Terraform workflow diff --git a/deploy/providers/AWS/CLAUDE.md b/deploy/providers/AWS/CLAUDE.md index 2fcc570fd..40b3e4419 100644 --- a/deploy/providers/AWS/CLAUDE.md +++ b/deploy/providers/AWS/CLAUDE.md @@ -29,7 +29,8 @@ | ------- | ------------- | --------- | | `stag` | Staging | `flipstag` | | `prod` | Production | `flipprod` | -| `lza-prod` | LZA estate (`PROD=lza`, FLIP#749; `FLIPAdminAccess` permission set) | `FLIPProduction` | +| `lza-prod` | LZA estate, production (`PROD=lza`, FLIP#749; `FLIPAdminAccess` permission set) | `FLIPProduction` | +| `lza-stag` | LZA estate, staging (`PROD=lza-stag`; `FLIPAdminAccess` permission set) | staging workload account (provisioning tracked on FLIP#749) | | `dev` | Development (the `dev/` root: Cognito + SES; `FlipDeveloperAccess` permission set) | `flipdev` | ## Key Deploy Commands @@ -37,7 +38,7 @@ ```bash make full-deploy PROD=stag # Full staging deploy make full-deploy PROD=true # Full prod deploy -make init/plan/apply PROD=lza # LZA estate, platform-managed network (env-gated; full-deploy chains untested there β€” see README "Deploying onto an LZA estate") +make init/plan/apply PROD=lza # LZA estate, platform-managed network (env-gated; full-deploy chains untested there β€” see README "Deploying onto an LZA estate"). PROD=lza-stag = staging semantics on the same mode (requires LZA_VPC_NAME in .env.lza-stag) make full-deploy-hybrid PROD= [LOCAL_TRUST_IP=] # Hybrid with on-prem trust make full-deploy-hub-only PROD= # Hub only, NO cloud Trust EC2 (all trusts on-prem, e.g. GPU hosts) β€” see README "Hub-only Deployment" make init/plan/apply # Terraform workflow diff --git a/deploy/providers/AWS/Makefile b/deploy/providers/AWS/Makefile index 8c590c437..cc10ea107 100644 --- a/deploy/providers/AWS/Makefile +++ b/deploy/providers/AWS/Makefile @@ -18,6 +18,12 @@ else ifeq ($(PROD),lza) # modes. Gitignored like the other env files β€” see the "Deploying onto an LZA # estate" section in README.md for the required values. MAIN_ENV_FILE=../../../.env.lza-prod +else ifeq ($(PROD),lza-stag) +# Staging on an LZA estate: same platform-managed mode as PROD=lza, but in the +# staging workload account with staging semantics (TF_VAR_environment=stag, so +# the prod-only RDS hardening stays off, and deploys track origin/develop like +# legacy stag). Requires LZA_VPC_NAME in the env file β€” see the guard below. +MAIN_ENV_FILE=../../../.env.lza-stag else # Sometimes PROD may not be set, default to staging environment in that case PROD=stag @@ -60,10 +66,11 @@ $(if $(FL_KIT_DATE),,$(error 🧠 $(if $(filter nvflare,$(FL_BACKEND)),FLARE_KIT # the env file or on the command line if your local profile names differ. PROD_AWS_PROFILE ?= prod STAG_AWS_PROFILE ?= stag -# The LZA workload account is reached via the Identity Center -# `FLIPAdminAccess` permission set; like prod/stag above, the default is -# a short local alias for it in ~/.aws/config (FLIP#749). +# The LZA workload accounts are reached via the Identity Center +# `FLIPAdminAccess` permission set; like prod/stag above, the defaults are +# short local aliases for it in ~/.aws/config (FLIP#749). LZA_AWS_PROFILE ?= lza-prod +LZA_STAG_AWS_PROFILE ?= lza-stag # Refuse to operate against an unintended AWS account. Profile name is the safety # guard because Make has no easy way to read the account ID without an `aws` call. @@ -75,6 +82,16 @@ else ifeq ($(PROD),lza) ifneq ($(AWS_PROFILE),$(LZA_AWS_PROFILE)) $(error "For LZA deployments, AWS_PROFILE must be set to '$(LZA_AWS_PROFILE)' (override via LZA_AWS_PROFILE)") endif +else ifeq ($(PROD),lza-stag) +ifneq ($(AWS_PROFILE),$(LZA_STAG_AWS_PROFILE)) +$(error "For LZA staging deployments, AWS_PROFILE must be set to '$(LZA_STAG_AWS_PROFILE)' (override via LZA_STAG_AWS_PROFILE)") +endif +# The variables.tf default VPC name is the PROD template (AWSAccelerator-eu-west-2-prod); +# there is no defaultable name for the staging VPC, so require it explicitly rather +# than letting the data lookup fail with an opaque "no results" in the wrong account. +ifeq ($(LZA_VPC_NAME),) +$(error "For LZA staging deployments, LZA_VPC_NAME must be set in .env.lza-stag (the Name tag of the staging platform VPC)") +endif else ifneq ($(AWS_PROFILE),$(STAG_AWS_PROFILE)) $(error "For staging deployments, AWS_PROFILE must be set to '$(STAG_AWS_PROFILE)' (override via STAG_AWS_PROFILE)") @@ -82,29 +99,31 @@ endif endif ## SETTING VARIABLES FOR TERRAFORM -# Map the Make-level PROD flag (true|lza|stag) onto a Terraform-friendly environment name. -# Used in main.tf to gate prod-only RDS hardening (deletion protection, final snapshot). -# PROD=lza is a production estate, so it keeps the prod hardening β€” what differs there -# is the platform-managed network, toggled by the orthogonal flag below (FLIP#749). +# Map the Make-level PROD flag (true|lza|lza-stag|stag) onto a Terraform-friendly +# environment name. Used in main.tf to gate prod-only RDS hardening (deletion +# protection, final snapshot). PROD=lza is a production estate, so it keeps the prod +# hardening; PROD=lza-stag is staging semantics on the platform-managed network β€” +# the two axes are orthogonal, toggled by the flag below (FLIP#749). export TF_VAR_environment=$(if $(filter true lza,$(PROD)),prod,stag) -# Platform-managed network toggle (FLIP#749): on the LZA FLIPProduction account -# the VPC layer is provisioned by the AWS Accelerator pipeline and SCP-denied to -# this stack, so Terraform switches from creating the network to data lookups +# Platform-managed network toggle (FLIP#749): on an LZA workload account the VPC +# layer is provisioned by the AWS Accelerator pipeline and SCP-denied to this +# stack, so Terraform switches from creating the network to data lookups # (see network_lza.tf). Off for every legacy environment. -export TF_VAR_lza_managed_network=$(if $(filter lza,$(PROD)),true,false) -# Name tag of the platform-managed VPC; only consulted when PROD=lza. Set +export TF_VAR_lza_managed_network=$(if $(filter lza lza-stag,$(PROD)),true,false) +# Name tag of the platform-managed VPC; only consulted on the LZA modes. Set # LZA_VPC_NAME in the env file to override the variables.tf default -# (AWSAccelerator-eu-west-2-prod) if the platform template is ever renamed. +# (AWSAccelerator-eu-west-2-prod) β€” optional on PROD=lza if the platform template +# keeps that name, mandatory on PROD=lza-stag (guard above). ifneq ($(LZA_VPC_NAME),) export TF_VAR_lza_vpc_name=$(LZA_VPC_NAME) endif # Kit-file env suffix: PROD=true β†’ .production kits, PROD=lza β†’ .lza-prod kits (a # separate namespace so the parallel-running legacy prod kits are never touched -# during the FLIP#749 migration), otherwise β†’ .stag. Selects the trust kit file -# trust/.env.. for the trust-deploy targets. -KIT_ENV_SUFFIX := $(if $(filter true,$(PROD)),production,$(if $(filter lza,$(PROD)),lza-prod,stag)) +# during the FLIP#749 migration), PROD=lza-stag β†’ .lza-stag kits, otherwise β†’ .stag. +# Selects the trust kit file trust/.env.. for the trust-deploy targets. +KIT_ENV_SUFFIX := $(if $(filter true,$(PROD)),production,$(if $(filter lza,$(PROD)),lza-prod,$(if $(filter lza-stag,$(PROD)),lza-stag,stag))) export TF_VAR_max_azs=2 export TF_VAR_vpc_cidr=10.0.0.0/16 export TF_VAR_public_subnets=["10.0.1.0/24", "10.0.2.0/24"] @@ -241,7 +260,8 @@ CENTRALHUB_ECS_SERVICES = flip-api fl-api-net-1 fl-server-net-1 # recorded commit and rollback-centralhub can repoint at the previous revision. The # env-file tags (DOCKER_TAG / DOCKER_FL_TAG) remain the bootstrap-only defaults Terraform # bakes into a (re)created task-definition family β€” day-to-day deploys never touch them. -# PROD=lza tracks origin/main like legacy prod (both are production estates, FLIP#749). +# PROD=lza tracks origin/main like legacy prod (both are production estates, FLIP#749); +# PROD=lza-stag falls through to origin/develop like legacy stag. DEPLOY_GIT_REF := $(if $(filter true lza,$(PROD)),origin/main,origin/develop) TRUST_IMAGES = \ @@ -812,7 +832,7 @@ deploy-ui: ## Build flip-ui from the working tree and deploy it to S3 + invalida @# cross-account invalidation permission -- BY DESIGN (FLIP#749): the hashed @# bundles are immutable and index.html/window.js carry no-cache metadata, so a @# deploy is visible on the next index.html fetch with no invalidation at all. -ifeq ($(PROD),lza) +ifneq ($(filter lza lza-stag,$(PROD)),) @echo "🧹 Skipping CloudFront invalidation on LZA (edge distribution honours the Cache-Control metadata)." else @echo "🧹 Invalidating CloudFront..." diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index 157f35f37..aa9a546ee 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -586,8 +586,9 @@ The `PROD` variable determines which environment files are loaded: - `PROD=stag` β†’ Uses the root `.env.stag` - `PROD=true` β†’ Uses the root `.env.production` -- `PROD=lza` β†’ Uses the root `.env.lza-prod` (an LZA-governed estate β€” see +- `PROD=lza` β†’ Uses the root `.env.lza-prod` (production on an LZA-governed estate β€” see [Deploying onto an LZA estate](#deploying-onto-an-lza-estate-prodlza)) +- `PROD=lza-stag` β†’ Uses the root `.env.lza-stag` (staging on an LZA-governed estate) If `PROD` is omitted when running the AWS provider Makefile, it defaults to staging. @@ -647,7 +648,7 @@ This Terraform root supports **two deployment modes**, both permanently: | Mode | Selected by | Network | Ingress | | --- | --- | --- | --- | | **Self-contained** (default) | `PROD=stag` / `PROD=true` | FLIP creates its own VPC, subnets, IGW, NAT | In-account CloudFront + public FL NLB | -| **Platform-managed (LZA)** | `PROD=lza` | Discovered from the accelerator-provisioned VPC; FLIP creates none of it | Shared networking account's two-tier edge, over the Transit Gateway | +| **Platform-managed (LZA)** | `PROD=lza` / `PROD=lza-stag` | Discovered from the accelerator-provisioned VPC; FLIP creates none of it | Shared networking account's two-tier edge, over the Transit Gateway | Self-contained single-account is the supported open-source deployment shape and is not going away. The LZA mode ([FLIP#749](https://github.com/londonaicentre/FLIP/issues/749)) is the multi-account shape for estates running AWS's @@ -655,9 +656,27 @@ Self-contained single-account is the supported open-source deployment shape and the network, guardrails and edge are owned by the accelerator pipeline rather than by FLIP. The two are one root module, not a fork: every LZA adaptation is gated behind `var.lza_managed_network` (set from -`PROD=lza`), so with `PROD=true`/`PROD=stag` the resolved configuration is identical to before β€” the self-contained -environments are never touched by LZA work. The AI Centre's own LZA target is the **FLIPProduction** workload -account in `eu-west-2`, reached through the `lza-prod` profile alias and running alongside legacy prod. +the LZA `PROD` values), so with `PROD=true`/`PROD=stag` the resolved configuration is identical to before β€” the +self-contained environments are never touched by LZA work. The AI Centre's own LZA targets are the +**FLIPProduction** workload account in `eu-west-2` (the `lza-prod` profile alias, running alongside legacy prod) +and a staging workload account reached as `lza-stag`. + +Environment (prod vs stag semantics) and network mode are **orthogonal axes**, and the two `PROD` values set them +independently: + +| | `PROD=lza` | `PROD=lza-stag` | +| --- | --- | --- | +| Env file / kit suffix | `.env.lza-prod` / `trust/.env..lza-prod` | `.env.lza-stag` / `trust/.env..lza-stag` | +| Profile guard | `lza-prod` (`LZA_AWS_PROFILE`) | `lza-stag` (`LZA_STAG_AWS_PROFILE`) | +| `TF_VAR_environment` | `prod` β€” RDS deletion protection + final snapshot on | `stag` β€” disposable, like legacy stag | +| `deploy-centralhub` git ref | `origin/main` | `origin/develop` | +| `LZA_VPC_NAME` | optional (defaults to the prod template name) | **required** in the env file β€” there is no defaultable staging VPC name, so the Makefile refuses to run without it rather than letting the lookup fail opaquely | + +Everything below reads naturally for either value; where it says `PROD=lza`, staging substitutes `lza-stag` and +its own account-scoped values (state bucket, `flip-lza-stag-*` bucket namespace, pull-through registry URL). Each +LZA environment is its **own workload account** β€” never co-tenant two environments in one account: the stack's +resource names (`flip-cluster`, `flip-api`, `flip-database-proxy`, the `/flip/networking/*` handoff params) are +fixed per account by design. **What `PROD=lza` selects:** diff --git a/docs/source/deploy-flip/deploy-central-hub.rst b/docs/source/deploy-flip/deploy-central-hub.rst index e35a7c2d5..b4b970aca 100644 --- a/docs/source/deploy-flip/deploy-central-hub.rst +++ b/docs/source/deploy-flip/deploy-central-hub.rst @@ -35,7 +35,7 @@ flag, not a fork of the code. - FLIP creates its own VPC, subnets, internet gateway and NAT - In-account CloudFront for the UI and ``/api/*``; a public NLB for FL traffic * - **Platform-managed** - - ``PROD=lza`` + - ``PROD=lza`` (production) / ``PROD=lza-stag`` (staging) - Discovered from a VPC provisioned by the AWS `Landing Zone Accelerator `_; FLIP creates no network resources @@ -91,12 +91,13 @@ Prerequisites 5. **SSH key pair** at ``~/.ssh/host-aws`` β€” uploaded to AWS and used as the identity file for the SSM ProxyCommand-based SSH config. 6. **Environment file** β€” ``.env.stag`` (staging), ``.env.production`` - (production), or ``.env.lza-prod`` (platform-managed) in the project root. + (production), or ``.env.lza-prod`` / ``.env.lza-stag`` (platform-managed) in + the project root. 7. **AWS Session Manager plugin** β€” required for ``ssh flip`` and ``make forward-trust``. -AWS profile aliases (``prod``, ``stag``, ``dev``, and ``lza-prod`` for a -platform-managed deployment) should be configured in ``~/.aws/config`` so the -Makefile guards can verify the active profile against the chosen environment. +AWS profile aliases (``prod``, ``stag``, ``dev``, and ``lza-prod`` / ``lza-stag`` +for a platform-managed deployment) should be configured in ``~/.aws/config`` so +the Makefile guards can verify the active profile against the chosen environment. ************************ Required IAM permissions @@ -162,21 +163,23 @@ This runs, in order: 13. ``status`` β€” comprehensive health checks. The ``PROD`` variable selects the environment file (``stag`` β†’ ``.env.stag``, -``true`` β†’ ``.env.production``, ``lza`` β†’ ``.env.lza-prod``) and is mapped onto -``TF_VAR_environment`` β€” ``stag`` for staging, ``prod`` for both ``true`` and -``lza`` β€” so Terraform can gate prod-only RDS hardening (deletion protection, -final snapshot). ``PROD=lza`` additionally sets the platform-managed-network -flag, which is orthogonal to the environment name: a platform-managed -deployment is a production estate and keeps every prod-only control. +``true`` β†’ ``.env.production``, ``lza`` β†’ ``.env.lza-prod``, ``lza-stag`` β†’ +``.env.lza-stag``) and is mapped onto ``TF_VAR_environment`` β€” ``prod`` for +``true`` and ``lza``, ``stag`` otherwise β€” so Terraform can gate prod-only RDS +hardening (deletion protection, final snapshot). The two ``lza`` values +additionally set the platform-managed-network flag, which is orthogonal to the +environment name: ``PROD=lza`` is a production estate and keeps every prod-only +control, while ``PROD=lza-stag`` is staging semantics on the same +platform-managed network. .. note:: ``full-deploy`` is the self-contained chain. Some of its steps β€” the cloud trust EC2, the SSH and Ansible provisioning β€” do not apply to a platform-managed deployment, and the chain, along with ``make status`` and - ``make destroy``, is not yet exercised there. On ``PROD=lza`` run ``init`` / - ``plan`` / ``apply`` followed by ``deploy-centralhub`` and ``deploy-ui``; see - the AWS README section linked under `Deployment modes`_. + ``make destroy``, is not yet exercised there. On the ``lza`` values run + ``init`` / ``plan`` / ``apply`` followed by ``deploy-centralhub`` and + ``deploy-ui``; see the AWS README section linked under `Deployment modes`_. Subsequent UI-only deploys do not need Terraform: @@ -196,7 +199,7 @@ For debugging or selective steps: .. code-block:: shell - export PROD=stag # or: export PROD=true, or PROD=lza (platform-managed) + export PROD=stag # or: export PROD=true, or PROD=lza / PROD=lza-stag (platform-managed) make github-login make aws-login From 84a577a3fcda8f69d2184e1b874fa5ffa881e543 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Fri, 28 Aug 2026 15:24:16 +0100 Subject: [PATCH 34/41] fix(deploy/AWS): teach register-trusts.sh and add_fl_kits.sh the LZA PROD values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PROD=lza-stag (ef6456e4) was only half-plumbed: the two scripts that re-derive the environment from PROD on their own never learned the LZA values. - register-trusts.sh mapped true/lza/* β€” so PROD=lza-stag wrote `.stag` kits while the deploy Makefile's KIT_ENV_SUFFIX (and every trust-deploy target behind it) reads `.lza-stag`: registration "succeeded" and the next target failed with kit-not-found. Add the lza-stag case; keep the two in lockstep. - add_fl_kits.sh refused anything but stag|true, so the documented pool-growth path was dead on BOTH LZA values. The LZA pair now rides the legacy workspaces / project YAMLs on purpose: their kits carry the same server FQDN and root CA so trusts keep their client kits across the DNS cutover (FLIP#749); a staging estate provisioned with a fresh CA trips the existing fingerprint guard instead of minting silently. test_add_fl_kits.sh gains a PROD_MODE knob and covers lza-stag β†’ stag env, lza β†’ prod env, and an unknown value being refused before anything is touched. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/scripts/add_fl_kits.sh | 20 +++++++++++------ .../providers/AWS/scripts/register-trusts.sh | 10 +++++---- .../AWS/scripts/tests/test_add_fl_kits.sh | 22 ++++++++++++++++++- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/deploy/providers/AWS/scripts/add_fl_kits.sh b/deploy/providers/AWS/scripts/add_fl_kits.sh index 2c1d5a6d3..3e5670437 100755 --- a/deploy/providers/AWS/scripts/add_fl_kits.sh +++ b/deploy/providers/AWS/scripts/add_fl_kits.sh @@ -32,7 +32,8 @@ # trust registration β€” flip-api reconciles its pool on a miss, so no restart and no # task-definition change is needed. # -# Invoked by `make -C deploy/providers/AWS add-fl-kits N= PROD=stag|true [YES=1]`, +# Invoked by `make -C deploy/providers/AWS add-fl-kits N= PROD= [YES=1]` +# (PROD=stag|true, or lza|lza-stag for the LZA estate β€” FLIP#749), # which exports the env-file vars this script reads (AICENTRE_BUCKET_NAME, # FLARE_KIT_DATE, AWS_PROFILE, FL_BACKEND, PROD) plus N / YES / MAIN_ENV_FILE. # @@ -52,9 +53,9 @@ set -euo pipefail N="${N:?N is required (number of kit slots to add, e.g. N=2)}" -PROD="${PROD:?PROD is required (stag|true)}" -AICENTRE_BUCKET_NAME="${AICENTRE_BUCKET_NAME:?AICENTRE_BUCKET_NAME must be set (run via make with PROD=stag|true)}" -FLARE_KIT_DATE="${FLARE_KIT_DATE:?FLARE_KIT_DATE must be set (run via make with PROD=stag|true)}" +PROD="${PROD:?PROD is required (stag|true|lza|lza-stag)}" +AICENTRE_BUCKET_NAME="${AICENTRE_BUCKET_NAME:?AICENTRE_BUCKET_NAME must be set (run via make with PROD=)}" +FLARE_KIT_DATE="${FLARE_KIT_DATE:?FLARE_KIT_DATE must be set (run via make with PROD=)}" MAIN_ENV_FILE="${MAIN_ENV_FILE:?MAIN_ENV_FILE must be set (the env file to append FL_KIT_SLOT_NAMES to)}" YES="${YES:-}" @@ -69,11 +70,16 @@ if ! [[ "${N}" =~ ^[0-9]+$ ]] || [[ "${N}" -lt 1 ]]; then exit 1 fi +# ENV_NAME selects the CA workspace and project YAML (provision/workspace-, +# net-N_project_.yml). The LZA environments deliberately reuse the legacy +# ones: their kits carry the same server FQDN and root CA, so trusts keep their +# client kits across the DNS cutover (FLIP#749). A staging estate provisioned +# with a fresh CA instead trips the fingerprint guard below, never a silent mint. case "${PROD}" in -true) ENV_NAME="prod" ;; -stag) ENV_NAME="stag" ;; +true | lza) ENV_NAME="prod" ;; +stag | lza-stag) ENV_NAME="stag" ;; *) - echo "❌ PROD must be 'stag' or 'true', got '${PROD}'." >&2 + echo "❌ PROD must be one of 'stag', 'true', 'lza', 'lza-stag' β€” got '${PROD}'." >&2 exit 1 ;; esac diff --git a/deploy/providers/AWS/scripts/register-trusts.sh b/deploy/providers/AWS/scripts/register-trusts.sh index e2060ccef..bcc980cb8 100755 --- a/deploy/providers/AWS/scripts/register-trusts.sh +++ b/deploy/providers/AWS/scripts/register-trusts.sh @@ -29,8 +29,8 @@ # Usage: # KIT= register-trusts.sh # register one kit (trust/.env..) # register-trusts.sh # register every live trust/.env.*. kit -# PROD=true selects .production kits, PROD=lza selects .lza-prod kits (FLIP#749); -# anything else selects .stag. +# PROD=true selects .production kits, PROD=lza selects .lza-prod kits and +# PROD=lza-stag selects .lza-stag kits (FLIP#749); anything else selects .stag. set -eo pipefail # Default file mode 077 so any tempfile / redirect this script (or sourced @@ -52,11 +52,13 @@ TASK_FAMILY="${TASK_FAMILY:-flip-api}" LOG_GROUP="${LOG_GROUP:-/ecs/flip-api}" # Kit-file env suffix: PROD=true β†’ .production kits, PROD=lza β†’ .lza-prod kits, -# otherwise β†’ .stag. Matches KIT_ENV_SUFFIX in the deploy Makefile and the env -# file it included (.env.production / .env.lza-prod / .env.stag). +# PROD=lza-stag β†’ .lza-stag kits, otherwise β†’ .stag. Must match KIT_ENV_SUFFIX in +# the deploy Makefile (which reads the kit this script writes back) and the env +# file it included (.env.production / .env.lza-prod / .env.lza-stag / .env.stag). case "${PROD:-stag}" in true) ENV_SUFFIX="production" ;; lza) ENV_SUFFIX="lza-prod" ;; + lza-stag) ENV_SUFFIX="lza-stag" ;; *) ENV_SUFFIX="stag" ;; esac diff --git a/deploy/providers/AWS/scripts/tests/test_add_fl_kits.sh b/deploy/providers/AWS/scripts/tests/test_add_fl_kits.sh index 9209a7ef1..c03529b2f 100644 --- a/deploy/providers/AWS/scripts/tests/test_add_fl_kits.sh +++ b/deploy/providers/AWS/scripts/tests/test_add_fl_kits.sh @@ -120,6 +120,7 @@ FAIL=0 STARTUP_OVERRIDES=() KEYCOUNT_OVERRIDES=() YES_MODE="" +PROD_MODE="" # PROD value for the next run_case (default stag) run_case() { local name="$1" N="$2" env_json="$3" nets="$4" @@ -144,7 +145,7 @@ run_case() { LAST_ENVF="$(mktemp --tmpdir="${TEST_ROOT}")" printf 'SOMETHING=1\nFL_KIT_SLOT_NAMES=%s\nOTHER=2\n' "${env_json}" > "${LAST_ENVF}" LAST_OUT="$(printf 'n\n' | env PATH="${MOCKBIN}:${PATH}" \ - N="${N}" PROD=stag AICENTRE_BUCKET_NAME=testbucket FLARE_KIT_DATE=2026-01-01 \ + N="${N}" PROD="${PROD_MODE:-stag}" AICENTRE_BUCKET_NAME=testbucket FLARE_KIT_DATE=2026-01-01 \ MAIN_ENV_FILE="${LAST_ENVF}" FL_BACKEND=nvflare FIXTURE_DIR="${fx}" YES="${YES_MODE}" \ bash "${SCRIPT}" 2>&1)" LAST_RC=$? @@ -154,6 +155,7 @@ run_case() { STARTUP_OVERRIDES=() KEYCOUNT_OVERRIDES=() YES_MODE="" + PROD_MODE="" } check() { # assert the plan's activate/mint lines @@ -289,6 +291,24 @@ check_refuses "refuses net-2 overwrite" "already has objects β€” refusing to ove check_refuses "reports the net-1 upload" "Kits already uploaded this run: net-1/Trust_3" check_refuses "warns against the instinctive re-run" "Do NOT re-run add-fl-kits" +# --- PROD values (FLIP#749): the LZA pair rides the legacy env workspaces; anything else is refused --- + +PROD_MODE=lza-stag +run_case "PROD=lza-stag plans as the stag env (N=2)" 2 '["Trust_1", "Trust_2"]' "net-1 net-2" \ + "net-1=Trust_1 Trust_2 fl-server-net-1" "net-2=Trust_1 Trust_2 fl-server-net-2" +check "activate none, mint Trust_3 Trust_4" "(none β€” no spare kits)" "Trust_3 Trust_4 (existing max: Trust_2)" +check_contains "plan header names the stag env" 1 "Plan (stag):" + +PROD_MODE=lza +run_case "PROD=lza plans as the prod env (N=1)" 1 '["Trust_1", "Trust_2"]' "net-1 net-2" \ + "net-1=Trust_1 Trust_2 fl-server-net-1" "net-2=Trust_1 Trust_2 fl-server-net-2" +check_contains "plan header names the prod env" 1 "Plan (prod):" + +PROD_MODE=lza-bogus +run_case "unknown PROD value refused (N=1)" 1 '["Trust_1", "Trust_2"]' "net-1 net-2" \ + "net-1=Trust_1 Trust_2 fl-server-net-1" "net-2=Trust_1 Trust_2 fl-server-net-2" +check_refuses "rejects PROD=lza-bogus before touching anything" "PROD must be one of" + echo "" echo "==== ${PASS} passed, ${FAIL} failed ====" [[ "${FAIL}" -eq 0 ]] From ac1951c9fb65430d7112c96f2345fc98a4b81efd Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Fri, 28 Aug 2026 15:24:16 +0100 Subject: [PATCH 35/41] fix(fl-services): load .env.lza-prod / .env.lza-stag for the kit-upload targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `make -C fl-services/ upload-kits-to-s3 PROD=lza-stag` (and Flower's upload-creds-to-s3) fell through the true|stag env-file chain to .env.development β€” the dev bucket, with no profile guard and no error. Add the two LZA branches, mirroring the deploy Makefile, and treat lza like true for FL_KIT_WORKSPACE (the prod workspace β€” same FQDN + CA by design, FLIP#749). Signed-off-by: at24_bioeng625-pc --- fl-services/flower/Makefile | 12 ++++++++---- fl-services/nvflare/Makefile | 20 +++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/fl-services/flower/Makefile b/fl-services/flower/Makefile index 9183ebfd2..7642dee90 100644 --- a/fl-services/flower/Makefile +++ b/fl-services/flower/Makefile @@ -19,13 +19,17 @@ REPO_ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/../..) # Load the env file selected by PROD (dev by default), exactly as -# fl-services/nvflare/Makefile does β€” with PROD=stag|true this brings in the -# upload vars (AICENTRE_BUCKET_NAME, FLOWER_KIT_DATE) used by -# upload-creds-to-s3. +# fl-services/nvflare/Makefile does β€” with PROD=stag|true (or lza|lza-stag for +# the LZA estate, FLIP#749) this brings in the upload vars (AICENTRE_BUCKET_NAME, +# FLOWER_KIT_DATE) used by upload-creds-to-s3. ifeq ($(PROD),true) MAIN_ENV_FILE := $(REPO_ROOT)/.env.production else ifeq ($(PROD),stag) MAIN_ENV_FILE := $(REPO_ROOT)/.env.stag +else ifeq ($(PROD),lza) +MAIN_ENV_FILE := $(REPO_ROOT)/.env.lza-prod +else ifeq ($(PROD),lza-stag) +MAIN_ENV_FILE := $(REPO_ROOT)/.env.lza-stag else MAIN_ENV_FILE := $(REPO_ROOT)/.env.development endif @@ -71,7 +75,7 @@ provision: # make provision NET_NUMBER=1 # make upload-creds-to-s3 NET_NUMBER=1 PROD=stag upload-creds-to-s3: - @[ -n "$(AICENTRE_BUCKET_NAME)" ] || { echo "❌ AICENTRE_BUCKET_NAME not set β€” run with PROD=stag|true"; exit 1; } + @[ -n "$(AICENTRE_BUCKET_NAME)" ] || { echo "❌ AICENTRE_BUCKET_NAME not set β€” run with PROD=stag|true|lza|lza-stag"; exit 1; } @[ -n "$(FLOWER_KIT_DATE)" ] || { echo "❌ FLOWER_KIT_DATE not set β€” add it to the env file (e.g. 20260811)"; exit 1; } @echo "⬆️ Uploading Flower creds (net-$(NET_NUMBER)) β†’ s3://$(AICENTRE_BUCKET_NAME)/fl-flower-participant-kits/$(FLOWER_KIT_DATE)/net-$(NET_NUMBER) $(DRYRUN)" aws s3 sync ./provision/creds/net-$(NET_NUMBER) s3://$(AICENTRE_BUCKET_NAME)/fl-flower-participant-kits/$(FLOWER_KIT_DATE)/net-$(NET_NUMBER) --delete --exclude "certificates/ca.key" $(DRYRUN) diff --git a/fl-services/nvflare/Makefile b/fl-services/nvflare/Makefile index 38036d932..23421e47c 100644 --- a/fl-services/nvflare/Makefile +++ b/fl-services/nvflare/Makefile @@ -21,12 +21,17 @@ REPO_ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/../..) # Load the env file selected by PROD (dev by default), exactly as the root Makefile # does, so compose + the provisioning targets see the dev vars (FL_API_PORT, -# LOG_LEVEL, NET_ENDPOINTS, FL_PORT, …) and, with PROD=stag|true, the upload vars -# (AICENTRE_BUCKET_NAME, FLARE_KIT_DATE) used by upload-kits-to-s3. +# LOG_LEVEL, NET_ENDPOINTS, FL_PORT, …) and, with PROD=stag|true (or lza|lza-stag +# for the LZA estate, FLIP#749), the upload vars (AICENTRE_BUCKET_NAME, +# FLARE_KIT_DATE) used by upload-kits-to-s3. ifeq ($(PROD),true) MAIN_ENV_FILE := $(REPO_ROOT)/.env.production else ifeq ($(PROD),stag) MAIN_ENV_FILE := $(REPO_ROOT)/.env.stag +else ifeq ($(PROD),lza) +MAIN_ENV_FILE := $(REPO_ROOT)/.env.lza-prod +else ifeq ($(PROD),lza-stag) +MAIN_ENV_FILE := $(REPO_ROOT)/.env.lza-stag else MAIN_ENV_FILE := $(REPO_ROOT)/.env.development endif @@ -110,13 +115,14 @@ provision-add-client-prod: # Upload the provisioned FLARE participant kits to S3 under the FLARE_KIT_DATE prefix # the deploy flow pulls from (deploy/providers/AWS Makefile `provision-local-trust`, # which reads s3://$(AICENTRE_BUCKET_NAME)/fl-flare-participant-kits/$(FLARE_KIT_DATE)/ -# net-N/services//). Run with PROD=stag|true so AICENTRE_BUCKET_NAME + FLARE_KIT_DATE -# load from the matching .env. Defaults to a dry run β€” pass DRYRUN= to upload for real. -FL_KIT_WORKSPACE := $(if $(filter true,$(PROD)),$(PROD_WORKSPACE_DIR),$(STAG_WORKSPACE_DIR)) +# net-N/services//). Run with PROD=stag|true (or lza|lza-stag) so AICENTRE_BUCKET_NAME + +# FLARE_KIT_DATE load from the matching .env. Defaults to a dry run β€” pass DRYRUN= to upload +# for real. The LZA pair maps onto the legacy workspaces (same FQDN + CA by design, FLIP#749). +FL_KIT_WORKSPACE := $(if $(filter true lza,$(PROD)),$(PROD_WORKSPACE_DIR),$(STAG_WORKSPACE_DIR)) DRYRUN ?= --dryrun upload-kits-to-s3: - @[ -n "$(AICENTRE_BUCKET_NAME)" ] || { echo "❌ AICENTRE_BUCKET_NAME not set β€” run with PROD=stag|true"; exit 1; } - @[ -n "$(FLARE_KIT_DATE)" ] || { echo "❌ FLARE_KIT_DATE not set β€” run with PROD=stag|true"; exit 1; } + @[ -n "$(AICENTRE_BUCKET_NAME)" ] || { echo "❌ AICENTRE_BUCKET_NAME not set β€” run with PROD=stag|true|lza|lza-stag"; exit 1; } + @[ -n "$(FLARE_KIT_DATE)" ] || { echo "❌ FLARE_KIT_DATE not set β€” run with PROD=stag|true|lza|lza-stag"; exit 1; } @echo "⬆️ Uploading FLARE kits (net-$(NET_NUMBER), $(FL_KIT_WORKSPACE)) β†’ s3://$(AICENTRE_BUCKET_NAME)/fl-flare-participant-kits/$(FLARE_KIT_DATE)/net-$(NET_NUMBER) $(DRYRUN)" aws s3 sync ./$(FL_KIT_WORKSPACE)/net-$(NET_NUMBER) s3://$(AICENTRE_BUCKET_NAME)/fl-flare-participant-kits/$(FLARE_KIT_DATE)/net-$(NET_NUMBER) --delete $(DRYRUN) From d5d74b6ba386dafeca4ea24c489a34e0b3a65645 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Fri, 28 Aug 2026 15:24:16 +0100 Subject: [PATCH 36/41] fix(make): name LZA trust kits .lza-prod / .lza-stag from the root kit-file targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `make new-trust … PROD=lza-stag` β€” the exact hint register-trusts.sh prints β€” scaffolded trust/.env..development, and sync-trust-kit(s) read .env.development, because the root Makefile's ENV / MAIN_ENV_FILE only knew true|stag. Both now follow the deploy Makefile's KIT_ENV_SUFFIX (.lza-prod / .lza-stag), and new_trust.py accepts the two suffixes and hints the PROD flag that reads them back (lza-prod β†’ PROD=lza). ENV is only ever a kit-file token at the root β€” the composes hardcode their own ENV β€” so nothing else moves. CLAUDE.md (+ the AGENTS.md twin): the PROD bullet now names the kit-file and FL-kit-upload targets the LZA values reach, and MAIN_ENV_FILE's default list carries the two LZA files. Signed-off-by: at24_bioeng625-pc --- AGENTS.md | 4 ++-- CLAUDE.md | 4 ++-- Makefile | 12 ++++++++++++ scripts/new_trust.py | 14 ++++++++++++-- scripts/sync_trust_kit.py | 6 +++--- scripts/tests/test_new_trust.py | 15 +++++++++++++++ 6 files changed, 46 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 72a272a51..78261c283 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -411,9 +411,9 @@ After changes, evaluate if docs need updating: - `FL_PROVISIONED_DIR` β€” path to the in-tree provisioned FL artifacts, derived per-backend by `deploy/fl_backend.mk` from `FL_BACKEND`: `fl-services/nvflare/provision/workspace-dev` (nvflare startup kits) or `fl-services/flower/provision/creds` (flower per-net TLS certs + SuperNode keys). Both gitignored. Read only by the dev compose overlays for the cert/workspace volume mounts; override at the CLI for a one-off (`make up FL_PROVISIONED_DIR=...`). FL Makefiles are **per-backend** β€” each `fl-services//Makefile` owns that backend's `build`/`provision`/`up`/`down`/`submit` (flower also `up-secure`); the root Makefile forwards only `build-fl` by `FL_BACKEND`. Each backend's `fl-services//Makefile` also owns its network provisioning (NVFLARE adds `provision`/`provision-2-nets`/`provision-stag`/`provision-prod`/`upload-kits-to-s3`; the project YAMLs, `scripts/`, and gitignored `workspace-{dev,stag,prod}/` output live under `provision/`). Provision with `make -C fl-services/nvflare provision-2-nets` (nvflare) or `make -C fl-services/flower provision NET_NUMBER=` (flower). To run a backend standalone + submit without the full stack: `make -C fl-services/ up` (or `up-secure`) then `make -C fl-services/ submit APP=`. - `FL_APP_BASE_DIR` β€” Local directory holding the base FL application templates (the repo's `fl-apps/` tree), baked into the flip-api image and bind-mounted in dev. flip-api walks `///` to bundle an application (uploading those files into `FL_APP_DESTINATION_BUCKET/`) and reads each backend's manifest from `//required_files.json`. Default `/app/fl-apps`; override to mount operator-provided templates. Replaces the removed `FL_APP_BASE_BUCKET` S3 dependency (FLIP#724): base templates are no longer published to S3 (the `fl-apps-push-s3-*` sync workflows are gone), so a template hotfix now ships by rebuilding + redeploying the flip-api image rather than syncing S3. `fl-apps/` is baked into the image via a BuildKit named build context (`fl_apps=../fl-apps`) since it sits outside flip-api's build context. For the Flower backend, the template pyprojects also steer Flower's **per-run dependency install** (`uv sync` on every app launch; SuperNodes opt in via `--allow-runtime-dependency-installation` in the composes): `[tool.uv.sources]` pins `flip-utils` to the source kept at `/opt/flip-utils` inside the FL images (never PyPI β€” FLIP#767; a flip-utils change ships by rebuilding the FL images, `make build-fl FL_BACKEND=flower`) and torch/torchvision to the cu128 index (PyPI's default cu130 wheels need driver >=580). - `FL_KIT_SLOT_NAMES` β€” JSON list (e.g. `["Trust_1", "Trust_2"]`) of FL kit-slot names for the hub's `fl_kit_slot` pool that `register_trust` claims from; each name must match a provisioned participant kit (in-tree workspace for dev, `s3:///fl-flare-participant-kits//net-/services//` for stag/prod; slot names are global across nets β€” every net carries a kit per name). The pool is seeded at flip-api boot and **reconciled on demand** when a registration finds it exhausted (`resolve_fl_kit_slot_names`, additive β€” never deletes or re-assigns rows); only then does `NoFreeKitSlotError` surface. Single source per env: dev = this env var (a `DevSettings`-only field; restart to change, settings load once); stag/prod = the `/flip/fl_kit_slot_names` SSM parameter (Terraform-rendered from this var β€” the list is plain config, not a secret; deliberately **no env fallback**, so a broken/missing parameter means the pool can't grow, loudly, never masked by stale task-def env). Growing the pool is an env-file edit + `make -C deploy/providers/AWS apply-fl-kit-slots` (targeted plan/apply of just the parameter, plain-text diff) β€” **no restart, no task-definition change**. One-command workflow: `make -C deploy/providers/AWS add-fl-kits N= PROD=stag|true` (N = "ensure N more live slots": activate spares toward N first, mint only the shortfall on every net β†’ additive S3 upload β†’ env edit β†’ parameter apply); full runbook in `fl-services/nvflare/README.md` ("Onboarding a new client onto an existing network"). NVFLARE-only dynamics β€” Flower's SuperNode key labelling reads the list at net startup. -- `PROD` β€” `true` (production), `stag` (staging), `lza` / `lza-stag` (production / staging on an AWS Landing Zone Accelerator estate, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets only; select the root `.env.lza-prod` / `.env.lza-stag` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying onto an LZA estate"), unset (development) +- `PROD` β€” `true` (production), `stag` (staging), `lza` / `lza-stag` (production / staging on an AWS Landing Zone Accelerator estate, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets, the FL-kit upload targets under `fl-services//`, and the kit-file targets here (`new-trust`, `sync-trust-kit[s]`), which must name `trust/.env..lza-prod` / `.lza-stag` the way the AWS side reads them back; select the root `.env.lza-prod` / `.env.lza-stag` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying onto an LZA estate"), unset (development) - `FLIP_INSTANCE` β€” names a **second dev hub** so two stacks can run on one host (FLIP#957). Unset (the norm) every derived name is exactly what it was before the knob existed. When set it prefixes four things, all of them names that are global to the docker daemon and so cannot be scoped by `-p`: the hub compose project name (`COMPOSE_PROJECT`), all six hub-side Docker networks, each trust's compose project (`TRUST_PROJECT`, `-trust`) and each trust's XNAT swarm stack (`XNAT_STACK`, `-xnat`). The last two carry it for the same reason the trust overlays do β€” FL kit slots are handed out per hub, so a second hub restarts its numbering at 1 and its first trust would otherwise adopt or deploy over the default stack's `trust1`/`xnat1`. Those six follow **one** name rule β€” `${FLIP_INSTANCE:+$FLIP_INSTANCE-}deploy_`, i.e. `deploy_central-hub-network`, `deploy_central-hub-trust-apis-network`, `deploy_fl-net-{1,2}` **and** `deploy_trust-network-{1,2}` β€” where `deploy_` names the hub compose project that owns the network, which is also exactly what compose would generate (`_`) if it still created them. It stopped generating anything once they became `external: true` ("the `name` field is used as is and is not scoped with the project name"), which is why the prefix is written out by hand in the composes, both Makefiles and `scripts/check_local_status.py`. FLIP#957 renamed **four** of them onto that rule: `central-hub-network` and `central-hub-trust-apis-network` had been left bare, and `shared-net-{1,2}` became `fl-net-{1,2}` β€” "shared" described who happened to be attached rather than what the network is for, and stopped being true the moment flip-api came off it. The two numbering axes are **not** parallel, which is the trap the new name defuses: `fl-net-` is numbered by **FL net** (the `fl_nets` table, `NET_NUMBER`, `fl-server-net-`) while `trust-network-` is numbered by **trust slot**, so one trust routinely sits on `trust-network-2` and `fl-net-1` at once. `fl-net-` is the FL **data plane**, and its membership is exactly two services β€” the hub's `fl-server-net-` and every trust's `fl-client-net-`, making it the FL twin of `central-hub-trust-apis-network` (flip-api ↔ trust-api). Everything else hub-side stays on `default` and reaches the FL server there over its control ports: **`flip-api` is deliberately not on it**, and neither are `fl-api-net-` or flower's `register-supernode-keys-net-`: it fronts the database, nothing on the FL data plane calls it (an fl-client carries no hub URL and no hub credential, only `TRUST_INTERNAL_SERVICE_KEY`), and the one callback that exists β€” fl-server's `FLIP_API_INTERNAL_URL` β€” goes over `default`, the hub-internal network, which both fl-servers now join. Because these networks are external and pre-created, all four renames are **not** transparent β€” an existing dev host must re-run `make create-networks`, recreate whatever was attached to the old ones, and `docker network rm` the four leftovers. The project name matters because compose derives it from the directory of the first `-f` file, always `deploy/`, so without it both stacks land in project `deploy` and `up` on one tears down the other. The networks are prefixed *separately* rather than left to `-p` because they are a cross-project contract: the hub project creates them and each trust β€” its own compose project β€” joins them `external: true` by literal name, which `-p` cannot scope. The trust overlays carry the prefix like everything else because a trust number alone does not isolate them: slots are handed out per-hub, so a second hub restarts its numbering at 1 and its first trust would otherwise land on the default stack's `deploy_trust-network-1`. Since every network belongs to exactly one instance, `make remove-networks FLIP_INSTANCE=` is instance-scoped and removes that instance's networks *including* its trust overlays. Both values are derived once in [`deploy/instance.mk`](deploy/instance.mk) (`INSTANCE_PREFIX` and `COMPOSE_PROJECT`), included by the root, trust, `trust/xnat`, flip-api and flip-ui Makefiles, which also exports `FLIP_INSTANCE` so the compose files see it; the compose files interpolate `${FLIP_INSTANCE}` rather than the make variable because they are also invoked directly, without make. **Container names are not prefixed and are not set at all** β€” no hub service declares `container_name`, in the development composes or the production ones, so compose names them from the project. On the default stack that is `deploy-flip-api-1`, `deploy-flip-db-1`, `deploy-flip-ui-1`, `deploy-pgadmin-1`, `deploy-fl-api-net-1-1`, `deploy-fl-server-net-1-1`; on a second stack `-deploy-…`. Since FLIP normally runs a single hub those default-stack names are deterministic, so **docs spell them out literally** β€” a concrete name is easier to read, copy and grep than a `docker compose -p deploy exec ` form β€” and only a doc explicitly about a second stack needs the `-deploy-…` variant. **Tooling must not**: a Makefile or script has to work on either instance, so it addresses the service through compose (`$(DOCKER_COMMAND) logs flip-api`) or resolves the container from the `com.docker.compose.project` / `.service` labels, as [`scripts/check_local_status.py`](scripts/check_local_status.py) does. (None of this reaches AWS: the production composes are a local prod-image harness, never a deployment target β€” on ECS the container names come from the task definitions in `deploy/providers/AWS/ecs_tasks.tf` and discovery from Cloud Map.) The compose **service keys** are the stable identity instead, and are load-bearing: docker registers each as a network alias on every network the container joins, which is how the FL kits keep resolving `fl-server-net-1` (`fed_client.json`'s `target`, and the SuperLink certificate's SAN) and how `NET_ENDPOINTS` reaches `fl-api-net-1`. Renaming a service key breaks TLS and the provisioned kits; renaming a container name breaks nothing. A second stack also needs its own value for every host port (`UI_PORT`, `API_PORT`, `DB_PORT`, `PGADMIN_PORT`, `FL_API_PORT`, `API_DEBUG_PORT`, `FL_API_DEBUG_PORT`, and on flower `FLOWER_SUPERLINK_NET_{1,2}_PORT`), its own `CENTRAL_HUB_API_URL` (it embeds `API_PORT`, so a copied env file silently points the second stack's UI and trust-api at the *first* stack's API β€” the one misconfiguration here that still starts cleanly and looks right), its own `XNAT_PORT` per trust kit, and its own trust numbers β€” all host-global and not covered by the prefix. Two traps when running one: `NET_ENDPOINTS` (which the hub seeds into `fl_nets` at boot) must carry the service name `fl-api-net-1`, never the pre-FLIP#957 `flip-fl-api-net-1`, which is now a DNS name in no stack at all; and `FL_PROVISIONED_DIR` may be given as either a relative or an absolute path (relative resolves against the repo root). -- `MAIN_ENV_FILE` β€” which repo-root env file the Makefiles load, as a bare **filename** (never a path). Defaults by `PROD` to `.env.production` / `.env.stag` / `.env.development`; override it to run a second stack from the *same checkout* β€” `make up MAIN_ENV_FILE=.env.b.development FLIP_INSTANCE=b` β€” instead of needing a second clone. The root Makefile `export`s it, and the flip-api, flip-ui and trust Makefiles each `include ../$(MAIN_ENV_FILE)`, which is why the value must stay a bare filename: each one prepends its own `../`. Every include is wildcard-guarded, so a value that resolves to nothing is **skipped silently** and the service builds with no environment at all rather than failing β€” check the `Using MAIN_ENV_FILE:` line each Makefile prints if a second stack comes up with empty config. `scripts/check_local_status.py` honours it too (via the environment), so a status check reports on the stack you are actually running. +- `MAIN_ENV_FILE` β€” which repo-root env file the Makefiles load, as a bare **filename** (never a path). Defaults by `PROD` to `.env.production` / `.env.stag` / `.env.lza-prod` / `.env.lza-stag` / `.env.development`; override it to run a second stack from the *same checkout* β€” `make up MAIN_ENV_FILE=.env.b.development FLIP_INSTANCE=b` β€” instead of needing a second clone. The root Makefile `export`s it, and the flip-api, flip-ui and trust Makefiles each `include ../$(MAIN_ENV_FILE)`, which is why the value must stay a bare filename: each one prepends its own `../`. Every include is wildcard-guarded, so a value that resolves to nothing is **skipped silently** and the service builds with no environment at all rather than failing β€” check the `Using MAIN_ENV_FILE:` line each Makefile prints if a second stack comes up with empty config. `scripts/check_local_status.py` honours it too (via the environment), so a status check reports on the stack you are actually running. - `DB_PORT` β€” in **dev** this is the *host* port flip-db is published on (the container side is pinned to 5432, which is what flip-api dials over the docker network); in `.env.stag` / `.env.production` it is the port the RDS Proxy listens on, since nothing is published there. Both default to 5432, so the difference only shows up when a second dev stack moves it. - `AES_KEY_BASE64` β€” encryption key for trust communication - A remote trust operator only needs their kit file (`trust/.env.`) β€” no hub `.env.` needed on trust hosts. diff --git a/CLAUDE.md b/CLAUDE.md index 4146ae743..7264ad6a7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -411,9 +411,9 @@ After changes, evaluate if docs need updating: - `FL_PROVISIONED_DIR` β€” path to the in-tree provisioned FL artifacts, derived per-backend by `deploy/fl_backend.mk` from `FL_BACKEND`: `fl-services/nvflare/provision/workspace-dev` (nvflare startup kits) or `fl-services/flower/provision/creds` (flower per-net TLS certs + SuperNode keys). Both gitignored. Read only by the dev compose overlays for the cert/workspace volume mounts; override at the CLI for a one-off (`make up FL_PROVISIONED_DIR=...`). FL Makefiles are **per-backend** β€” each `fl-services//Makefile` owns that backend's `build`/`provision`/`up`/`down`/`submit` (flower also `up-secure`); the root Makefile forwards only `build-fl` by `FL_BACKEND`. Each backend's `fl-services//Makefile` also owns its network provisioning (NVFLARE adds `provision`/`provision-2-nets`/`provision-stag`/`provision-prod`/`upload-kits-to-s3`; the project YAMLs, `scripts/`, and gitignored `workspace-{dev,stag,prod}/` output live under `provision/`). Provision with `make -C fl-services/nvflare provision-2-nets` (nvflare) or `make -C fl-services/flower provision NET_NUMBER=` (flower). To run a backend standalone + submit without the full stack: `make -C fl-services/ up` (or `up-secure`) then `make -C fl-services/ submit APP=`. - `FL_APP_BASE_DIR` β€” Local directory holding the base FL application templates (the repo's `fl-apps/` tree), baked into the flip-api image and bind-mounted in dev. flip-api walks `///` to bundle an application (uploading those files into `FL_APP_DESTINATION_BUCKET/`) and reads each backend's manifest from `//required_files.json`. Default `/app/fl-apps`; override to mount operator-provided templates. Replaces the removed `FL_APP_BASE_BUCKET` S3 dependency (FLIP#724): base templates are no longer published to S3 (the `fl-apps-push-s3-*` sync workflows are gone), so a template hotfix now ships by rebuilding + redeploying the flip-api image rather than syncing S3. `fl-apps/` is baked into the image via a BuildKit named build context (`fl_apps=../fl-apps`) since it sits outside flip-api's build context. For the Flower backend, the template pyprojects also steer Flower's **per-run dependency install** (`uv sync` on every app launch; SuperNodes opt in via `--allow-runtime-dependency-installation` in the composes): `[tool.uv.sources]` pins `flip-utils` to the source kept at `/opt/flip-utils` inside the FL images (never PyPI β€” FLIP#767; a flip-utils change ships by rebuilding the FL images, `make build-fl FL_BACKEND=flower`) and torch/torchvision to the cu128 index (PyPI's default cu130 wheels need driver >=580). - `FL_KIT_SLOT_NAMES` β€” JSON list (e.g. `["Trust_1", "Trust_2"]`) of FL kit-slot names for the hub's `fl_kit_slot` pool that `register_trust` claims from; each name must match a provisioned participant kit (in-tree workspace for dev, `s3:///fl-flare-participant-kits//net-/services//` for stag/prod; slot names are global across nets β€” every net carries a kit per name). The pool is seeded at flip-api boot and **reconciled on demand** when a registration finds it exhausted (`resolve_fl_kit_slot_names`, additive β€” never deletes or re-assigns rows); only then does `NoFreeKitSlotError` surface. Single source per env: dev = this env var (a `DevSettings`-only field; restart to change, settings load once); stag/prod = the `/flip/fl_kit_slot_names` SSM parameter (Terraform-rendered from this var β€” the list is plain config, not a secret; deliberately **no env fallback**, so a broken/missing parameter means the pool can't grow, loudly, never masked by stale task-def env). Growing the pool is an env-file edit + `make -C deploy/providers/AWS apply-fl-kit-slots` (targeted plan/apply of just the parameter, plain-text diff) β€” **no restart, no task-definition change**. One-command workflow: `make -C deploy/providers/AWS add-fl-kits N= PROD=stag|true` (N = "ensure N more live slots": activate spares toward N first, mint only the shortfall on every net β†’ additive S3 upload β†’ env edit β†’ parameter apply); full runbook in `fl-services/nvflare/README.md` ("Onboarding a new client onto an existing network"). NVFLARE-only dynamics β€” Flower's SuperNode key labelling reads the list at net startup. -- `PROD` β€” `true` (production), `stag` (staging), `lza` / `lza-stag` (production / staging on an AWS Landing Zone Accelerator estate, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets only; select the root `.env.lza-prod` / `.env.lza-stag` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying onto an LZA estate"), unset (development) +- `PROD` β€” `true` (production), `stag` (staging), `lza` / `lza-stag` (production / staging on an AWS Landing Zone Accelerator estate, FLIP#749 β€” meaningful for `deploy/providers/AWS` targets, the FL-kit upload targets under `fl-services//`, and the kit-file targets here (`new-trust`, `sync-trust-kit[s]`), which must name `trust/.env..lza-prod` / `.lza-stag` the way the AWS side reads them back; select the root `.env.lza-prod` / `.env.lza-stag` and the platform-managed-network Terraform path, see `deploy/providers/AWS/README.md` "Deploying onto an LZA estate"), unset (development) - `FLIP_INSTANCE` β€” names a **second dev hub** so two stacks can run on one host (FLIP#957). Unset (the norm) every derived name is exactly what it was before the knob existed. When set it prefixes four things, all of them names that are global to the docker daemon and so cannot be scoped by `-p`: the hub compose project name (`COMPOSE_PROJECT`), all six hub-side Docker networks, each trust's compose project (`TRUST_PROJECT`, `-trust`) and each trust's XNAT swarm stack (`XNAT_STACK`, `-xnat`). The last two carry it for the same reason the trust overlays do β€” FL kit slots are handed out per hub, so a second hub restarts its numbering at 1 and its first trust would otherwise adopt or deploy over the default stack's `trust1`/`xnat1`. Those six follow **one** name rule β€” `${FLIP_INSTANCE:+$FLIP_INSTANCE-}deploy_`, i.e. `deploy_central-hub-network`, `deploy_central-hub-trust-apis-network`, `deploy_fl-net-{1,2}` **and** `deploy_trust-network-{1,2}` β€” where `deploy_` names the hub compose project that owns the network, which is also exactly what compose would generate (`_`) if it still created them. It stopped generating anything once they became `external: true` ("the `name` field is used as is and is not scoped with the project name"), which is why the prefix is written out by hand in the composes, both Makefiles and `scripts/check_local_status.py`. FLIP#957 renamed **four** of them onto that rule: `central-hub-network` and `central-hub-trust-apis-network` had been left bare, and `shared-net-{1,2}` became `fl-net-{1,2}` β€” "shared" described who happened to be attached rather than what the network is for, and stopped being true the moment flip-api came off it. The two numbering axes are **not** parallel, which is the trap the new name defuses: `fl-net-` is numbered by **FL net** (the `fl_nets` table, `NET_NUMBER`, `fl-server-net-`) while `trust-network-` is numbered by **trust slot**, so one trust routinely sits on `trust-network-2` and `fl-net-1` at once. `fl-net-` is the FL **data plane**, and its membership is exactly two services β€” the hub's `fl-server-net-` and every trust's `fl-client-net-`, making it the FL twin of `central-hub-trust-apis-network` (flip-api ↔ trust-api). Everything else hub-side stays on `default` and reaches the FL server there over its control ports: **`flip-api` is deliberately not on it**, and neither are `fl-api-net-` or flower's `register-supernode-keys-net-`: it fronts the database, nothing on the FL data plane calls it (an fl-client carries no hub URL and no hub credential, only `TRUST_INTERNAL_SERVICE_KEY`), and the one callback that exists β€” fl-server's `FLIP_API_INTERNAL_URL` β€” goes over `default`, the hub-internal network, which both fl-servers now join. Because these networks are external and pre-created, all four renames are **not** transparent β€” an existing dev host must re-run `make create-networks`, recreate whatever was attached to the old ones, and `docker network rm` the four leftovers. The project name matters because compose derives it from the directory of the first `-f` file, always `deploy/`, so without it both stacks land in project `deploy` and `up` on one tears down the other. The networks are prefixed *separately* rather than left to `-p` because they are a cross-project contract: the hub project creates them and each trust β€” its own compose project β€” joins them `external: true` by literal name, which `-p` cannot scope. The trust overlays carry the prefix like everything else because a trust number alone does not isolate them: slots are handed out per-hub, so a second hub restarts its numbering at 1 and its first trust would otherwise land on the default stack's `deploy_trust-network-1`. Since every network belongs to exactly one instance, `make remove-networks FLIP_INSTANCE=` is instance-scoped and removes that instance's networks *including* its trust overlays. Both values are derived once in [`deploy/instance.mk`](deploy/instance.mk) (`INSTANCE_PREFIX` and `COMPOSE_PROJECT`), included by the root, trust, `trust/xnat`, flip-api and flip-ui Makefiles, which also exports `FLIP_INSTANCE` so the compose files see it; the compose files interpolate `${FLIP_INSTANCE}` rather than the make variable because they are also invoked directly, without make. **Container names are not prefixed and are not set at all** β€” no hub service declares `container_name`, in the development composes or the production ones, so compose names them from the project. On the default stack that is `deploy-flip-api-1`, `deploy-flip-db-1`, `deploy-flip-ui-1`, `deploy-pgadmin-1`, `deploy-fl-api-net-1-1`, `deploy-fl-server-net-1-1`; on a second stack `-deploy-…`. Since FLIP normally runs a single hub those default-stack names are deterministic, so **docs spell them out literally** β€” a concrete name is easier to read, copy and grep than a `docker compose -p deploy exec ` form β€” and only a doc explicitly about a second stack needs the `-deploy-…` variant. **Tooling must not**: a Makefile or script has to work on either instance, so it addresses the service through compose (`$(DOCKER_COMMAND) logs flip-api`) or resolves the container from the `com.docker.compose.project` / `.service` labels, as [`scripts/check_local_status.py`](scripts/check_local_status.py) does. (None of this reaches AWS: the production composes are a local prod-image harness, never a deployment target β€” on ECS the container names come from the task definitions in `deploy/providers/AWS/ecs_tasks.tf` and discovery from Cloud Map.) The compose **service keys** are the stable identity instead, and are load-bearing: docker registers each as a network alias on every network the container joins, which is how the FL kits keep resolving `fl-server-net-1` (`fed_client.json`'s `target`, and the SuperLink certificate's SAN) and how `NET_ENDPOINTS` reaches `fl-api-net-1`. Renaming a service key breaks TLS and the provisioned kits; renaming a container name breaks nothing. A second stack also needs its own value for every host port (`UI_PORT`, `API_PORT`, `DB_PORT`, `PGADMIN_PORT`, `FL_API_PORT`, `API_DEBUG_PORT`, `FL_API_DEBUG_PORT`, and on flower `FLOWER_SUPERLINK_NET_{1,2}_PORT`), its own `CENTRAL_HUB_API_URL` (it embeds `API_PORT`, so a copied env file silently points the second stack's UI and trust-api at the *first* stack's API β€” the one misconfiguration here that still starts cleanly and looks right), its own `XNAT_PORT` per trust kit, and its own trust numbers β€” all host-global and not covered by the prefix. Two traps when running one: `NET_ENDPOINTS` (which the hub seeds into `fl_nets` at boot) must carry the service name `fl-api-net-1`, never the pre-FLIP#957 `flip-fl-api-net-1`, which is now a DNS name in no stack at all; and `FL_PROVISIONED_DIR` may be given as either a relative or an absolute path (relative resolves against the repo root). -- `MAIN_ENV_FILE` β€” which repo-root env file the Makefiles load, as a bare **filename** (never a path). Defaults by `PROD` to `.env.production` / `.env.stag` / `.env.development`; override it to run a second stack from the *same checkout* β€” `make up MAIN_ENV_FILE=.env.b.development FLIP_INSTANCE=b` β€” instead of needing a second clone. The root Makefile `export`s it, and the flip-api, flip-ui and trust Makefiles each `include ../$(MAIN_ENV_FILE)`, which is why the value must stay a bare filename: each one prepends its own `../`. Every include is wildcard-guarded, so a value that resolves to nothing is **skipped silently** and the service builds with no environment at all rather than failing β€” check the `Using MAIN_ENV_FILE:` line each Makefile prints if a second stack comes up with empty config. `scripts/check_local_status.py` honours it too (via the environment), so a status check reports on the stack you are actually running. +- `MAIN_ENV_FILE` β€” which repo-root env file the Makefiles load, as a bare **filename** (never a path). Defaults by `PROD` to `.env.production` / `.env.stag` / `.env.lza-prod` / `.env.lza-stag` / `.env.development`; override it to run a second stack from the *same checkout* β€” `make up MAIN_ENV_FILE=.env.b.development FLIP_INSTANCE=b` β€” instead of needing a second clone. The root Makefile `export`s it, and the flip-api, flip-ui and trust Makefiles each `include ../$(MAIN_ENV_FILE)`, which is why the value must stay a bare filename: each one prepends its own `../`. Every include is wildcard-guarded, so a value that resolves to nothing is **skipped silently** and the service builds with no environment at all rather than failing β€” check the `Using MAIN_ENV_FILE:` line each Makefile prints if a second stack comes up with empty config. `scripts/check_local_status.py` honours it too (via the environment), so a status check reports on the stack you are actually running. - `DB_PORT` β€” in **dev** this is the *host* port flip-db is published on (the container side is pinned to 5432, which is what flip-api dials over the docker network); in `.env.stag` / `.env.production` it is the port the RDS Proxy listens on, since nothing is published there. Both default to 5432, so the difference only shows up when a second dev stack moves it. - `AES_KEY_BASE64` β€” encryption key for trust communication - A remote trust operator only needs their kit file (`trust/.env.`) β€” no hub `.env.` needed on trust hosts. diff --git a/Makefile b/Makefile index 77911c465..8fceed843 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,18 @@ else ifeq ($(PROD),stag) MAIN_ENV_FILE=.env.stag __DCKR_SUFFIX=production ENV=stag +# LZA estate (FLIP#749): the hub itself deploys via deploy/providers/AWS, but the +# kit-file targets here (new-trust, sync-trust-kit[s]) must name +# trust/.env..$(ENV) the way that Makefile's KIT_ENV_SUFFIX and +# register-trusts.sh read it back β€” .lza-prod / .lza-stag, never .stag. +else ifeq ($(PROD),lza) +MAIN_ENV_FILE=.env.lza-prod +__DCKR_SUFFIX=production +ENV=lza-prod +else ifeq ($(PROD),lza-stag) +MAIN_ENV_FILE=.env.lza-stag +__DCKR_SUFFIX=production +ENV=lza-stag else MAIN_ENV_FILE=.env.development __DCKR_SUFFIX=development diff --git a/scripts/new_trust.py b/scripts/new_trust.py index 74fac6843..8ea43d2e6 100644 --- a/scripts/new_trust.py +++ b/scripts/new_trust.py @@ -31,13 +31,23 @@ from pathlib import Path +# Kit-file env suffix β†’ the PROD flag that selects it (root Makefile ENV / deploy +# Makefile KIT_ENV_SUFFIX). Suffixes that spell their own flag are absent. +_PROD_FLAG_FOR_ENV = {"production": "true", "lza-prod": "lza"} + + def main() -> None: """CLI entry point: scaffold trust/.env.. from the base template.""" parser = argparse.ArgumentParser(description="Scaffold a new trust kit file.") parser.add_argument("--code", required=True, help="Trust short code β€” the kit-file handle (e.g. KCH).") parser.add_argument("--name", required=True, help="Trust display name registered on the hub.") parser.add_argument("--region", default=None, help="Optional NHS region.") - parser.add_argument("--env", required=True, choices=("development", "stag", "production"), help="Target env.") + parser.add_argument( + "--env", + required=True, + choices=("development", "stag", "production", "lza-prod", "lza-stag"), + help="Target env β€” the kit-file suffix (lza-prod / lza-stag = the LZA estate, FLIP#749).", + ) parser.add_argument("--template", default=None, help="Base template (default trust/.env.example).") parser.add_argument("--trust-dir", default=None, help="Override the trust/ directory (testing).") args = parser.parse_args() @@ -65,7 +75,7 @@ def main() -> None: target.write_text("\n".join(identity) + "\n" + template.read_text()) target.chmod(0o600) print(f"βœ… Scaffolded {target}") - print(f" Next: make register-trust KIT={args.code} PROD={'true' if args.env == 'production' else args.env}") + print(f" Next: make register-trust KIT={args.code} PROD={_PROD_FLAG_FOR_ENV.get(args.env, args.env)}") if __name__ == "__main__": diff --git a/scripts/sync_trust_kit.py b/scripts/sync_trust_kit.py index 7c47e0685..0be9765c2 100755 --- a/scripts/sync_trust_kit.py +++ b/scripts/sync_trust_kit.py @@ -25,9 +25,9 @@ The caller (the root Makefile) is responsible for sourcing the right env file: `include $(MAIN_ENV_FILE)` + `export` populates os.environ with -.env.development / .env.stag / .env.production values before invoking this -script. No docker compose exec, no ECS round-trip, no per-trust lookup β€” just a -fileβ†’file copy keyed on KIT, portable across dev/stag/prod. +.env.development / .env.stag / .env.production (or .env.lza-prod / .env.lza-stag) +values before invoking this script. No docker compose exec, no ECS round-trip, no +per-trust lookup β€” just a fileβ†’file copy keyed on KIT, portable across dev/stag/prod. """ from __future__ import annotations diff --git a/scripts/tests/test_new_trust.py b/scripts/tests/test_new_trust.py index b78e7ea8c..c6ab7548c 100644 --- a/scripts/tests/test_new_trust.py +++ b/scripts/tests/test_new_trust.py @@ -125,12 +125,27 @@ def test_region_optional() -> None: shutil.rmtree(root) +def test_lza_envs_name_the_kit_and_hint_the_prod_flag() -> None: + print("β–Ά LZA envs scaffold .env..lza-{prod,stag} and hint the PROD flag that reads them back") + for env, prod_flag in (("lza-stag", "lza-stag"), ("lza-prod", "lza")): + root = _temp_repo() + try: + res = _run(root, "--code", "LZA", "--name", "LZA Trust", "--env", env) + target = root / "trust" / f".env.LZA.{env}" + _assert(res.returncode == 0, f"--env {env}: exit 0", res.stderr) + _assert(target.exists(), f"--env {env}: kit scaffolded at .env..{env}") + _assert(f"PROD={prod_flag}" in res.stdout, f"--env {env}: next-step hint says PROD={prod_flag}", res.stdout) + finally: + shutil.rmtree(root) + + def main() -> None: if not NEW_TRUST.is_file(): sys.exit(f"❌ {NEW_TRUST} not found") test_scaffolds_kit_with_identity_and_defaults() test_refuses_to_overwrite() test_region_optional() + test_lza_envs_name_the_kit_and_hint_the_prod_flag() print("β€”") print(f"PASS={PASS} FAIL={FAIL}") sys.exit(0 if FAIL == 0 else 1) From e6cda8736c2932a56537d21edeed180770311f99 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Fri, 28 Aug 2026 15:24:16 +0100 Subject: [PATCH 37/41] docs(deploy/AWS): list the raw TF_VAR exports an LZA env file carries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .env.lza-prod block omitted the four Terraform inputs that have no Makefile mapping and ride as `export TF_VAR_…=` lines (ingress-VPC CIDRs, LogArchive ELB-logs bucket, edge domain + distribution ARN), so a staging env file could not be built from the doc alone. List them with placeholders, and say which are estate-wide on staging and which stay empty until a staging edge exists. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index be5c1e385..2c8ad88f3 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -677,7 +677,9 @@ independently: | `LZA_VPC_NAME` | optional (defaults to the prod template name) | **required** in the env file β€” there is no defaultable staging VPC name, so the Makefile refuses to run without it rather than letting the lookup fail opaquely | Everything below reads naturally for either value; where it says `PROD=lza`, staging substitutes `lza-stag` and -its own account-scoped values (state bucket, `flip-lza-stag-*` bucket namespace, pull-through registry URL). Each +its own account-scoped values (state bucket, `flip-lza-stag-*` bucket namespace, pull-through registry URL); the raw +`TF_VAR_` exports at the end of the env block below are estate-wide values shared by both (one ingress VPC, one +LogArchive bucket), except the two edge ones, which stay empty until a staging edge exists. Each LZA environment is its **own workload account** β€” never co-tenant two environments in one account: the stack's resource names (`flip-cluster`, `flip-api`, `flip-database-proxy`, the `/flip/networking/*` handoff params) are fixed per account by design. @@ -782,6 +784,19 @@ MANAGE_DNS=false ALB_SUBDOMAIN=app.flip.aicentre.co.uk NLB_SUBDOMAIN=fl.app.flip.aicentre.co.uk +# LZA-only Terraform inputs with no Makefile mapping of their own β€” exported +# straight from the env file (the Makefile includes it as make syntax, so an +# `export TF_VAR_…=` line reaches Terraform unchanged). Values come from the +# networking account. +# ingress-VPC subnet CIDRs (edge NLB + CloudFront relay path over the TGW) +export TF_VAR_networking_ingress_cidrs=[""] +# the LogArchive account's central ELB access-logs bucket (accelerator guardrail) +export TF_VAR_lza_elb_access_logs_bucket= +# the edge distribution β€” EMPTY on the first apply (the edge is built from this +# stack's outputs, see "Edge wiring is two-phase"), then filled in + re-applied +export TF_VAR_lza_web_edge_domain= +export TF_VAR_lza_web_edge_distribution_arn= + # Optional: only needed if the platform VPC template is renamed. # LZA_VPC_NAME=AWSAccelerator-eu-west-2-prod ``` From e1ee31eaaab3116ace7f33dd6a97702981356af6 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Fri, 28 Aug 2026 15:30:49 +0100 Subject: [PATCH 38/41] fix(deploy/AWS): satisfy the checkov lint on fl_ingress_lza.tf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit develop's checkov-lint job (#1057) now scans the LZA files for the first time: - CKV_TF_2: pin module.fl_server_internal_nlb to `~> 10.0` like its legacy twin β€” the resolved version is already 10.5.0, so no change on apply. - CKV2_AWS_34 x4: the /flip/networking/* handoff parameters are non-secret values read cross-account by the networking account's edge stack, where an AWS-managed CMK could not be decrypted β€” the same rationale develop applied to the legacy TGW handoff params. Acknowledged in-code, not by weakening the check list. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/fl_ingress_lza.tf | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deploy/providers/AWS/fl_ingress_lza.tf b/deploy/providers/AWS/fl_ingress_lza.tf index 44d83e635..b7f57286f 100644 --- a/deploy/providers/AWS/fl_ingress_lza.tf +++ b/deploy/providers/AWS/fl_ingress_lza.tf @@ -111,7 +111,8 @@ resource "aws_ec2_tag" "fl_internal_nlb_security_group_flip_sg" { } module "fl_server_internal_nlb" { - source = "terraform-aws-modules/alb/aws" + source = "terraform-aws-modules/alb/aws" + version = "~> 10.0" # LZA counterpart of the legacy module.fl_server_nlb above β€” created on # exactly the opposite gate, so precisely one FL NLB exists per environment. create = var.lza_managed_network @@ -196,6 +197,7 @@ resource "aws_security_group_rule" "alb_ingress_web_from_networking" { # configuration (aicentre-lza-iac), replacing the e2e harness's # /flip-e2e/networking/ prefix with the real one. resource "aws_ssm_parameter" "lza_fl_nlb_private_ips" { + # checkov:skip=CKV2_AWS_34:non-secret networking value read CROSS-ACCOUNT by the networking account's edge stack (aicentre-lza-iac) β€” an AWS-managed CMK cannot be decrypted from another account count = var.lza_managed_network ? 1 : 0 name = "/flip/networking/fl_nlb_private_ips" description = "Internal FL NLB static private IPs (comma-separated, assigned via subnet_mapping so they are stable by construction; TGW-reachable AZs only) - registered as IP targets on the networking account's edge NLB FL listener" @@ -204,6 +206,7 @@ resource "aws_ssm_parameter" "lza_fl_nlb_private_ips" { } resource "aws_ssm_parameter" "lza_fl_port" { + # checkov:skip=CKV2_AWS_34:non-secret networking value read CROSS-ACCOUNT by the networking account's edge stack (aicentre-lza-iac) β€” an AWS-managed CMK cannot be decrypted from another account count = var.lza_managed_network ? 1 : 0 name = "/flip/networking/fl_port" description = "Workload-side FL ingress port (internal NLB listener) - consumed by the networking account's edge configuration" @@ -212,6 +215,7 @@ resource "aws_ssm_parameter" "lza_fl_port" { } resource "aws_ssm_parameter" "lza_alb_dns_name" { + # checkov:skip=CKV2_AWS_34:non-secret networking value read CROSS-ACCOUNT by the networking account's edge stack (aicentre-lza-iac) β€” an AWS-managed CMK cannot be decrypted from another account count = var.lza_managed_network ? 1 : 0 name = "/flip/networking/alb_dns_name" description = "Internal web ALB DNS name - the networking account's target-sync Lambda resolves this on a cadence to keep the web relay NLB targets current (ALB IPs rotate)" @@ -220,6 +224,7 @@ resource "aws_ssm_parameter" "lza_alb_dns_name" { } resource "aws_ssm_parameter" "lza_web_port" { + # checkov:skip=CKV2_AWS_34:non-secret networking value read CROSS-ACCOUNT by the networking account's edge stack (aicentre-lza-iac) β€” an AWS-managed CMK cannot be decrypted from another account count = var.lza_managed_network ? 1 : 0 name = "/flip/networking/web_port" description = "Workload-side web ingress port (the ALB's main listener; plain HTTP on the zone-less bring-up) - the relay NLB's target port on the networking side" From 09c37827582a8d9ad4fc9c40d748ac721e55bfaa Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Tue, 1 Sep 2026 11:32:50 +0100 Subject: [PATCH 39/41] docs: how a real domain attaches to the LZA edge (alias + us-east-1 ACM) A short shape-of-the-thing note in the MANAGE_DNS section: CloudFront is addressed by alias record, never by IP; the zone and the us-east-1 ACM cert live in the networking account with the edge; the parent DNS host delegates the subdomain. Placeholder names only - the concrete cutover runbook stays in the private platform repos. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index 9e6cd8ab6..994f9a12c 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -821,6 +821,16 @@ revert by flipping `MANAGE_DNS=true` + `make plan`/`apply` once the zone lands: - Trusts polling the hub would need the CloudFront domain as `CENTRAL_HUB_API_URL` β€” fine for WP3 smoke trusts; the real cutover is DNS-only and happens after the zone migrates. +**Attaching the real domain to the edge.** CloudFront is never addressed by IP β€” distribution IPs are anycast +and rotate β€” so the domain is wired up DNS-side, in the networking account where the edge lives: a public +Route 53 hosted zone for the FLIP subdomain (e.g. `flip.example.org`) is created there; a +DNS-validated ACM certificate for the canonical names is minted **in us-east-1** (a CloudFront requirement) +and attached to the distribution as its alternate domain names; the zone then carries an **A/AAAA alias +record** β€” e.g. `app.flip.example.org` β†’ `d111111abcdef8.cloudfront.net` (an alias, never a literal +IP) β€” and the parent domain's DNS host delegates the subdomain to the zone's four `ns-*.awsdns-*` name +servers. The FL name (`fl.app.flip.example.org`) is the same shape aliasing the edge NLB. The +concrete cutover runbook stays in the private platform repos. + **State of the LZA path** (tracked on #749): - **Multi-AZ has landed platform-side** (verified against the live account 2026-08-24): both AZs carry an From 6490a5829dcdbdb5559addc2fe8a4124cdcc8124 Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Tue, 1 Sep 2026 13:22:49 +0100 Subject: [PATCH 40/41] docs: the fresh-account Secrets Manager/KMS-alias bootstrap trap On a brand-new LZA workload account the first 'make plan' fails on data.aws_kms_alias.secretsmanager with 'empty result' - alias/aws/secretsmanager is AWS-managed and created lazily on first use of Secrets Manager, so an account that has never held a secret has no alias to read. Creating the ECR pull-through credential (needed anyway) mints it. Found on the FLIPStaging bring-up. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index 994f9a12c..c51070a9d 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -821,6 +821,23 @@ revert by flipping `MANAGE_DNS=true` + `make plan`/`apply` once the zone lands: - Trusts polling the hub would need the CloudFront domain as `CENTRAL_HUB_API_URL` β€” fine for WP3 smoke trusts; the real cutover is DNS-only and happens after the zone migrates. +**Fresh-account trap: create a Secrets Manager secret before the first `plan`.** On a +brand-new workload account `make plan` fails at the very end with + +``` +Error: reading KMS Alias (alias/aws/secretsmanager): empty result + with data.aws_kms_alias.secretsmanager, on rds_proxy.tf line 37 +``` + +even though the plan itself computed cleanly. `alias/aws/secretsmanager` is an +**AWS-managed** key: AWS creates it lazily, the first time the account uses Secrets +Manager, so in an account that has never held a secret the alias genuinely does not +exist yet and the data source has nothing to read. Nothing is wrong with the +configuration β€” the account is simply too new. Creating *any* secret mints the key, +and the ECR pull-through credential below is the one every LZA account needs anyway, +so do that step first and the plan completes. (Verified on the FLIPStaging bring-up, +2026-09-01.) + **Attaching the real domain to the edge.** CloudFront is never addressed by IP β€” distribution IPs are anycast and rotate β€” so the domain is wired up DNS-side, in the networking account where the edge lives: a public Route 53 hosted zone for the FLIP subdomain (e.g. `flip.example.org`) is created there; a From 80b601ccc6da336caf68abeb3db59b916872e3cb Mon Sep 17 00:00:00 2001 From: at24_bioeng625-pc Date: Tue, 1 Sep 2026 13:35:56 +0100 Subject: [PATCH 41/41] docs: make the LZA out-of-band prerequisites reusable per account The prerequisites list was written for FLIPProduction specifically and named the rules without saying how to create them or that their order matters. Generalised to any PROD=lza* account and given the concrete commands from the FLIPStaging bring-up: secret before the rule that references it, both before the first plan (the KMS-alias trap), the PAT handled via a 0600 temp file so it is never echoed, plus the empty-cache and mutable-tag-staleness caveats. Signed-off-by: at24_bioeng625-pc --- deploy/providers/AWS/README.md | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/deploy/providers/AWS/README.md b/deploy/providers/AWS/README.md index c51070a9d..61e1c0712 100644 --- a/deploy/providers/AWS/README.md +++ b/deploy/providers/AWS/README.md @@ -733,16 +733,39 @@ bucket yet) and `local.ui_origin` is a placeholder. Once the edge stack is up, s bucket CORS + Cognito URLs at the edge domain. This ordering is why the two variables deliberately carry no "required-when-LZA" validation β€” it would hard-fail the legitimate first apply. -**Prerequisites (already provisioned out-of-band in the account, not Terraform-managed here):** +**Prerequisites (provisioned out-of-band in each LZA account, not Terraform-managed here).** Every +`PROD=lza*` account needs these three before its first `plan`; the commands below are the ones the +FLIPStaging bring-up used (2026-09-01), with `PROD`/profile swapped per environment. -- TF state bucket `flip-terraform-state-lza` (versioned, SSE-KMS, public access blocked). `make create-backend - PROD=lza` is idempotent against it. +- TF state bucket (`flip-terraform-state-lza`, or `-lza-stag`; versioned, SSE-KMS, public access blocked): + `make create-backend PROD=lza` β€” idempotent, reads the bucket name from the env file. - ECR **pull-through cache rules** β€” the account has no internet egress, so images come from in-account mirrors over the central `ecr.api`/`ecr.dkr` endpoints: prefix `ghcr/` mirroring `ghcr.io` (upstream auth via a read-only GHCR PAT in the `ecr-pullthroughcache/ghcr` Secrets Manager secret) and the credential-less `ecr-public/` prefix mirroring `public.ecr.aws` (used for the EFS-provision utility image). The execution role's `ecr:BatchImportUpstreamImage`/`ecr:CreateRepository` grant for first-pull imports IS Terraform-managed - (`iam_ecs.tf`, LZA-gated). + (`iam_ecs.tf`, LZA-gated). **Create the secret before the rule that references it, and both before the first + `plan`** (see the KMS-alias trap above): + + ```bash + # The GHCR PAT. Copy the existing read-only one from another LZA account (shown), + # or mint a fresh token per account for tighter isolation. The value must never + # be echoed β€” write it via a 0600 temp file. + TMP=$(mktemp) && chmod 600 "$TMP" + aws secretsmanager get-secret-value --profile \ + --secret-id ecr-pullthroughcache/ghcr --query SecretString --output text > "$TMP" + aws secretsmanager create-secret --profile \ + --name ecr-pullthroughcache/ghcr --secret-string "file://$TMP" --query ARN --output text + shred -u "$TMP" + + aws ecr create-pull-through-cache-rule --profile \ + --ecr-repository-prefix ghcr --upstream-registry-url ghcr.io --credential-arn + aws ecr create-pull-through-cache-rule --profile \ + --ecr-repository-prefix ecr-public --upstream-registry-url public.ecr.aws + ``` + + Nothing is pre-populated: the first pull of each image pays the upstream fetch. Cached **mutable** tags can + serve up to ~24h stale, which is why deploys pin the immutable `sha-` tags. - An `lza-prod` profile in `~/.aws/config` for the account's Identity Center `FLIPAdminAccess` permission set (an `aws configure sso` against the account, then rename the generated profile β€” same short-alias convention as `prod`/`stag`; override the expected name via `LZA_AWS_PROFILE`).