From 14d5dec91d7057fad3c32f78a3036faf6e3b2e99 Mon Sep 17 00:00:00 2001 From: Sergio Ruiz Date: Wed, 5 Aug 2026 15:56:44 -0500 Subject: [PATCH 1/4] OSAC-3616: Netris MaaS/CaaS infra feats & fixes MaaS suite (SUITE / setup-maas / disk grow / g5 labeling / resource map / cluster params), CaaS reliability (agent wait, HCP PSS, AAP pin sync, disk-setup, VPN host-key, destroy/force-destroy), and docs. Signed-off-by: Sergio Ruiz Assisted-by: Cursor Co-authored-by: Cursor --- infra/netris/CLAUDE.md | 27 +- infra/netris/Makefile | 76 ++++- infra/netris/README.md | 37 +- infra/netris/inventory/group_vars/all.yml | 17 + infra/netris/netris-lab/group_vars/all.yml | 2 +- .../roles/connectivity/tasks/vpn.yml | 7 + infra/netris/playbooks/destroy-full.yml | 4 +- infra/netris/playbooks/force-destroy-caas.yml | 10 + infra/netris/roles/create-caas/tasks/main.yml | 119 ++++++- .../netris/roles/destroy-caas/tasks/main.yml | 96 +++++- .../netris/roles/discover-caas/tasks/main.yml | 8 +- infra/netris/roles/disk-setup/tasks/main.yml | 73 +++- .../files/netris_orphan_cleanup.py | 173 ++++++++++ .../roles/force-destroy-caas/tasks/main.yml | 321 ++++++++++++++++++ .../roles/patch-osac-refresh/tasks/main.yml | 11 +- .../tasks/sync-aap-project-pin.yml | 287 ++++++++++++++++ infra/netris/roles/restore-ocp/tasks/main.yml | 22 ++ infra/netris/roles/setup-caas/tasks/main.yml | 160 ++++++++- infra/netris/roles/setup-infra/tasks/main.yml | 37 ++ infra/netris/scripts/deploy-jump.sh | 14 +- infra/netris/scripts/redeploy-server.sh | 106 +++++- 21 files changed, 1513 insertions(+), 94 deletions(-) create mode 100644 infra/netris/playbooks/force-destroy-caas.yml create mode 100755 infra/netris/roles/force-destroy-caas/files/netris_orphan_cleanup.py create mode 100644 infra/netris/roles/force-destroy-caas/tasks/main.yml create mode 100644 infra/netris/roles/patch-osac-refresh/tasks/sync-aap-project-pin.yml diff --git a/infra/netris/CLAUDE.md b/infra/netris/CLAUDE.md index 1c82df084..de794474e 100644 --- a/infra/netris/CLAUDE.md +++ b/infra/netris/CLAUDE.md @@ -1,6 +1,6 @@ # netris-test-infra -OSAC test infrastructure that deploys and tests OpenShift Assisted Cluster on a simulated Netris Spectrum-X GPU cluster. Tests three service models: VMaaS, BMaaS (planned), and CaaS. +OSAC test infrastructure that deploys and tests OpenShift Assisted Cluster on a simulated Netris Spectrum-X GPU cluster. Tests three service models: VMaaS, BMaaS (planned), CaaS, and MaaS. Uses [netris-lab](netris-lab/) as a git submodule for the underlying network infrastructure. @@ -16,12 +16,13 @@ roles/ # Ansible roles (each has tasks/main.yml) cache-snapshot/ # Pull + cache flavor OCI image (skopeo) prep-osac/ # Clone mono-repo, patch Helm values (fresh install) prep-refresh-osac/ # Clone mono-repo, patch values, rebuild CLI (snapshot refresh) - patch-osac-refresh/ # Post-refresh: patch Netris config + SSH keys into running cluster + patch-osac-refresh/ # Post-refresh: Netris/SSH + AAP project pin sync + CaC discover-caas/ # Boot discovery VMs with InfraEnv ISO setup-caas/ # Label agents, register host type - create-caas/ # Create CaaS cluster via fulfillment API + create-caas/ # Create CaaS/MaaS cluster via fulfillment API destroy-infra/ # Teardown netris-lab - destroy-caas/ # Teardown CaaS resources + destroy-caas/ # Teardown CaaS/MaaS (osac delete + agents/InfraEnv/VMs) + force-destroy-caas/ # Force-clean stuck orders/namespaces + Netris orphans playbooks/ # Ansible playbooks (one per workflow phase) inventory/ local.yml # Inventory (localhost, local connection) @@ -34,7 +35,7 @@ vendor/ # Vendored Ansible collections ``` make deploy # Full pipeline: deploy-infra + deploy-ocp + deploy-osac (fresh Helm install) -make deploy-fast # Snapshot pipeline: deploy-infra + deploy-ocp + deploy-osac (snapshot refresh) +make deploy-fast # Snapshot pipeline: same steps with OSAC_DEPLOY_MODE=snapshot make setup-infra # Install prerequisites, cache images + snapshot flavor make deploy-infra # Deploy netris-lab make deploy-ocp # Configure Netris networking + restore OCP SNO from snapshot @@ -42,15 +43,22 @@ make deploy-osac # Deploy OSAC (fresh Helm install or snapshot refres make connectivity # Re-run lab connectivity (VPN, BGP, softgate agents) make setup-caas # CaaS setup: discover hosts, label agents, register host type make deploy-caas # CaaS: create cluster +make setup-maas # MaaS setup (wrappers over setup-caas with MaaS overrides) +make deploy-maas # MaaS: create cluster (ocp_4_20_ai_maas + -p params) make destroy-full # Teardown all infrastructure make destroy-osac # Teardown OSAC only make destroy-ocp # Reset OCP for reinstall make destroy-infra # Teardown netris-lab -make destroy-caas # Teardown CaaS resources +make destroy-caas # Teardown CaaS/MaaS cluster + discovery +make force-destroy-caas # destroy-caas + strip stuck leftovers / Netris orphans +make destroy-maas # destroy-caas with MaaS overrides +make force-destroy-maas # force-destroy-caas with MaaS overrides +make redeploy-fresh # destroy-full + full BM pipeline (SUITE / OSAC_DEPLOY_MODE) make vendor-update # Refresh vendored Ansible collections make gather-infra # Gather diagnostic info from the cluster # Override variables: make EXTRA_VARS="key=value" # Image overrides: make deploy-osac EXTRA_VARS="fulfillment_service_image=quay.io/..." +# Suite / mode: make redeploy-fresh SUITE=maas OSAC_DEPLOY_MODE=snapshot ``` ## Workflow Order @@ -61,6 +69,11 @@ make gather-infra # Gather diagnostic info from the cluster **CaaS:** deploy or deploy-fast → setup-caas → deploy-caas +**MaaS:** deploy or deploy-fast → setup-maas → deploy-maas +(or `make redeploy-fresh SUITE=maas OSAC_DEPLOY_MODE=snapshot`) + +**VMaaS / BMaaS:** not yet implemented + ## Ansible Configuration - Inventory: `inventory/local.yml` @@ -79,7 +92,7 @@ All variables in `inventory/group_vars/all.yml`. Key sections: - **OSAC**: `osac_repo/branch`, `osac_namespace`, `osac_values_file` - **Component images**: `osac_operator_image`, `fulfillment_service_image` (empty = defaults) - **Snapshot**: `snapshot_flavor_image`, `snapshot_flavor_dir`, `snapshot_recert_image`, `snapshot_osac_namespace`, `snapshot_osac_values_file` -- **CaaS**: `caas_discovery_vm_patterns`, `caas_host_type_id`, `caas_cluster_name`, `caas_agents` +- **CaaS / MaaS**: `caas_discovery_vm_patterns`, `caas_host_type_id`, `caas_cluster_name`, `caas_agents`, `caas_resource_class_hostnames` (empty = all agents); MaaS overrides via Makefile `MAAS_*` ## External Dependencies diff --git a/infra/netris/Makefile b/infra/netris/Makefile index 083eab3a4..d6f1a82cb 100644 --- a/infra/netris/Makefile +++ b/infra/netris/Makefile @@ -5,6 +5,11 @@ EXTRA_VARS ?= ANSIBLE_EXTRA = $(if $(EXTRA_VARS),-e '$(EXTRA_VARS)') -e osac_deploy_mode=$(OSAC_DEPLOY_MODE) -e osac_values_file=$(OSAC_VALUES_FILE) ENV_INFRA := .env.infra +# Suite selects CaaS vs MaaS for restore disk grow + redeploy-server.sh flow. +# SUITE=caas (default) — setup-caas / deploy-caas. +# SUITE=maas — grow h00 to ocp_snapshot_disk_gb; setup-maas / deploy-maas. +SUITE ?= caas + OSAC_NAMESPACE ?= osac-e2e-ci OSAC_VALUES_FILE ?= values/caas-ci/values.yaml OSAC_PULL_SECRET_PATH ?= /root/pull-secret @@ -39,7 +44,7 @@ deploy-infra: ansible-playbook playbooks/deploy-infra.yml $(ANSIBLE_EXTRA) deploy-ocp: - ansible-playbook playbooks/deploy-ocp.yml $(ANSIBLE_EXTRA) + ansible-playbook playbooks/deploy-ocp.yml -e suite=$(SUITE) $(ANSIBLE_EXTRA) deploy-osac: @if [ "$(OSAC_DEPLOY_MODE)" = "snapshot" ]; then \ @@ -81,6 +86,9 @@ destroy-setup: destroy-caas: ansible-playbook playbooks/destroy-caas.yml $(ANSIBLE_EXTRA) +force-destroy-caas: + ansible-playbook playbooks/force-destroy-caas.yml $(ANSIBLE_EXTRA) + gather-infra: ansible-playbook playbooks/gather-infra.yml $(ANSIBLE_EXTRA) @@ -102,6 +110,53 @@ destroy-bmaas: gather-bmaas: ansible-playbook playbooks/gather-bmaas.yml $(ANSIBLE_EXTRA) +# --------------------------------------------------------------------------- +# MaaS — thin wrappers over CaaS playbooks with MaaS-specific variable overrides +# Override any default with MAAS_CLUSTER_NAME=..., MAAS_HOST_TYPE_ID=..., etc. +# --------------------------------------------------------------------------- +MAAS_CLUSTER_NAME ?= maas-cluster +MAAS_HOST_TYPE_ID ?= g5 +MAAS_CLUSTER_TEMPLATE ?= osac.templates.ocp_4_20_ai_maas +MAAS_CATALOG_ITEM ?= maas-ocp +# Discovery VM sizing for setup-maas only (CaaS keeps inventory defaults: 4 vCPU / 16GiB). +# Default MaaS: 10 vCPU / 20GiB for h02–h03; h01 stays at 4 (unused for MaaS workers). +MAAS_DISCOVERY_VCPU ?= 10 +MAAS_DISCOVERY_MEMORY_MB ?= 20480 +MAAS_DISCOVERY_VCPU_OVERRIDES ?= {"hgx-pod00-su0-h01":4} +# Only these inventory hostnames get resource_class=g5 (and approval). Empty = all agents. +# Comma-separated. Keeps h01 out of the MaaS claim pool; setup-caas still labels all as ci-worker. +MAAS_LABEL_HOSTNAMES ?= hgx-pod00-su0-h02,hgx-pod00-su0-h03 +# Template -p flags for osac create (deploy-maas only). Comma-separated key=value. +MAAS_CLUSTER_PARAMS ?= enable_maas=true,enable_keycloak=true,enable_observability=true,enable_dashboard=true +MAAS_PARAMS_JSON = $(shell python3 -c 'import json; print(json.dumps([p.strip() for p in """$(MAAS_CLUSTER_PARAMS)""".split(",") if p.strip()]))') +MAAS_LABEL_HOSTNAMES_JSON = $(shell python3 -c 'import json; print(json.dumps([h.strip() for h in """$(MAAS_LABEL_HOSTNAMES)""".split(",") if h.strip()]))') + +MAAS_EXTRA = \ + -e 'caas_cluster_name=$(MAAS_CLUSTER_NAME)' \ + -e 'caas_host_type_id=$(MAAS_HOST_TYPE_ID)' \ + -e 'caas_host_type_title=MaaS Node' \ + -e 'caas_host_type_description=Bare-metal nodes for MaaS testing' \ + -e 'caas_cluster_template=$(MAAS_CLUSTER_TEMPLATE)' \ + -e 'caas_catalog_item=$(MAAS_CATALOG_ITEM)' \ + -e 'caas_catalog_item_title=MaaS OCP' \ + -e 'caas_catalog_item_description=OCP cluster for MaaS testing' \ + -e 'caas_discovery_vcpu=$(MAAS_DISCOVERY_VCPU)' \ + -e 'caas_discovery_memory_mb=$(MAAS_DISCOVERY_MEMORY_MB)' \ + -e '{"caas_discovery_vcpu_overrides":$(MAAS_DISCOVERY_VCPU_OVERRIDES)}' \ + -e '{"caas_resource_class_hostnames":$(MAAS_LABEL_HOSTNAMES_JSON)}' + +setup-maas: + ansible-playbook playbooks/setup-caas.yml $(MAAS_EXTRA) $(ANSIBLE_EXTRA) + +deploy-maas: + ansible-playbook playbooks/deploy-caas.yml $(MAAS_EXTRA) -e '{"caas_cluster_params":$(MAAS_PARAMS_JSON)}' $(ANSIBLE_EXTRA) + +destroy-maas: + ansible-playbook playbooks/destroy-caas.yml $(MAAS_EXTRA) $(ANSIBLE_EXTRA) + +force-destroy-maas: + ansible-playbook playbooks/force-destroy-caas.yml $(MAAS_EXTRA) $(ANSIBLE_EXTRA) + # ─── Internal sub-targets (deploy-osac implementation) ───────────── # Fresh install: clone mono-repo, patch Helm values, run make install @@ -120,6 +175,7 @@ post-osac: ansible-playbook playbooks/post-osac.yml $(ANSIBLE_EXTRA) # Snapshot refresh: clone mono-repo, patch values, run refresh-after-snapshot.py +# post-refresh-osac → patch-osac-refresh (incl. AAP project pin sync + CaC) refresh-osac: prep-refresh-osac run-refresh-osac post-refresh-osac prep-refresh-osac: @@ -157,15 +213,23 @@ cleanup-dns: destroy-full: ansible-playbook playbooks/destroy-full.yml $(ANSIBLE_EXTRA) +# OSAC_DEPLOY_MODE=fresh (default) or snapshot — same meaning as deploy / deploy-fast. +# SUITE=caas (default) or maas — selects setup/deploy flow in redeploy-server.sh. redeploy-fresh: @echo "=== Destroying all infrastructure ===" -$(MAKE) destroy-full @sync - @echo "=== Starting fresh deploy ===" - scripts/redeploy-server.sh --fresh "$(EXTRA_VARS)" 2>&1 | tee -a /root/deploy.log + @echo "=== Starting deploy (SUITE=$(SUITE) OSAC_DEPLOY_MODE=$(OSAC_DEPLOY_MODE)) ===" + SUITE=$(SUITE) OSAC_DEPLOY_MODE=$(OSAC_DEPLOY_MODE) \ + scripts/redeploy-server.sh --fresh \ + $(if $(filter snapshot,$(OSAC_DEPLOY_MODE)),--snapshot) \ + "$(EXTRA_VARS)" 2>&1 | tee -a /root/deploy.log redeploy-continue: - scripts/redeploy-server.sh "$(EXTRA_VARS)" 2>&1 | tee -a /root/deploy.log + SUITE=$(SUITE) OSAC_DEPLOY_MODE=$(OSAC_DEPLOY_MODE) \ + scripts/redeploy-server.sh \ + $(if $(filter snapshot,$(OSAC_DEPLOY_MODE)),--snapshot) \ + "$(EXTRA_VARS)" 2>&1 | tee -a /root/deploy.log deploy-jump: @scripts/deploy-jump.sh @@ -187,7 +251,9 @@ health-check: .PHONY: deploy deploy-fast \ deploy-setup setup-infra deploy-infra deploy-ocp deploy-osac \ setup-caas deploy-caas \ - destroy-osac destroy-ocp destroy-infra destroy-setup destroy-caas destroy-full \ + destroy-osac destroy-ocp destroy-infra destroy-setup destroy-full \ + destroy-caas force-destroy-caas \ + setup-maas deploy-maas destroy-maas force-destroy-maas \ gather-infra gather-mgmt-cluster gather-caas \ setup-bmc setup-bmaas destroy-bmaas gather-bmaas \ install-osac prep-osac run-osac-setup post-osac \ diff --git a/infra/netris/README.md b/infra/netris/README.md index 30ddc133f..e4b299771 100644 --- a/infra/netris/README.md +++ b/infra/netris/README.md @@ -34,7 +34,7 @@ Internet access for OCP image pulls flows through: hgx-00 → NS VNet → softga ## Prerequisites - **Bare-metal host** running RHEL 9.x or Rocky Linux 9.x with KVM support -- **System packages** — `dnf install -y git make python3-pip ansible-core && pip3 install ansible` (all other tools are installed automatically by `make setup-infra`) +- **System packages** — `dnf install -y git make python3-pip ansible-core && pip3 install ansible`. `make setup-infra` then installs the rest of the BM bootstrap (`sshpass`, `tmux`, `policycoreutils-python-utils`), EPEL from the Fedora URL (required on RHEL), and the Python `kubernetes` client. - **Resources**: ~32+ CPU cores, 128+ GB RAM (lab VMs + OCP SNO VM) - **Storage**: ~400GB+ for OCP disk images, K3s, and containers. If the root partition is smaller, use a secondary data disk and enable automatic provisioning with `disk_setup_enabled=true` (via `EXTRA_VARS` or in the config). Run `make disk-setup` standalone or pass the variable during setup: `make setup-infra EXTRA_VARS="disk_setup_enabled=true"` - **Netris license key** — place at repo root as `license.key` @@ -101,6 +101,8 @@ After deployment, the kubeconfig is at `/root/.kube/config`. |--------|-------------|------| | `make setup-caas` | Discover hosts, label agents, register host type, configure osac CLI | ~30 min | | `make deploy-caas` | Create CaaS cluster using `ocp_ci_small` template | ~60 min | +| `make setup-maas` | Same as setup-caas with MaaS host type / sizing; labels only `MAAS_LABEL_HOSTNAMES` (default h02+h03) with `g5` | ~30 min | +| `make deploy-maas` | Create MaaS cluster (`ocp_4_20_ai_maas` + `-p` template params) | ~60 min | ### Destroy @@ -111,6 +113,10 @@ After deployment, the kubeconfig is at `/root/.kube/config`. | `make destroy-ocp` | Reset OCP for reinstall: delete cluster, recreate disk, boot VM | | `make destroy-infra` | Tear down netris-lab (VMs, K3s, topology) | | `make destroy-caas` | CaaS teardown: stop discovery VMs, remove disks/ISO, delete namespace, clean DNS | +| `make force-destroy-caas` | destroy-caas + strip stuck orders/namespaces, wipe VMs, Netris orphans | +| `make destroy-maas` | destroy-caas with MaaS cluster/host-type overrides | +| `make force-destroy-maas` | force-destroy-caas with MaaS overrides | +| `make destroy-bmaas` | BMaaS teardown: remove BMH/K8s resources, stop BMaaS VMs, clean disks, delete BMH namespace | ### Recovery and Utilities @@ -174,12 +180,30 @@ make setup-caas # discover hosts, label agents, register host type make deploy-caas # create cluster ``` +**Deploy MaaS after OSAC is up:** +```bash +make setup-maas # same discovery path; labels only MAAS_LABEL_HOSTNAMES (default h02+h03) with g5 +make deploy-maas # create cluster with ocp_4_20_ai_maas + enable_* params +``` + **Rebuild from scratch:** ```bash -make destroy # tear down everything +make destroy-full # tear down everything make deploy-fast # full redeploy (snapshot path) ``` +**BM full pipeline (redeploy-server.sh):** +```bash +# CaaS + fresh OSAC install (default) +make redeploy-fresh + +# CaaS + snapshot OSAC refresh (same mode as deploy-fast) +make redeploy-fresh OSAC_DEPLOY_MODE=snapshot + +# MaaS + snapshot refresh (grows h00 disk; setup-maas / deploy-maas) +make redeploy-fresh SUITE=maas OSAC_DEPLOY_MODE=snapshot +``` + ### Bare-Metal Lab Deployment (from laptop) For persistent bare-metal servers behind NAT (e.g., Red Hat lab infrastructure), use the remote deploy workflow. This handles image caching, bootstrapping, and resilient multi-step deploys from your laptop. @@ -222,12 +246,15 @@ source scripts/env-mylab.sh && make deploy-jump | Target | Description | |--------|-------------| -| `make redeploy-fresh` | Destroy + wipe progress + full fresh deploy | +| `make redeploy-fresh` | Destroy + wipe progress + full pipeline (`SUITE`, `OSAC_DEPLOY_MODE`) | +| `make redeploy-continue` | Resume failed redeploy from last incomplete step | | `make disk-setup` | Auto-detect and mount data disk | | `make post-install` | Fix Keycloak/UI + generate access doc | | `make access-doc` | Generate handover documentation only | | `make health-check` | Quick status verification | +`redeploy-server.sh` steps: `setup-infra` → `deploy-infra` → `deploy-ocp` → `deploy-osac` → `setup-caas|setup-maas` → `deploy-caas|deploy-maas` → `post-install`. Pass `OSAC_DEPLOY_MODE=snapshot` (or `--snapshot`) for refresh instead of Helm install; default is `fresh`. + See the PR description for known issues and workarounds specific to BM/RHEL environments. ## Accessing OCP Routes @@ -258,10 +285,12 @@ The flow runs three Ansible roles in sequence: 1. **`configure-netris`** — creates VPC, VNet (DHCP disabled), subnet, SNAT/DNAT rules via Netris API 2. **`configure-dns`** — creates Route 53 DNS records and local dnsmasq config for the cluster domain -3. **`restore-ocp`** — creates copy-on-write disk overlays backed by the cached flavor, mounts the OS disk via qemu-nbd to write pre-boot config (hostname, nodeip hint, dnsmasq overrides, nmstate config for static IP, OVN/OVS cleanup), runs recert (via JSON config file, matching LCA's approach) to regenerate all TLS certificates and cluster identity, then waits for cluster health +3. **`restore-ocp`** — creates copy-on-write disk overlays backed by the cached flavor, mounts the OS disk via qemu-nbd to write pre-boot config (hostname, nodeip hint, dnsmasq overrides, nmstate config for static IP, OVN/OVS cleanup), runs recert (via JSON config file, matching LCA's approach) to regenerate all TLS certificates and cluster identity, then waits for cluster health. When `suite=maas` (via `SUITE=maas`), also grows the h00 OS disk to `ocp_snapshot_disk_gb` before pre-boot config. The snapshot flavor is pulled and cached during `make setup-infra` (one-time ~60GB download). Subsequent deploys use copy-on-write overlays, so only changed blocks are written. +OSAC itself is handled by `make deploy-osac` (not part of `deploy-ocp`). In `OSAC_DEPLOY_MODE=snapshot`, that runs `refresh-osac` → `post-refresh-osac` → **`patch-osac-refresh`**, which wires Netris/SSH/socat and then applies `config-as-code-ig`’s `AAP_PROJECT_GIT_*` pin onto the live AAP Project and launches `osac-config-as-code` once (snapshot AAP DB keeps the bake-time `scm_branch`; waiting for the hourly CaC schedule is too late for `setup-maas` / `publish-templates`). + ## How deploy-osac Works `make deploy-osac` runs in three phases: diff --git a/infra/netris/inventory/group_vars/all.yml b/infra/netris/inventory/group_vars/all.yml index 860e07105..a956b1217 100644 --- a/infra/netris/inventory/group_vars/all.yml +++ b/infra/netris/inventory/group_vars/all.yml @@ -76,6 +76,8 @@ snapshot_flavor_dir: >- snapshot_recert_image: "quay.io/osac-project/recert:latest" snapshot_osac_namespace: "osac-e2e-ci" snapshot_osac_values_file: "values/caas-ci/values.yaml" +# Grow CoW OS overlay during restore-snapshot when SUITE=maas (flavor is ~100Gi). +ocp_snapshot_disk_gb: 150 # AWS configuration for Route 53 DNS (read from config file or environment) aws_access_key_id: "{{ lookup('ini', 'aws_access_key_id section=default file=' + config_path, errors='ignore') | default(lookup('env', 'AWS_ACCESS_KEY_ID'), true) }}" @@ -92,6 +94,8 @@ caas_discovery_vm_patterns: - "hgx-pod00-su0-h02" - "hgx-pod00-su0-h03" caas_discovery_vcpu: 4 +# Optional per-VM vCPU overrides (VM name → count). Empty = use caas_discovery_vcpu for all. +caas_discovery_vcpu_overrides: {} caas_discovery_memory_mb: 16384 caas_discovery_disk_gb: 100 caas_discovery_infraenv_name: "infraenv" @@ -104,11 +108,24 @@ caas_server_cluster_template_gateway: "192.168.101.1/24" caas_host_type_id: "ci-worker" caas_host_type_title: "CI Worker" caas_host_type_description: "Worker nodes for CI testing" +# Netris per-resource-class entry merged into cluster-fulfillment-ig +# NETRIS_RESOURCE_CLASS_MAP when the host type key is missing (setup-caas / setup-maas). +caas_netris_server_cluster_template_id: 2 +caas_netris_mgmt_interface: "ens4" +caas_netris_vpc_interfaces: + - "ens5" caas_cluster_template: "osac.templates.ocp_ci_small" caas_catalog_item: "ci-ocp-small" caas_catalog_item_title: "CI OCP Small" caas_catalog_item_description: "Small OCP cluster for CI testing" caas_cluster_name: "caas-ci-cluster" +# When non-empty, osac create uses --pull-secret-file / --ssh-public-key-file and +# -p for each entry instead of -f pull_secret= / -f ssh_public_key= (MaaS templates). +caas_cluster_params: [] +# Empty = label/approve every discovered agent with resource_class (setup-caas). +# Non-empty = only these inventory hostnames get osac.openshift.io/resource_class +# (others are unlabeled). setup-maas defaults to h02+h03 so MaaS claims those workers. +caas_resource_class_hostnames: [] caas_agents: - vm_pattern: "hgx-pod00-su0-h01" netris_server_name: "hgx-pod00-su0-h01" diff --git a/infra/netris/netris-lab/group_vars/all.yml b/infra/netris/netris-lab/group_vars/all.yml index aacc5239b..949780f6d 100644 --- a/infra/netris/netris-lab/group_vars/all.yml +++ b/infra/netris/netris-lab/group_vars/all.yml @@ -83,7 +83,7 @@ ns_fabric: oob_gpu_per_switch: 4 oob_uplink_into_spine: 1 softgate_count: 4 - softgate_roles: '["general", "general", "general", "general"]' + softgate_roles: '["general", "general", "snat", "snat"]' softgate_leaf_list: "[0, 1]" softgate_node_type: "softgate_hs" # installer requires _hs to find the package; dpdk disabled post-install leaf_to_softgate_start_port: 176 diff --git a/infra/netris/netris-lab/roles/connectivity/tasks/vpn.yml b/infra/netris/netris-lab/roles/connectivity/tasks/vpn.yml index f95e991e5..a73a4bbfc 100644 --- a/infra/netris/netris-lab/roles/connectivity/tasks/vpn.yml +++ b/infra/netris/netris-lab/roles/connectivity/tasks/vpn.yml @@ -19,6 +19,13 @@ timeout: 600 sleep: 15 +- name: Remove stale SSH host key for mgmt-server + # After destroy/redeploy the mgmt-server VM gets a new host key; clear + # known_hosts so the OpenVPN SSH check below does not fail with + # "REMOTE HOST IDENTIFICATION HAS CHANGED". + shell: ssh-keygen -R 192.168.122.10 2>/dev/null || true + changed_when: false + - name: Wait for mgmt-server OpenVPN server shell: ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@192.168.122.10 "systemctl is-active openvpn@server" register: vpn_server_check diff --git a/infra/netris/playbooks/destroy-full.yml b/infra/netris/playbooks/destroy-full.yml index 468e49b35..f075d8422 100644 --- a/infra/netris/playbooks/destroy-full.yml +++ b/infra/netris/playbooks/destroy-full.yml @@ -308,7 +308,9 @@ - name: Remove stale Ansible collection artifacts from repo ansible.builtin.shell: - cmd: rm -rf /root/netris-test-infra/netris-lab/collections/ansible_collections 2>/dev/null || true + cmd: | + rm -rf /root/osac-test-infra/infra/netris/netris-lab/collections/ansible_collections 2>/dev/null || true + rm -rf /root/netris-test-infra/netris-lab/collections/ansible_collections 2>/dev/null || true changed_when: true # --- Phase 9: Final status --- diff --git a/infra/netris/playbooks/force-destroy-caas.yml b/infra/netris/playbooks/force-destroy-caas.yml new file mode 100644 index 000000000..9247b0c65 --- /dev/null +++ b/infra/netris/playbooks/force-destroy-caas.yml @@ -0,0 +1,10 @@ +--- +# Force destroy: normal destroy-caas (osac delete + agents/InfraEnv), then +# force-destroy-caas (strip stuck orders/namespaces, wipe VMs, Netris orphans). +# No graceful wait — force-clean runs immediately (like destroy-caas.sh --now). +- name: Force destroy CaaS / MaaS + hosts: all + gather_facts: false + roles: + - destroy-caas + - force-destroy-caas diff --git a/infra/netris/roles/create-caas/tasks/main.yml b/infra/netris/roles/create-caas/tasks/main.yml index f303ba656..19fbcd1aa 100644 --- a/infra/netris/roles/create-caas/tasks/main.yml +++ b/infra/netris/roles/create-caas/tasks/main.yml @@ -1,12 +1,24 @@ --- - name: Check if cluster already exists ansible.builtin.shell: - cmd: osac get clusters -o json | python3 -c "import sys,json; clusters=json.load(sys.stdin); print('exists' if any(c.get('name')=='{{ caas_cluster_name }}' for c in clusters) else 'absent')" + cmd: >- + osac get clusters -o json | python3 -c " + import sys, json + raw = json.load(sys.stdin) + clusters = raw if isinstance(raw, list) else (raw.get('items') or [raw]) + def n(c): + meta = c.get('metadata') if isinstance(c.get('metadata'), dict) else {} + return meta.get('name') or c.get('name') or '' + print('exists' if any(n(c)=='{{ caas_cluster_name }}' for c in clusters if isinstance(c, dict)) else 'absent') + " register: _cluster_exists changed_when: false failed_when: false -- name: Create cluster +# Default CaaS path: pull_secret / ssh_public_key are template -f fields + release-image. +# When caas_cluster_params is non-empty (e.g. deploy-maas), use --pull-secret-file / +# --ssh-public-key-file plus -p for template-specific parameters instead. +- name: Create cluster (CaaS template field params) ansible.builtin.command: cmd: >- osac create cluster @@ -15,24 +27,110 @@ -f pull_secret={{ pull_secret_path }} -f ssh_public_key={{ ssh_public_key_path }} --name {{ caas_cluster_name }} - register: _cluster_create - when: _cluster_exists.stdout | default('') != 'exists' + register: _cluster_create_fields + when: + - _cluster_exists.stdout | default('') != 'exists' + - (caas_cluster_params | default([]) | length) == 0 + changed_when: true + +- name: Create cluster (explicit -p params) + ansible.builtin.shell: | + set -euo pipefail + args=( + osac create cluster + --template "{{ caas_cluster_template }}" + --pull-secret-file "{{ pull_secret_path }}" + --ssh-public-key-file "{{ ssh_public_key_path }}" + --name "{{ caas_cluster_name }}" + ) + {% for p in caas_cluster_params | default([]) %} + args+=(-p {{ p | quote }}) + {% endfor %} + printf '+'; + printf ' %q' "${args[@]}" + printf '\n' + "${args[@]}" + args: + executable: /bin/bash + register: _cluster_create_params + when: + - _cluster_exists.stdout | default('') != 'exists' + - (caas_cluster_params | default([]) | length) > 0 changed_when: true +- name: Select create result + ansible.builtin.set_fact: + _cluster_create: "{{ _cluster_create_params if (caas_cluster_params | default([]) | length) > 0 else _cluster_create_fields }}" + when: _cluster_exists.stdout | default('') != 'exists' + - name: Get existing cluster ID ansible.builtin.shell: - cmd: osac get clusters -o json | python3 -c "import sys,json; clusters=json.load(sys.stdin); print(next(c['id'] for c in clusters if c.get('name')=='{{ caas_cluster_name }}'))" + cmd: >- + osac get clusters -o json | python3 -c " + import sys, json + raw = json.load(sys.stdin) + clusters = raw if isinstance(raw, list) else (raw.get('items') or [raw]) + def n(c): + meta = c.get('metadata') if isinstance(c.get('metadata'), dict) else {} + return meta.get('name') or c.get('name') or '' + print(next(c['id'] for c in clusters if isinstance(c, dict) and n(c)=='{{ caas_cluster_name }}')) + " register: _existing_cluster_id when: _cluster_exists.stdout | default('') == 'exists' changed_when: false - name: Extract cluster ID ansible.builtin.set_fact: - caas_cluster_id: "{{ _cluster_create.stdout | regex_search('[0-9a-f-]{36}') if _cluster_create is not skipped else _existing_cluster_id.stdout }}" + caas_cluster_id: "{{ _existing_cluster_id.stdout if (_cluster_exists.stdout | default('')) == 'exists' else (_cluster_create.stdout | regex_search('[0-9a-f-]{36}')) }}" - name: Report cluster creation started ansible.builtin.debug: - msg: "Cluster '{{ caas_cluster_name }}' {{ 'created' if _cluster_create is not skipped else 'already exists' }} with ID {{ caas_cluster_id }}" + msg: "Cluster '{{ caas_cluster_name }}' {{ 'already exists' if (_cluster_exists.stdout | default('')) == 'exists' else 'created' }} with ID {{ caas_cluster_id }}" + +# The HyperShift operator creates the hosted control plane namespace with +# PodSecurity enforce=restricted, which blocks ALL CP pods (etcd, apiserver, +# capi-provider). We must relabel it to privileged before they can start. +- name: Wait for hosted control plane namespace to appear + ansible.builtin.shell: + cmd: >- + oc get ns -l hypershift.openshift.io/hosted-control-plane=true --no-headers 2>/dev/null + | grep '{{ snapshot_osac_namespace | default(osac_namespace) }}' + | awk '{print $1}' + | grep -v '^$' + | head -1 + environment: + KUBECONFIG: /root/.kube/config + register: _cp_namespace + until: _cp_namespace.stdout | length > 0 + retries: 30 + delay: 10 + changed_when: false + +- name: Label hosted control plane namespace with privileged PodSecurity + ansible.builtin.command: + cmd: >- + oc label ns {{ _cp_namespace.stdout }} + pod-security.kubernetes.io/enforce=privileged + pod-security.kubernetes.io/audit=privileged + pod-security.kubernetes.io/warn=privileged + --overwrite + environment: + KUBECONFIG: /root/.kube/config + changed_when: true + +- name: Also label the HostedCluster namespace with privileged PodSecurity + ansible.builtin.shell: + cmd: >- + HC_NS=$(oc get hostedcluster -A --no-headers 2>/dev/null + | grep '{{ snapshot_osac_namespace | default(osac_namespace) }}' + | awk '{print $1}' + | head -1); + [ -n "$HC_NS" ] && oc label ns "$HC_NS" + pod-security.kubernetes.io/enforce=privileged + --overwrite 2>/dev/null || true + environment: + KUBECONFIG: /root/.kube/config + changed_when: false - name: Wait for cluster to be ready ansible.builtin.shell: @@ -52,6 +150,9 @@ msg: "CaaS cluster '{{ caas_cluster_name }}' ({{ caas_cluster_id }}) is in FAILED state" when: _cluster_state.stdout == "CLUSTER_STATE_FAILED" -- name: Report cluster ready +- name: Report final cluster state ansible.builtin.debug: - msg: "CaaS cluster '{{ caas_cluster_name }}' ({{ caas_cluster_id }}) is READY" + msg: >- + CaaS cluster '{{ caas_cluster_name }}' ({{ caas_cluster_id }}) is + {{ _cluster_state.stdout }}{% if _cluster_state.stdout == 'CLUSTER_STATE_PROGRESSING' %}. + Please keep on monitoring the cluster state manually{% endif %} diff --git a/infra/netris/roles/destroy-caas/tasks/main.yml b/infra/netris/roles/destroy-caas/tasks/main.yml index d5ab24ca2..70cfd549a 100644 --- a/infra/netris/roles/destroy-caas/tasks/main.yml +++ b/infra/netris/roles/destroy-caas/tasks/main.yml @@ -1,30 +1,96 @@ --- -# Step 1: Fire-and-forget cluster deletion via osac CLI -- name: Get cluster ID +# Delete the named CaaS/MaaS cluster via osac CLI, then tear down discovery +# agents / InfraEnv / worker VM disks. +# +# Cluster delete previously no-oped: osac was never logged in in this role, so +# "get clusters" returned nothing and the delete task was skipped. + +- name: Resolve OSAC namespace for destroy + ansible.builtin.set_fact: + _destroy_osac_ns: "{{ snapshot_osac_namespace | default(osac_namespace) }}" + +- name: Get fulfillment internal API host ansible.builtin.shell: cmd: >- - osac get clusters -o json | - python3 -c "import sys,json; clusters=json.load(sys.stdin); - matches=[c['id'] for c in clusters if c.get('name')=='{{ caas_cluster_name }}']; - print(matches[0] if matches else '')" + oc get route fulfillment-internal-api -n {{ _destroy_osac_ns }} + -o jsonpath='{.status.ingress[0].host}' + environment: + KUBECONFIG: /root/.kube/config + register: _destroy_ful_host + changed_when: false + failed_when: _destroy_ful_host.rc != 0 or (_destroy_ful_host.stdout | trim | length) == 0 + +- name: Ensure osac CLI config directory exists + ansible.builtin.file: + path: /root/.config/osac + state: directory + mode: "0700" + +- name: Log in osac CLI (admin token) + ansible.builtin.shell: | + set -euo pipefail + rm -f /root/.config/osac/secrets.json + osac logout >/dev/null 2>&1 || true + osac login --insecure --private \ + --token-script "oc create token -n {{ _destroy_osac_ns }} admin --duration 1h" \ + --address "https://{{ _destroy_ful_host.stdout | trim }}" + environment: + KUBECONFIG: /root/.kube/config + changed_when: true + +- name: Resolve cluster ID for {{ caas_cluster_name }} + ansible.builtin.shell: | + set -euo pipefail + osac get clusters -o json | python3 -c ' + import json, sys + raw = json.load(sys.stdin) + clusters = raw if isinstance(raw, list) else (raw.get("items") or [raw]) + name = sys.argv[1] + + def cluster_name(c): + if not isinstance(c, dict): + return "" + # Table "NAME" column is metadata.name; top-level name is often unset. + meta = c.get("metadata") if isinstance(c.get("metadata"), dict) else {} + return (meta.get("name") or c.get("name") or "") + + matches = [c.get("id", "") for c in clusters if cluster_name(c) == name and c.get("id")] + print(matches[0] if matches else "") + ' "{{ caas_cluster_name }}" + environment: + KUBECONFIG: /root/.kube/config register: _cluster_id changed_when: false failed_when: false -# Fire-and-forget: async with no polling -- name: Delete cluster +- name: Report when named cluster is already absent + ansible.builtin.debug: + msg: "No osac cluster named '{{ caas_cluster_name }}' — skip osac delete" + when: (_cluster_id.stdout | trim | length) == 0 + +- name: Delete cluster via osac CLI ansible.builtin.command: - cmd: osac delete cluster {{ _cluster_id.stdout }} - when: _cluster_id.stdout | length > 0 - async: 1 - poll: 0 - changed_when: true + cmd: osac delete cluster {{ _cluster_id.stdout | trim }} + environment: + KUBECONFIG: /root/.kube/config + register: _osac_delete + when: (_cluster_id.stdout | trim | length) > 0 + changed_when: _osac_delete.rc == 0 failed_when: false +- name: Warn if osac delete failed + ansible.builtin.debug: + msg: >- + osac delete cluster {{ _cluster_id.stdout | trim }} failed + (rc={{ _osac_delete.rc }}): {{ _osac_delete.stderr | default(_osac_delete.stdout) | trim }} + when: + - _osac_delete is not skipped + - _osac_delete.rc | default(0) != 0 + # Step 2: Force-delete agents by removing finalizers - name: Get agent names ansible.builtin.shell: - cmd: oc get agent -n {{ caas_agent_namespace }} -o jsonpath='{.items[*].metadata.name}' 2>/dev/null + cmd: oc get agent -n {{ caas_agent_namespace }} -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true environment: KUBECONFIG: /root/.kube/config register: _agents @@ -82,7 +148,7 @@ # Step 5: Shut down discovery VMs and clean up disks - name: Find discovery VMs ansible.builtin.shell: - cmd: virsh list --all --name | grep "{{ item }}" + cmd: virsh list --all --name | grep "{{ item }}" || true register: _discovery_vm_lookup changed_when: false failed_when: false diff --git a/infra/netris/roles/discover-caas/tasks/main.yml b/infra/netris/roles/discover-caas/tasks/main.yml index 6d65b0dc7..312a94e6a 100644 --- a/infra/netris/roles/discover-caas/tasks/main.yml +++ b/infra/netris/roles/discover-caas/tasks/main.yml @@ -202,8 +202,8 @@ - name: Set maximum vCPUs ansible.builtin.command: - cmd: virsh setvcpus {{ item.item }} {{ caas_discovery_vcpu }} --config --maximum - when: item.stdout | int != caas_discovery_vcpu | int + cmd: virsh setvcpus {{ item.item }} {{ caas_discovery_vcpu_overrides[item.item] | default(caas_discovery_vcpu) }} --config --maximum + when: item.stdout | int != (caas_discovery_vcpu_overrides[item.item] | default(caas_discovery_vcpu)) | int loop: "{{ _current_vcpus.results }}" loop_control: label: "{{ item.item }}" @@ -211,8 +211,8 @@ - name: Set vCPUs ansible.builtin.command: - cmd: virsh setvcpus {{ item.item }} {{ caas_discovery_vcpu }} --config - when: item.stdout | int != caas_discovery_vcpu | int + cmd: virsh setvcpus {{ item.item }} {{ caas_discovery_vcpu_overrides[item.item] | default(caas_discovery_vcpu) }} --config + when: item.stdout | int != (caas_discovery_vcpu_overrides[item.item] | default(caas_discovery_vcpu)) | int loop: "{{ _current_vcpus.results }}" loop_control: label: "{{ item.item }}" diff --git a/infra/netris/roles/disk-setup/tasks/main.yml b/infra/netris/roles/disk-setup/tasks/main.yml index c367d8553..de25e919c 100644 --- a/infra/netris/roles/disk-setup/tasks/main.yml +++ b/infra/netris/roles/disk-setup/tasks/main.yml @@ -3,34 +3,59 @@ # Symlinks /var/lib/libvirt/images, /var/lib/rancher, /var/lib/containers # to a mounted data partition to prevent root filesystem exhaustion. -- name: Get root device +# Resolve the physical disk backing / (plain partition or LVM/mapper). +# `lsblk -ndo PKNAME` is empty for LVM roots (e.g. /dev/mapper/rhel-root), +# which previously left _root_device as "/dev/" and allowed wiping the OS disk. +- name: Resolve root physical disk ansible.builtin.shell: - cmd: lsblk -ndo PKNAME $(findmnt -no SOURCE /) 2>/dev/null || findmnt -no SOURCE / | sed 's/[0-9]*$//' + cmd: | + set -euo pipefail + src=$(findmnt -no SOURCE /) + # Walk parent devices until TYPE=disk (works for partitions and LVM). + disk=$(lsblk -nslo NAME,TYPE "$src" 2>/dev/null \ + | awk '$2 == "disk" { print $1; exit }') + if [ -z "${disk}" ]; then + disk=$(lsblk -ndo PKNAME "$src" 2>/dev/null | awk 'NF { print; exit }') + fi + if [ -z "${disk}" ]; then + echo "ERROR: could not resolve root disk from SOURCE=${src}" >&2 + exit 1 + fi + echo "/dev/${disk}" register: _root_device_raw changed_when: false - name: Set root device fact ansible.builtin.set_fact: - _root_device: "/dev/{{ _root_device_raw.stdout | trim | basename }}" + _root_device: "{{ _root_device_raw.stdout | trim }}" -- name: Get all block devices +- name: Find largest unused data disk ansible.builtin.shell: - cmd: lsblk -dnpo NAME,SIZE,TYPE --bytes | awk '$3 == "disk" {print $1, $2}' - register: _all_disks + cmd: | + set -euo pipefail + root_disk="{{ _root_device }}" + # NAME SIZE for real disks, largest first; skip root, nbd, and any disk + # that already has a mounted filesystem (safety for LVM/multi-disk hosts). + while read -r name size; do + [ -n "$name" ] || continue + case "$name" in + "$root_disk"|/dev/nbd*) continue ;; + esac + if lsblk -npo MOUNTPOINT "$name" 2>/dev/null | grep -q '[^[:space:]]'; then + continue + fi + echo "$name" + exit 0 + done < <(lsblk -dnpo NAME,SIZE,TYPE --bytes \ + | awk '$3 == "disk" { print $1, $2 }' \ + | sort -k2 -nr) + exit 0 + register: _data_disk_raw changed_when: false -- name: Find largest unused disk +- name: Set data disk fact ansible.builtin.set_fact: - _data_disk: >- - {{ _all_disks.stdout_lines - | map('split', ' ') - | rejectattr('0', 'equalto', _root_device) - | sort(attribute='1', reverse=true) - | map(attribute='0') - | first - | default('') - }} - + _data_disk: "{{ _data_disk_raw.stdout | trim }}" - name: Check if data mount already exists ansible.builtin.shell: cmd: mountpoint -q /mnt/data && echo "mounted" || echo "not_mounted" @@ -78,6 +103,20 @@ register: _disk_check changed_when: false + - name: Refuse to wipe a disk that still has mounted filesystems + ansible.builtin.shell: + cmd: | + set -euo pipefail + mounts=$(lsblk -npo NAME,MOUNTPOINT {{ _data_disk | trim }} \ + | awk 'NF == 2 { print $1 " -> " $2 }') + if [ -n "$mounts" ]; then + echo "Refusing to wipe {{ _data_disk | trim }}; mounted paths:" >&2 + echo "$mounts" >&2 + exit 1 + fi + when: "'needs_setup' in _disk_check.stdout" + changed_when: false + - name: Wipe and create fresh partition table ansible.builtin.shell: cmd: | diff --git a/infra/netris/roles/force-destroy-caas/files/netris_orphan_cleanup.py b/infra/netris/roles/force-destroy-caas/files/netris_orphan_cleanup.py new file mode 100755 index 000000000..dbf84e909 --- /dev/null +++ b/infra/netris/roles/force-destroy-caas/files/netris_orphan_cleanup.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Remove orphan Netris order-* server clusters / VPCs / NAT / vnets. + +Args: [force_order ...] +Env: NETRIS_LIVE_ORDERS_FILE — newline-separated live ClusterOrder/HC names +""" +import json +import os +import subprocess +import sys + +url, jar, user, password = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] +force = [x for x in sys.argv[5:] if x] + + +def curl(method, path, body=None, fail_ok=False): + cmd = [ + "curl", "-sk", "--connect-timeout", "15", + "-b", jar, "-c", jar, + "-w", "\nHTTP:%{http_code}", + "-X", method, url + path, + ] + if body is not None: + cmd += ["-H", "Content-Type: application/json", "-d", json.dumps(body)] + try: + out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL).decode() + except subprocess.CalledProcessError: + if fail_ok: + return "000", "" + raise + body_out, _, code = out.rpartition("HTTP:") + return code.strip(), body_out + + +subprocess.run( + [ + "curl", "-sk", "-c", jar, "-X", "POST", f"{url}/api/auth", + "-H", "Content-Type: application/json", + "-d", json.dumps({"user": user, "password": password, "auth_scheme_id": 1}), + ], + check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, +) + + +def get_list(path): + code, raw = curl("GET", path, fail_ok=True) + if code != "200": + print(f" ! GET {path} HTTP {code} — skip", file=sys.stderr) + return [] + try: + return json.loads(raw).get("data") or [] + except Exception: + return [] + + +live = set() +live_file = os.environ.get("NETRIS_LIVE_ORDERS_FILE", "") +if live_file and os.path.isfile(live_file): + with open(live_file) as f: + live = {ln.strip() for ln in f if ln.strip()} + +scs = get_list("/api/v2/server-cluster") +vpcs = get_list("/api/v2/vpc") +nats = get_list("/api/v2/nat") + + +def is_order_name(name): + return bool(name) and str(name).startswith("order-") + + +def should_delete(name): + if not is_order_name(name): + return False + if force: + return name in force + return name not in live + + +targets_sc = [c for c in scs if should_delete(c.get("name"))] +targets_vpc = [ + v for v in vpcs + if should_delete(v.get("name")) + and not v.get("isSystem") + and not v.get("isDefault") + and v.get("name") not in ("Default", "ocp-sno") +] + +print(f" · live OSAC orders: {sorted(live) or '(none)'}") +if force: + print(f" · forced: {force}") +print(f" · server clusters to delete: {[c.get('name') for c in targets_sc] or '(none)'}") +print(f" · VPCs to delete: {[v.get('name') for v in targets_vpc] or '(none)'}") + +if not targets_sc and not targets_vpc and not force: + print(" ✓ nothing to clean") + sys.exit(0) + +deleted = 0 +orphan_names = {c.get("name") for c in targets_sc} | {v.get("name") for v in targets_vpc} | set(force) +orphan_vpc_ids = set() +for v in targets_vpc: + if v.get("id") is not None: + orphan_vpc_ids.add(v["id"]) +for c in targets_sc: + vpc = c.get("vpc") or {} + if vpc.get("id") is not None: + orphan_vpc_ids.add(vpc["id"]) + if vpc.get("name"): + orphan_names.add(vpc["name"]) + +for n in nats: + name = n.get("name") or "" + vpc = n.get("vpc") or {} + hit = ( + any(o and o in name for o in orphan_names) + or vpc.get("id") in orphan_vpc_ids + or vpc.get("name") in orphan_names + ) + if not hit: + continue + nid = n.get("id") + print(f" · delete NAT {nid} ({name})") + code, _ = curl("DELETE", f"/api/v2/nat/{nid}", fail_ok=True) + print(f" HTTP {code}") + if code.startswith("2"): + deleted += 1 + +for c in targets_sc: + cid, name = c.get("id"), c.get("name") + servers = c.get("servers") or [] + print(f" · delete server-cluster {cid} ({name}) servers={servers}") + code, raw = curl("DELETE", f"/api/v2/server-cluster/{cid}", fail_ok=True) + print(f" HTTP {code}") + if code.startswith("2"): + deleted += 1 + else: + print(f" body: {raw[:240]}", file=sys.stderr) + +vnets = get_list("/api/v2/vnet") +for vn in vnets: + name = vn.get("name") or "" + vpc = vn.get("vpc") or {} + hit = ( + any(o and o in name for o in orphan_names) + or vpc.get("id") in orphan_vpc_ids + or vpc.get("name") in orphan_names + ) + if not hit: + continue + vid = vn.get("id") + print(f" · delete vnet {vid} ({name})") + code, _ = curl("DELETE", f"/api/v2/vnet/{vid}", fail_ok=True) + print(f" HTTP {code}") + if code.startswith("2"): + deleted += 1 + +vpcs = get_list("/api/v2/vpc") +for v in vpcs: + name = v.get("name") or "" + if not should_delete(name): + continue + if v.get("isSystem") or v.get("isDefault") or name in ("Default", "ocp-sno"): + continue + vid = v.get("id") + print(f" · delete VPC {vid} ({name})") + code, raw = curl("DELETE", f"/api/v2/vpc/{vid}", fail_ok=True) + print(f" HTTP {code}") + if code.startswith("2"): + deleted += 1 + else: + print(f" body: {raw[:240]}", file=sys.stderr) + +print(f" ✓ Netris orphan cleanup done ({deleted} deletes)") diff --git a/infra/netris/roles/force-destroy-caas/tasks/main.yml b/infra/netris/roles/force-destroy-caas/tasks/main.yml new file mode 100644 index 000000000..a1aa80dc7 --- /dev/null +++ b/infra/netris/roles/force-destroy-caas/tasks/main.yml @@ -0,0 +1,321 @@ +--- +# Force-clean leftovers after destroy-caas (no graceful wait — equivalent to +# destroy-caas.sh --now). Strips stuck HostedCluster / ClusterOrder / ManagedCluster +# and related namespaces (incl. HCP ns + ghost ns with Terminating pods/deploys), +# deletes leftover order-* apps LB services, detaches/wipes discovery worker VMs, +# sweeps orphan Netris order-* server clusters and VPCs. + +- name: Resolve OSAC namespace for force destroy + ansible.builtin.set_fact: + _force_osac_ns: "{{ snapshot_osac_namespace | default(osac_namespace) }}" + _force_agent_ns: "{{ caas_agent_namespace }}" + +- name: Collect leftover order names + ansible.builtin.shell: | + set -euo pipefail + ns="{{ _force_osac_ns }}" + { + oc get clusterorder -n "$ns" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true + oc get hostedcluster -A --no-headers 2>/dev/null | awk '{print $2}' | grep '^order-' || true + oc get ns -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ + | while read -r n; do + case "$n" in + klusterlet-order-*) echo "order-${n#klusterlet-order-}" ;; + "${ns}"-order-*-order-*) + # HCP ns: osac-devel-order-xxx-order-xxx + if [[ "$n" =~ ${ns}-(order-[a-z0-9]+)-order- ]]; then + echo "${BASH_REMATCH[1]}" + fi + ;; + "${ns}"-order-*) echo "${n#"${ns}-"}" ;; + esac + done + # Ghost ns: object gone but pods/deploys/HCP CRs remain + { + oc get pods -A --no-headers 2>/dev/null || true + oc get deploy -A --no-headers 2>/dev/null || true + oc get hostedcontrolplanes.hypershift.openshift.io -A --no-headers 2>/dev/null || true + } | awk -v pfx="${ns}-" '$1 ~ pfx"order-" {print $1}' | sort -u \ + | while read -r n; do + if [[ "$n" =~ ${ns}-(order-[a-z0-9]+)-order- ]]; then + echo "${BASH_REMATCH[1]}" + elif [[ "$n" =~ ${ns}-(order-[a-z0-9]+)$ ]]; then + echo "${BASH_REMATCH[1]}" + fi + done + # Leftover apps LB services in OSAC ns (order-xxx-apps-wa-lb) + oc get svc -n "$ns" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ + | while read -r svc; do + if [[ "$svc" =~ ^(order-[a-z0-9]+) ]]; then + echo "${BASH_REMATCH[1]}" + fi + done + } | grep -E '^order-' | sort -u + environment: + KUBECONFIG: /root/.kube/config + register: _force_orders_raw + changed_when: false + failed_when: false + +- name: Set order list + ansible.builtin.set_fact: + force_destroy_orders: "{{ _force_orders_raw.stdout_lines | map('trim') | reject('equalto', '') | list }}" + +- name: Report force-destroy plan + ansible.builtin.debug: + msg: >- + Force-destroy orders={{ force_destroy_orders | default([]) }} + worker_vms={{ caas_discovery_vm_patterns }} + +- name: Force-clean each leftover order + ansible.builtin.shell: | + set -euo pipefail + order="{{ item }}" + ns="{{ _force_osac_ns }}" + hc_ns="${ns}-${order}" + hcp_ns="${hc_ns}-${order}" + + strip() { + local kind="$1" name="$2" n="${3:-}" + if [[ -n "$n" ]]; then + oc patch "$kind" "$name" -n "$n" -p '{"metadata":{"finalizers":null}}' --type=merge 2>/dev/null || true + oc delete "$kind" "$name" -n "$n" --force --grace-period=0 --wait=false 2>/dev/null || true + else + oc patch "$kind" "$name" -p '{"metadata":{"finalizers":null}}' --type=merge 2>/dev/null || true + oc delete "$kind" "$name" --force --grace-period=0 --wait=false 2>/dev/null || true + fi + } + + ns_has_orphans() { + local n="$1" + oc get ns "$n" >/dev/null 2>&1 && return 0 + oc get pods -n "$n" --no-headers 2>/dev/null | grep -q . && return 0 + oc get hostedcontrolplanes.hypershift.openshift.io -n "$n" --no-headers 2>/dev/null | grep -q . && return 0 + oc get deployments.apps -n "$n" --no-headers 2>/dev/null | grep -q . && return 0 + return 1 + } + + force_ns_contents() { + local n="$1" kind name obj + echo " force contents of ns ${n}" + for name in $(oc get pods -n "$n" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do + strip pod "$name" "$n" + done + for kind in \ + hostedcontrolplanes.hypershift.openshift.io \ + controlplanecomponents.hypershift.openshift.io \ + hostedcluster nodepool clusterdeployment agentclusterinstall \ + clusters.cluster.x-k8s.io \ + agentclusters.capi-provider.agent-install.openshift.io \ + agentmachinetemplates.capi-provider.agent-install.openshift.io \ + machinedeployments.cluster.x-k8s.io machinesets.cluster.x-k8s.io machines.cluster.x-k8s.io \ + machinedeployment machineset machine agents \ + deployments.apps replicasets.apps statefulsets.apps daemonsets.apps \ + jobs.batch cronjobs.batch \ + services endpoints endpointslices.discovery.k8s.io \ + persistentvolumeclaims configmaps secrets serviceaccounts \ + roles.rbac.authorization.k8s.io rolebindings.rbac.authorization.k8s.io \ + roles.authorization.openshift.io rolebindings.authorization.openshift.io \ + leases.coordination.k8s.io controllerrevisions.apps \ + networkpolicies.networking.k8s.io poddisruptionbudgets.policy \ + imagestreams.image.openshift.io; do + for name in $(oc get "$kind" -n "$n" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do + strip "$kind" "$name" "$n" + done + done + if oc get pods -n "$n" --no-headers 2>/dev/null | grep -q . \ + || oc get deployments.apps -n "$n" --no-headers 2>/dev/null | grep -q .; then + while read -r kind; do + [[ -z "$kind" ]] && continue + for obj in $(oc get "$kind" -n "$n" -o name 2>/dev/null); do + strip "${obj%%/*}" "${obj#*/}" "$n" + done + done < <(oc api-resources --verbs=delete --namespaced -o name 2>/dev/null || true) + fi + } + + force_ns() { + local n="$1" i left + ns_has_orphans "$n" || return 0 + echo " force ns ${n}" + # Ghost ns: recreate so deletes can finish (premature finalize leaves orphans) + if ! oc get ns "$n" >/dev/null 2>&1; then + echo " ns ${n} missing but orphans remain — recreating to finish purge" + oc create ns "$n" 2>/dev/null || true + fi + force_ns_contents "$n" + if oc get ns "$n" >/dev/null 2>&1; then + oc delete ns "$n" --wait=false 2>/dev/null || true + for i in 1 2 3 4 5 6 7 8 9 10; do + oc get ns "$n" >/dev/null 2>&1 || { echo " ns ${n}: gone"; return 0; } + left=$(oc get pods,deployments.apps,hostedcontrolplanes.hypershift.openshift.io \ + -n "$n" --no-headers 2>/dev/null | wc -l | tr -d ' ') + if [[ "${left:-0}" -gt 0 ]]; then + echo " ns ${n}: ${left} objs still present — re-force before finalize" + force_ns_contents "$n" + sleep 2 + continue + fi + oc patch ns "$n" --type=json -p='[{"op":"remove","path":"/metadata/finalizers"}]' 2>/dev/null || true + oc patch ns "$n" --type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true + oc get ns "$n" -o json 2>/dev/null \ + | python3 -c 'import json,sys; d=json.load(sys.stdin); d["metadata"].pop("finalizers",None); d.setdefault("spec",{})["finalizers"]=[]; json.dump(d,sys.stdout)' \ + | oc replace --raw "/api/v1/namespaces/${n}/finalize" -f - >/dev/null 2>&1 || true + sleep 2 + done + fi + left=$(oc get pods,deployments.apps -n "$n" --no-headers 2>/dev/null | wc -l | tr -d ' ') + if oc get ns "$n" >/dev/null 2>&1 || [[ "${left:-0}" -gt 0 ]]; then + echo " WARN: ns ${n} still present (objs left=${left:-0})" + else + echo " ns ${n}: gone" + fi + } + + force_order_services() { + local name + for name in $(oc get svc -n "$ns" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ + | grep -E "^${order}(-|$)" || true); do + echo " delete leftover svc ${ns}/${name}" + strip service "$name" "$ns" + done + for name in $(oc get route -n "$ns" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ + | grep -E "^${order}" || true); do + echo " delete leftover route ${ns}/${name}" + strip route "$name" "$ns" + done + } + + echo "force-clean order ${order}" + for np in $(oc get nodepool -n "$hc_ns" -o name 2>/dev/null); do + strip "${np%%/*}" "${np#*/}" "$hc_ns" + done + strip hostedcluster "$order" "$hc_ns" + strip clusterorder "$order" "$ns" + strip managedcluster "$order" + force_order_services + # HCP ns first (Terminating control-plane pods/deploys), then HC ns + force_ns "$hcp_ns" + force_ns "$hc_ns" + force_ns "$order" + force_ns "klusterlet-${order}" + environment: + KUBECONFIG: /root/.kube/config + loop: "{{ force_destroy_orders }}" + when: force_destroy_orders | length > 0 + failed_when: false + changed_when: true + +- name: Wipe discovery worker VMs (detach disks/ISO + remove qcow2) + ansible.builtin.shell: | + set -euo pipefail + vm="{{ item }}" + if ! virsh dominfo "$vm" >/dev/null 2>&1; then + echo "VM $vm not found — skip" + exit 0 + fi + if virsh list --name 2>/dev/null | grep -qx "$vm"; then + virsh destroy "$vm" 2>/dev/null || true + fi + while read -r target; do + [[ -z "$target" ]] && continue + virsh change-media "$vm" "$target" --eject --config 2>/dev/null || true + virsh detach-disk "$vm" "$target" --config 2>/dev/null || true + done < <(virsh domblklist "$vm" 2>/dev/null | awk 'NR>2 && NF>=1 {print $1}') + rm -f "/var/lib/libvirt/images/${vm}-disk.qcow2" + echo "$vm powered off; disks/ISO detached; qcow2 removed" + loop: "{{ caas_discovery_vm_patterns }}" + failed_when: false + changed_when: true + +- name: Delete any remaining Agent CRs for wiped VMs + ansible.builtin.shell: | + set -euo pipefail + vm="{{ item }}" + agent_ns="{{ _force_agent_ns }}" + agent=$(oc get agent -n "$agent_ns" -o json 2>/dev/null | python3 -c ' + import json, sys + vm = sys.argv[1] + data = json.load(sys.stdin) + for a in data.get("items") or []: + host = ((a.get("status") or {}).get("inventory") or {}).get("hostname") or "" + if host == vm: + print(a["metadata"]["name"]) + break + ' "$vm" || true) + if [[ -n "$agent" ]]; then + oc delete agent "$agent" -n "$agent_ns" --wait=false 2>/dev/null || true + echo "deleted agent $agent ($vm)" + fi + environment: + KUBECONFIG: /root/.kube/config + loop: "{{ caas_discovery_vm_patterns }}" + failed_when: false + changed_when: true + +- name: Resolve Netris controller URL + ansible.builtin.shell: | + set -euo pipefail + ns="{{ _force_osac_ns }}" + url=$(oc get cm cluster-fulfillment-ig -n "$ns" \ + -o jsonpath='{.data.NETRIS_CONTROLLER_URL}' 2>/dev/null || true) + if [[ -z "$url" && -f /etc/rancher/k3s/k3s.yaml ]]; then + port=$(KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get svc traefik -n kube-system \ + -o jsonpath='{.spec.ports[?(@.name=="web")].nodePort}' 2>/dev/null || true) + [[ -n "$port" ]] && url="http://198.51.100.9:${port}" + fi + printf '%s' "${url%/}" + environment: + KUBECONFIG: /root/.kube/config + register: _netris_url + changed_when: false + failed_when: false + +- name: Resolve Netris password + ansible.builtin.shell: | + set -euo pipefail + ns="{{ _force_osac_ns }}" + pass=$(oc get secret cluster-fulfillment-ig -n "$ns" \ + -o jsonpath='{.data.NETRIS_PASSWORD}' 2>/dev/null | base64 -d 2>/dev/null || true) + printf '%s' "${pass:-netris}" + environment: + KUBECONFIG: /root/.kube/config + register: _netris_pass + changed_when: false + failed_when: false + no_log: true + +- name: Write live-orders file for Netris sweep + ansible.builtin.copy: + dest: /tmp/force-destroy-live-orders.txt + mode: "0600" + content: "" + when: (_netris_url.stdout | trim | length) > 0 + +- name: Sweep orphan Netris order resources + ansible.builtin.shell: | + set -euo pipefail + python3 "{{ role_path }}/files/netris_orphan_cleanup.py" \ + "{{ _netris_url.stdout | trim }}" \ + /tmp/force-destroy-netris.ck \ + "{{ force_destroy_netris_user | default('netris') }}" \ + "$NETRIS_PASS" \ + {{ force_destroy_orders | map('quote') | join(' ') }} + environment: + NETRIS_LIVE_ORDERS_FILE: /tmp/force-destroy-live-orders.txt + NETRIS_PASS: "{{ _netris_pass.stdout }}" + register: _netris_cleanup + when: (_netris_url.stdout | trim | length) > 0 + failed_when: false + changed_when: "'deletes' in (_netris_cleanup.stdout | default(''))" + no_log: false + +- name: Warn if Netris URL undiscoverable + ansible.builtin.debug: + msg: "Netris URL undiscoverable — skipped orphan cleanup" + when: (_netris_url.stdout | trim | length) == 0 + +- name: Show Netris cleanup output + ansible.builtin.debug: + msg: "{{ _netris_cleanup.stdout_lines | default([]) }}" + when: _netris_cleanup is not skipped diff --git a/infra/netris/roles/patch-osac-refresh/tasks/main.yml b/infra/netris/roles/patch-osac-refresh/tasks/main.yml index 83a512a0b..3431f002f 100644 --- a/infra/netris/roles/patch-osac-refresh/tasks/main.yml +++ b/infra/netris/roles/patch-osac-refresh/tasks/main.yml @@ -120,6 +120,15 @@ # (http://osac-aap/api/controller), not an external route hostname that # would change when the snapshot's identity is refreshed. +# Apply Helm/config-as-code-ig AAP project pin to the live Controller Project +# and run CaC once. Snapshot AAP DB keeps the bake-time scm_branch; hourly +# CaC alone is too late for setup-maas publish (see sync-aap-project-pin.yml). +- name: Sync AAP project pin from config-as-code-ig and run CaC + ansible.builtin.include_tasks: sync-aap-project-pin.yml + - name: Report OSAC post-refresh completed ansible.builtin.debug: - msg: "OSAC post-refresh completed — Netris config, SSH keys, and socat configured (AAP token refresh now handled automatically by osac-aap's create-api-token Helm hook)" + msg: >- + OSAC post-refresh completed — Netris config, SSH keys, socat, and AAP + project pin/CaC applied (AAP token refresh handled by osac-aap + create-api-token Helm hook) diff --git a/infra/netris/roles/patch-osac-refresh/tasks/sync-aap-project-pin.yml b/infra/netris/roles/patch-osac-refresh/tasks/sync-aap-project-pin.yml new file mode 100644 index 000000000..12904d1d2 --- /dev/null +++ b/infra/netris/roles/patch-osac-refresh/tasks/sync-aap-project-pin.yml @@ -0,0 +1,287 @@ +# Apply aap.configAsCode.projectGit* from config-as-code-ig onto the live +# AAP Project, then run osac-config-as-code once. +# +# Why: refresh-after-snapshot Helm-upgrades the secret/values pin, but the +# golden snapshot's AAP DB keeps the bake-time scm_branch. Hourly CaC has +# not necessarily run yet (and may never have on a fresh restore), so +# publish-templates / setup-maas can still enumerate roles from the old +# checkout (e.g. pre-OSAC-2816 → ocp_4_20_ai_maas skipped). +# +# Prefer an explicit Project PATCH + /update/ (same idea as installer +# prepare-fulfillment sync_aap_project) over waiting on the schedule. +# Launching CaC afterward reconciles EE image and other Controller objects +# from the same secret. +# +# Mono-repo pins (…/osac, not …/osac-aap): snapshot JTs still use standalone +# root playbook paths (playbook_osac_*.yml). After Project sync, rewrite those +# to osac-aap/… before launching CaC, or CaC fails with "playbook could not +# be found". + +- name: Get AAP route + ansible.builtin.shell: + cmd: oc get route osac-aap -n {{ snapshot_osac_namespace }} -o jsonpath='{.spec.host}' + environment: + KUBECONFIG: /root/.kube/config + register: _aap_pin_route + changed_when: false + until: _aap_pin_route.rc == 0 and _aap_pin_route.stdout | length > 0 + retries: 20 + delay: 15 + +- name: Get AAP admin password + ansible.builtin.shell: + cmd: >- + oc get secret osac-aap-controller-admin-password + -n {{ snapshot_osac_namespace }} + -o jsonpath='{.data.password}' | base64 -d + environment: + KUBECONFIG: /root/.kube/config + register: _aap_pin_admin_pw + changed_when: false + no_log: true + +- name: Read intended AAP project pin from config-as-code-ig + ansible.builtin.shell: + cmd: | + echo "uri=$(oc get secret config-as-code-ig -n {{ snapshot_osac_namespace }} -o jsonpath='{.data.AAP_PROJECT_GIT_URI}' | base64 -d)" + echo "branch=$(oc get secret config-as-code-ig -n {{ snapshot_osac_namespace }} -o jsonpath='{.data.AAP_PROJECT_GIT_BRANCH}' | base64 -d)" + environment: + KUBECONFIG: /root/.kube/config + register: _aap_pin_secret + changed_when: false + +- name: Parse intended AAP project pin + ansible.builtin.set_fact: + _aap_expected_uri: "{{ _aap_pin_secret.stdout_lines | select('match', '^uri=') | first | regex_replace('^uri=', '') }}" + _aap_expected_branch: "{{ _aap_pin_secret.stdout_lines | select('match', '^branch=') | first | regex_replace('^branch=', '') }}" + +- name: Fail if config-as-code-ig pin is incomplete + ansible.builtin.fail: + msg: >- + config-as-code-ig missing AAP_PROJECT_GIT_URI/BRANCH + (uri={{ _aap_expected_uri | default('') }}, branch={{ _aap_expected_branch | default('') }}) + when: (_aap_expected_uri | length) == 0 or (_aap_expected_branch | length) == 0 + +- name: Wait for AAP controller API + ansible.builtin.uri: + url: "https://{{ _aap_pin_route.stdout }}/api/controller/v2/ping/" + method: GET + user: admin + password: "{{ _aap_pin_admin_pw.stdout }}" + force_basic_auth: true + validate_certs: false + status_code: [200] + register: _aap_pin_ping + until: _aap_pin_ping.status == 200 + retries: 30 + delay: 10 + no_log: true + +- name: Resolve AAP project id from osac-publish-templates job template + ansible.builtin.uri: + url: "https://{{ _aap_pin_route.stdout }}/api/controller/v2/job_templates/?name=osac-publish-templates" + method: GET + user: admin + password: "{{ _aap_pin_admin_pw.stdout }}" + force_basic_auth: true + validate_certs: false + register: _aap_pin_jt + until: _aap_pin_jt.status == 200 and (_aap_pin_jt.json.count | default(0)) > 0 + retries: 30 + delay: 10 + no_log: true + +- name: Set AAP project id + ansible.builtin.set_fact: + _aap_project_id: "{{ _aap_pin_jt.json.results[0].project }}" + +- name: Read live AAP project SCM + ansible.builtin.uri: + url: "https://{{ _aap_pin_route.stdout }}/api/controller/v2/projects/{{ _aap_project_id }}/" + method: GET + user: admin + password: "{{ _aap_pin_admin_pw.stdout }}" + force_basic_auth: true + validate_certs: false + register: _aap_project + no_log: true + +- name: Normalize SCM URIs for comparison (ignore trailing .git / slash) + ansible.builtin.set_fact: + _aap_live_uri_norm: "{{ (_aap_project.json.scm_url | default('')) | regex_replace('/$', '') | regex_replace('\\.git$', '') }}" + _aap_expected_uri_norm: "{{ _aap_expected_uri | regex_replace('/$', '') | regex_replace('\\.git$', '') }}" + _aap_live_branch: "{{ _aap_project.json.scm_branch | default('') }}" + +- name: Show AAP project pin plan + ansible.builtin.debug: + msg: >- + AAP project {{ _aap_project_id }}: + live={{ _aap_live_uri_norm }}@{{ _aap_live_branch }} + expected={{ _aap_expected_uri_norm }}@{{ _aap_expected_branch }} + +- name: Patch AAP project SCM to config-as-code-ig pin + ansible.builtin.uri: + url: "https://{{ _aap_pin_route.stdout }}/api/controller/v2/projects/{{ _aap_project_id }}/" + method: PATCH + user: admin + password: "{{ _aap_pin_admin_pw.stdout }}" + force_basic_auth: true + validate_certs: false + body_format: json + body: + scm_url: "{{ _aap_expected_uri }}" + scm_branch: "{{ _aap_expected_branch }}" + status_code: [200] + register: _aap_project_patch + when: >- + _aap_live_uri_norm != _aap_expected_uri_norm + or _aap_live_branch != _aap_expected_branch + no_log: true + +- name: Trigger AAP project update + ansible.builtin.uri: + url: "https://{{ _aap_pin_route.stdout }}/api/controller/v2/projects/{{ _aap_project_id }}/update/" + method: POST + user: admin + password: "{{ _aap_pin_admin_pw.stdout }}" + force_basic_auth: true + validate_certs: false + status_code: [202] + register: _aap_project_update + retries: 5 + delay: 15 + until: _aap_project_update.status == 202 + no_log: true + +- name: Wait for AAP project sync to succeed + ansible.builtin.uri: + url: "https://{{ _aap_pin_route.stdout }}/api/controller/v2/projects/{{ _aap_project_id }}/" + method: GET + user: admin + password: "{{ _aap_pin_admin_pw.stdout }}" + force_basic_auth: true + validate_certs: false + register: _aap_project_status + until: _aap_project_status.json.status in ['successful', 'failed', 'error'] + retries: 40 + delay: 10 + no_log: true + +- name: Fail if AAP project sync did not succeed + ansible.builtin.fail: + msg: >- + AAP project {{ _aap_project_id }} sync finished with status + {{ _aap_project_status.json.status }} (expected successful; + pin {{ _aap_expected_uri }}@{{ _aap_expected_branch }}) + when: _aap_project_status.json.status != 'successful' + +- name: Fail if synced revision does not match SHA pin + ansible.builtin.fail: + msg: >- + AAP project synced to {{ _aap_project_status.json.scm_revision }} + but config-as-code-ig pin is {{ _aap_expected_branch }} + when: + - _aap_expected_branch is match('^[0-9a-f]{40}$') + - (_aap_project_status.json.scm_revision | default('')) != _aap_expected_branch + +# Snapshot JTs assume standalone osac-aap (playbooks at repo root). Mono-repo +# checkouts keep those files under osac-aap/. Rewrite before CaC or launch fails. +- name: Detect mono-repo AAP project pin + ansible.builtin.set_fact: + _aap_pin_is_monorepo: >- + {{ (_aap_expected_uri_norm | regex_search('/osac-aap$')) is none + and (_aap_expected_uri_norm | regex_search('/osac$')) is not none }} + +- name: List job templates for AAP project + ansible.builtin.uri: + url: "https://{{ _aap_pin_route.stdout }}/api/controller/v2/job_templates/?project={{ _aap_project_id }}&page_size=200" + method: GET + user: admin + password: "{{ _aap_pin_admin_pw.stdout }}" + force_basic_auth: true + validate_certs: false + register: _aap_project_jts + when: _aap_pin_is_monorepo | bool + no_log: true + +- name: Build JT playbook rewrites for mono-repo layout + ansible.builtin.set_fact: + _aap_jt_playbook_rewrites: "{{ (_aap_jt_playbook_rewrites | default([])) + [_aap_jt_rewrite] }}" + vars: + _aap_jt_rewrite: + id: "{{ item.id }}" + name: "{{ item.name }}" + playbook: "osac-aap/{{ item.playbook }}" + loop: "{{ _aap_project_jts.json.results | default([]) }}" + when: + - _aap_pin_is_monorepo | bool + - (item.playbook | default('')) is match('^(playbook_osac_.*\\.yml|collections/.*)$') + - not ((item.playbook | default('')) is match('^osac-aap/')) + no_log: true + +- name: Show mono-repo JT playbook rewrite plan + ansible.builtin.debug: + msg: >- + Mono-repo pin: rewriting {{ _aap_jt_playbook_rewrites | default([]) | length }} + job template playbook path(s) under osac-aap/ + when: _aap_pin_is_monorepo | bool + +- name: Prefix mono-repo playbook paths with osac-aap/ + ansible.builtin.uri: + url: "https://{{ _aap_pin_route.stdout }}/api/controller/v2/job_templates/{{ item.id }}/" + method: PATCH + user: admin + password: "{{ _aap_pin_admin_pw.stdout }}" + force_basic_auth: true + validate_certs: false + body_format: json + body: + playbook: "{{ item.playbook }}" + status_code: [200] + loop: "{{ _aap_jt_playbook_rewrites | default([]) }}" + loop_control: + label: "{{ item.name | default(item.id) }} → {{ item.playbook }}" + when: (_aap_jt_playbook_rewrites | default([]) | length) > 0 + no_log: true + +- name: Launch osac-config-as-code (apply pin-driven Controller config) + ansible.builtin.uri: + url: "https://{{ _aap_pin_route.stdout }}/api/controller/v2/job_templates/osac-config-as-code/launch/" + method: POST + user: admin + password: "{{ _aap_pin_admin_pw.stdout }}" + force_basic_auth: true + validate_certs: false + status_code: [201] + register: _aap_cac_job + retries: 5 + delay: 20 + until: _aap_cac_job.status == 201 + +- name: Wait for osac-config-as-code job to complete + ansible.builtin.uri: + url: "https://{{ _aap_pin_route.stdout }}/api/controller/v2/jobs/{{ _aap_cac_job.json.id }}/" + method: GET + user: admin + password: "{{ _aap_pin_admin_pw.stdout }}" + force_basic_auth: true + validate_certs: false + register: _aap_cac_status + until: _aap_cac_status.json.status in ['successful', 'failed', 'error', 'canceled'] + retries: 60 + delay: 15 + no_log: true + +- name: Fail if osac-config-as-code did not succeed + ansible.builtin.fail: + msg: >- + osac-config-as-code job {{ _aap_cac_job.json.id }} finished with status + {{ _aap_cac_status.json.status }} + when: _aap_cac_status.json.status != 'successful' + +- name: Report AAP project pin applied + ansible.builtin.debug: + msg: >- + AAP project {{ _aap_project_id }} at + {{ _aap_project_status.json.scm_revision }} + (pin {{ _aap_expected_branch }}); CaC job {{ _aap_cac_job.json.id }} successful diff --git a/infra/netris/roles/restore-ocp/tasks/main.yml b/infra/netris/roles/restore-ocp/tasks/main.yml index f59ea21e4..bd925a549 100644 --- a/infra/netris/roles/restore-ocp/tasks/main.yml +++ b/infra/netris/roles/restore-ocp/tasks/main.yml @@ -59,6 +59,12 @@ -F qcow2 /var/lib/libvirt/images/hgx-00-lvm.qcow2 +# MaaS: grow CoW OS overlay before nbd mount (flavor root is ~100Gi). +- name: Resize CoW OS overlay for MaaS + ansible.builtin.command: + cmd: qemu-img resize /var/lib/libvirt/images/hgx-00-ocp.qcow2 {{ ocp_snapshot_disk_gb }}G + when: (suite | default('caas')) == 'maas' + - name: Set disk ownership ansible.builtin.file: path: "{{ item }}" @@ -87,6 +93,17 @@ ansible.builtin.pause: seconds: 2 + - name: Grow GPT and root partition to fill overlay (MaaS) + ansible.builtin.shell: | + set -euo pipefail + sgdisk -e /dev/nbd0 + partprobe /dev/nbd0 || true + sleep 1 + parted -s /dev/nbd0 resizepart 4 100% + partprobe /dev/nbd0 || true + sleep 1 + when: (suite | default('caas')) == 'maas' + - name: Create mount point ansible.builtin.file: path: /tmp/flavor-mount @@ -99,6 +116,11 @@ fstype: xfs state: mounted + - name: Grow root XFS filesystem (MaaS) + ansible.builtin.command: + cmd: xfs_growfs /tmp/flavor-mount + when: (suite | default('caas')) == 'maas' + - name: Find ostree deploy directory ansible.builtin.shell: cmd: ls -d /tmp/flavor-mount/ostree/deploy/rhcos/deploy/*/ | head -1 diff --git a/infra/netris/roles/setup-caas/tasks/main.yml b/infra/netris/roles/setup-caas/tasks/main.yml index f0a71b684..b25e24557 100644 --- a/infra/netris/roles/setup-caas/tasks/main.yml +++ b/infra/netris/roles/setup-caas/tasks/main.yml @@ -3,6 +3,8 @@ - name: Wait for expected number of agents ansible.builtin.shell: cmd: oc get agent -n {{ caas_agent_namespace }} --no-headers 2>/dev/null | wc -l + environment: + KUBECONFIG: /root/.kube/config register: _agent_count until: _agent_count.stdout | int >= caas_agents | length retries: 120 @@ -25,6 +27,74 @@ caas_agent_names: "{{ _agent_names.stdout.split() }}" # Step 2: Label agents with resource class and netris server name +# Agent CRs appear before status.inventory is populated; wait until every +# expected agent has a hostname before building the label map. +- name: Get agent hostnames + ansible.builtin.shell: + cmd: | + oc get agent -n {{ caas_agent_namespace }} -o json | python3 -c ' + import json, sys + agents = json.load(sys.stdin).get("items") or [] + expected = {{ caas_agents | length }} + if len(agents) < expected: + raise SystemExit(f"only {len(agents)}/{expected} agents present") + missing = [ + a["metadata"]["name"] + for a in agents + if not ((a.get("status") or {}).get("inventory") or {}).get("hostname") + ] + if missing: + raise SystemExit("inventory hostname missing for: " + ",".join(missing)) + print(json.dumps({ + a["metadata"]["name"]: a["status"]["inventory"]["hostname"] + for a in agents + })) + ' + environment: + KUBECONFIG: /root/.kube/config + register: _agent_hostnames_raw + until: _agent_hostnames_raw.rc == 0 + retries: 40 + delay: 15 + changed_when: false + +- name: Build agent-to-hostname map + ansible.builtin.set_fact: + _agent_hostname_map: "{{ _agent_hostnames_raw.stdout | from_json }}" + +# caas_resource_class_hostnames empty → all agents (CaaS). Non-empty → only those +# inventory hostnames get resource_class (MaaS defaults to h02+h03). +- name: Build resource_class agent allow-list by hostname + ansible.builtin.set_fact: + _agents_for_resource_class: >- + {{ + caas_agent_names + if (caas_resource_class_hostnames | default([]) | length) == 0 else + ( + _agent_hostname_map + | dict2items + | selectattr('value', 'in', caas_resource_class_hostnames) + | map(attribute='key') + | list + ) + }} + +- name: Show resource_class labeling plan + ansible.builtin.debug: + msg: >- + resource_class={{ caas_host_type_id }} on + {{ _agents_for_resource_class | length }}/{{ caas_agent_names | length }} agents + (filter={{ caas_resource_class_hostnames | default([]) }}) + +- name: Fail if resource_class hostname filter matched no agents + ansible.builtin.fail: + msg: >- + caas_resource_class_hostnames={{ caas_resource_class_hostnames }} matched 0 agents. + Known hostnames: {{ _agent_hostname_map.values() | list }} + when: + - (caas_resource_class_hostnames | default([]) | length) > 0 + - (_agents_for_resource_class | length) == 0 + - name: Label agent with resource class ansible.builtin.command: cmd: >- @@ -33,27 +103,27 @@ --overwrite environment: KUBECONFIG: /root/.kube/config - loop: "{{ caas_agent_names }}" + loop: "{{ _agents_for_resource_class }}" register: _label_rc changed_when: "'labeled' in _label_rc.stdout and 'not labeled' not in _label_rc.stdout" until: _label_rc.rc == 0 retries: 5 delay: 10 -- name: Get agent hostnames - ansible.builtin.shell: +- name: Remove resource_class from agents outside the allow-list + ansible.builtin.command: cmd: >- - oc get agent -n {{ caas_agent_namespace }} -o json | - python3 -c "import sys,json; agents=json.load(sys.stdin)['items']; - print(json.dumps({a['metadata']['name']: a['status']['inventory']['hostname'] for a in agents}))" + oc label agent/{{ item }} -n {{ caas_agent_namespace }} + osac.openshift.io/resource_class- --overwrite environment: KUBECONFIG: /root/.kube/config - register: _agent_hostnames_raw - changed_when: false - -- name: Build agent-to-hostname map - ansible.builtin.set_fact: - _agent_hostname_map: "{{ _agent_hostnames_raw.stdout | from_json }}" + when: + - (caas_resource_class_hostnames | default([]) | length) > 0 + - item not in _agents_for_resource_class + loop: "{{ caas_agent_names }}" + register: _unlabel_rc + changed_when: "'labeled' in _unlabel_rc.stdout or 'unlabeled' in _unlabel_rc.stdout" + failed_when: false - name: Label agent with netris server name ansible.builtin.command: @@ -82,7 +152,7 @@ until: _agent_approved.rc == 0 retries: 5 delay: 10 - loop: "{{ caas_agent_names }}" + loop: "{{ _agents_for_resource_class }}" - name: Approve agents ansible.builtin.command: @@ -164,6 +234,70 @@ validate_certs: false status_code: [200, 201, 409] +# AAP netris.steps.cluster_infra looks up netris_resource_class_map[resourceClass] +# from env NETRIS_RESOURCE_CLASS_MAP (cluster-fulfillment-ig). Install only seeds +# the then-current caas_host_type_id (e.g. ci-worker); setup-maas needs g5 too. +# Append only when the key is missing — never overwrite an existing entry. +- name: Read NETRIS_RESOURCE_CLASS_MAP from cluster-fulfillment-ig + ansible.builtin.shell: + cmd: >- + oc get configmap cluster-fulfillment-ig + -n {{ snapshot_osac_namespace | default(osac_namespace) }} + -o jsonpath='{.data.NETRIS_RESOURCE_CLASS_MAP}' + environment: + KUBECONFIG: /root/.kube/config + register: _netris_rc_map_raw + changed_when: false + failed_when: false + +- name: Ensure host type is present in NETRIS_RESOURCE_CLASS_MAP + ansible.builtin.shell: | + set -euo pipefail + python3 <<'PY' + import json, os, subprocess, sys + + raw = os.environ.get("NETRIS_RC_MAP_RAW") or "{}" + try: + data = json.loads(raw) + except json.JSONDecodeError: + data = {} + if not isinstance(data, dict): + data = {} + + name = os.environ["NETRIS_RC_NAME"] + if name in data and isinstance(data[name], dict) and data[name]: + print(f"unchanged {name} already in NETRIS_RESOURCE_CLASS_MAP") + sys.exit(0) + + entry = { + "server_cluster_template_id": int(os.environ["NETRIS_RC_TEMPLATE_ID"]), + "mgmt_interface": os.environ["NETRIS_RC_MGMT_IF"], + "vpc_interfaces": json.loads(os.environ["NETRIS_RC_VPC_IFS"]), + } + data[name] = entry + new_json = json.dumps(data, separators=(",", ":")) + patch = json.dumps({"data": {"NETRIS_RESOURCE_CLASS_MAP": new_json}}) + subprocess.run( + [ + "oc", "patch", "configmap", "cluster-fulfillment-ig", + "-n", os.environ["NETRIS_RC_NS"], + "--type=merge", "-p", patch, + ], + check=True, + ) + print(f"patched added {name}={json.dumps(entry)}") + PY + environment: + KUBECONFIG: /root/.kube/config + NETRIS_RC_NS: "{{ snapshot_osac_namespace | default(osac_namespace) }}" + NETRIS_RC_MAP_RAW: "{{ _netris_rc_map_raw.stdout | default('') }}" + NETRIS_RC_NAME: "{{ caas_host_type_id }}" + NETRIS_RC_TEMPLATE_ID: "{{ caas_netris_server_cluster_template_id | default(2) }}" + NETRIS_RC_MGMT_IF: "{{ caas_netris_mgmt_interface | default('ens4') }}" + NETRIS_RC_VPC_IFS: "{{ caas_netris_vpc_interfaces | default(['ens5']) | to_json }}" + register: _netris_rc_map_ensure + changed_when: "'patched ' in _netris_rc_map_ensure.stdout" + # Step 4: Trigger AAP publish-templates job and wait for cluster template - name: Get AAP route ansible.builtin.shell: diff --git a/infra/netris/roles/setup-infra/tasks/main.yml b/infra/netris/roles/setup-infra/tasks/main.yml index 7ac20ab3e..87100e16c 100644 --- a/infra/netris/roles/setup-infra/tasks/main.yml +++ b/infra/netris/roles/setup-infra/tasks/main.yml @@ -17,10 +17,47 @@ chdir: "{{ netris_lab_dir }}" changed_when: false +# Local BM install (no deploy-jump): same bootstrap as scripts/deploy-jump.sh. +# ansible-core / ansible must already be present to run this playbook; the rest +# is filled in here so `make setup` works after a minimal host bootstrap. +- name: Install BM bootstrap packages + ansible.builtin.dnf: + name: + - git + - make + - ansible-core + - python3-pip + - sshpass + - tmux + - policycoreutils-python-utils + state: present + +# RHEL has no epel-release in base repos; netris-lab prerequisites does +# `dnf install epel-release` which fails without this URL install first. +- name: Pre-install EPEL from URL (RHEL has no epel-release in base repos) + ansible.builtin.shell: + cmd: | + if rpm -q epel-release &>/dev/null; then + echo "already_installed" + else + dnf install -y "https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(rpm -E %rhel).noarch.rpm" + echo "installed" + fi + register: _epel_install + changed_when: "'installed' == _epel_install.stdout | trim" + when: ansible_os_family == "RedHat" + - name: Run prerequisites ansible.builtin.include_role: name: prerequisites +# kubernetes.core.k8s (caas_setup) needs the Python kubernetes client; not installed by +# netris-lab prerequisites (only bcrypt/netaddr). +- name: Install Python kubernetes client + ansible.builtin.pip: + name: kubernetes + state: present + # netris-lab's own prerequisites role drags in an openssl-libs upgrade as a # side effect of its package installs, and that OpenSSL version breaks # Python's ssl.SSLContext.load_verify_locations(cadata=...) bytes-based diff --git a/infra/netris/scripts/deploy-jump.sh b/infra/netris/scripts/deploy-jump.sh index fd598e6e4..d5b5564ae 100755 --- a/infra/netris/scripts/deploy-jump.sh +++ b/infra/netris/scripts/deploy-jump.sh @@ -109,7 +109,7 @@ run_rsync \ --exclude='config' \ --exclude='license.key' \ --exclude='license.zip' \ - "${REPO_ROOT}/" "root@${SERVER}:/root/netris-test-infra/" + "${REPO_ROOT}/" "root@${SERVER}:/root/osac-test-infra/" echo "Repository synced." echo "" @@ -130,7 +130,7 @@ run_ssh "rpm -q epel-release >/dev/null 2>&1 || dnf install -y https://dl.fedora echo "" echo "=== [7/9] Running disk setup on server ===" -run_ssh "cd /root/netris-test-infra/infra/netris && make disk-setup" +run_ssh "cd /root/osac-test-infra/infra/netris && make disk-setup" echo "" echo "=== [8/9] Writing config file ===" @@ -145,20 +145,20 @@ else lab_name = ${LAB_NAME} dns_mode = local" fi -run_ssh "cat > /root/netris-test-infra/infra/netris/config << 'CONFIGEOF' +run_ssh "cat > /root/osac-test-infra/infra/netris/config << 'CONFIGEOF' ${AWS_CONFIG} CONFIGEOF" -run_ssh "cp /root/netris-test-infra/infra/netris/config /root/.netris-config && chmod 0600 /root/.netris-config" +run_ssh "cp /root/osac-test-infra/infra/netris/config /root/.netris-config && chmod 0600 /root/.netris-config" # Symlink license files into repo -run_ssh "ln -sf /root/license.key /root/netris-test-infra/infra/netris/license.key" -run_ssh "ln -sf /root/license.zip /root/netris-test-infra/infra/netris/license.zip" +run_ssh "ln -sf /root/license.key /root/osac-test-infra/infra/netris/license.key" +run_ssh "ln -sf /root/license.zip /root/osac-test-infra/infra/netris/license.zip" echo "Config written." echo "" echo "=== [9/9] Starting deploy in tmux session ===" DEPLOY_TARGET="${DEPLOY_TARGET:-redeploy-fresh}" run_ssh "tmux kill-session -t deploy 2>/dev/null || true" -run_ssh "tmux new-session -d -s deploy -x 200 -y 50 'cd /root/netris-test-infra/infra/netris && make ${DEPLOY_TARGET} 2>&1 | tee -a /root/deploy.log; exec bash'" +run_ssh "tmux new-session -d -s deploy -x 200 -y 50 'cd /root/osac-test-infra/infra/netris && make ${DEPLOY_TARGET} 2>&1 | tee -a /root/deploy.log; exec bash'" echo "" echo "============================================" echo " Deploy started on ${SERVER} in tmux" diff --git a/infra/netris/scripts/redeploy-server.sh b/infra/netris/scripts/redeploy-server.sh index 26f3b1653..7c62a30e8 100755 --- a/infra/netris/scripts/redeploy-server.sh +++ b/infra/netris/scripts/redeploy-server.sh @@ -9,12 +9,27 @@ # unique run ID). This is purely observational — any status-tracking failure # is silently ignored to avoid breaking the deploy flow. # -# Steps: setup-infra → deploy-infra → deploy-ocp → deploy-osac → setup-caas → deploy-caas → post-install +# Steps: +# setup-infra → deploy-infra → deploy-ocp → deploy-osac +# → setup-caas|setup-maas → deploy-caas|deploy-maas → post-install +# +# OSAC_DEPLOY_MODE (fresh|snapshot) is passed through to make for deploy-ocp / +# deploy-osac (same meaning as make deploy vs make deploy-fast): +# fresh — default; deploy-osac does Helm install +# snapshot — deploy-ocp uses snapshot flavor; deploy-osac runs refresh # # Usage: -# make redeploy-fresh → runs this script with --fresh -# make redeploy-continue → runs this script (resumes current run) -# make deploy-jump → from laptop, triggers --fresh via tmux +# make redeploy-fresh → --fresh, SUITE=caas, fresh OSAC +# make redeploy-fresh SUITE=maas → --fresh, MaaS flow +# make redeploy-fresh OSAC_DEPLOY_MODE=snapshot +# → --fresh --snapshot +# make redeploy-continue → resumes (pass same SUITE / mode) +# make deploy-jump → from laptop, triggers --fresh via tmux +# +# Direct: +# scripts/redeploy-server.sh --fresh +# scripts/redeploy-server.sh --fresh --snapshot +# SUITE=maas scripts/redeploy-server.sh --fresh --snapshot set -euo pipefail # --- Configuration --- @@ -24,16 +39,47 @@ STEP_LOG="/tmp/.deploy-step-output" MAX_RETRIES=2 RETRY_DELAY=120 FRESH=false +SNAPSHOT=false +SUITE="${SUITE:-caas}" +OSAC_DEPLOY_MODE="${OSAC_DEPLOY_MODE:-fresh}" + +case "$SUITE" in + caas) + SETUP_FLOW=setup-caas + DEPLOY_FLOW=deploy-caas + ;; + maas) + SETUP_FLOW=setup-maas + DEPLOY_FLOW=deploy-maas + ;; + *) + echo "ERROR: SUITE must be 'caas' or 'maas' (got: $SUITE)" >&2 + exit 1 + ;; +esac # Parse flags while [[ $# -gt 0 ]]; do case "$1" in --fresh) FRESH=true; shift ;; + --snapshot) SNAPSHOT=true; shift ;; *) break ;; esac done EXTRA_VARS="${1:-}" +if [[ "$SNAPSHOT" == "true" ]]; then + OSAC_DEPLOY_MODE=snapshot +fi + +case "$OSAC_DEPLOY_MODE" in + fresh|snapshot) ;; + *) + echo "ERROR: OSAC_DEPLOY_MODE must be 'fresh' or 'snapshot' (got: $OSAC_DEPLOY_MODE)" >&2 + exit 1 + ;; +esac + REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)" RUN_ID="$(date +%Y%m%d-%H%M%S)" @@ -44,9 +90,24 @@ status_init() { if [[ ! -f "$STATUS_FILE" ]]; then echo '{"runs":{},"current_run":null}' > "$STATUS_FILE" fi - local steps_json='{"setup-infra":{"status":"pending"},"deploy-infra":{"status":"pending"},"deploy-ocp":{"status":"pending"},"deploy-osac":{"status":"pending"},"setup-caas":{"status":"pending"},"deploy-caas":{"status":"pending"},"post-install":{"status":"pending"}}' - jq --arg id "$RUN_ID" --argjson steps "$steps_json" \ - '.runs[$id] = {"started_at": (now | todate), "overall_status": "running", "steps": $steps} | .current_run = $id' \ + local steps_json + steps_json=$(jq -nc \ + --arg setup_flow "$SETUP_FLOW" \ + --arg deploy_flow "$DEPLOY_FLOW" \ + '{ + "setup-infra":{"status":"pending"}, + "deploy-infra":{"status":"pending"}, + "deploy-ocp":{"status":"pending"}, + "deploy-osac":{"status":"pending"}, + ($setup_flow):{"status":"pending"}, + ($deploy_flow):{"status":"pending"}, + "post-install":{"status":"pending"} + }') + jq --arg id "$RUN_ID" \ + --arg suite "$SUITE" \ + --arg mode "$OSAC_DEPLOY_MODE" \ + --argjson steps "$steps_json" \ + '.runs[$id] = {"started_at": (now | todate), "suite": $suite, "osac_deploy_mode": $mode, "overall_status": "running", "steps": $steps} | .current_run = $id' \ "$STATUS_FILE" > "${STATUS_FILE}.tmp" && mv "${STATUS_FILE}.tmp" "$STATUS_FILE" } 2>/dev/null || true } @@ -55,6 +116,20 @@ status_resume() { { if [[ -f "$STATUS_FILE" ]]; then RUN_ID=$(jq -r '.current_run // empty' "$STATUS_FILE" 2>/dev/null) || true + # Prefer saved mode/suite from the in-progress run when continuing + local saved_mode saved_suite + saved_mode=$(jq -r --arg run "$RUN_ID" '.runs[$run].osac_deploy_mode // empty' "$STATUS_FILE" 2>/dev/null) || true + saved_suite=$(jq -r --arg run "$RUN_ID" '.runs[$run].suite // empty' "$STATUS_FILE" 2>/dev/null) || true + if [[ -n "$saved_mode" ]]; then + OSAC_DEPLOY_MODE="$saved_mode" + fi + if [[ -n "$saved_suite" ]]; then + SUITE="$saved_suite" + case "$SUITE" in + caas) SETUP_FLOW=setup-caas; DEPLOY_FLOW=deploy-caas ;; + maas) SETUP_FLOW=setup-maas; DEPLOY_FLOW=deploy-maas ;; + esac + fi fi if [[ -z "${RUN_ID:-}" ]]; then RUN_ID="$(date +%Y%m%d-%H%M%S)" @@ -147,18 +222,22 @@ echo "" echo "################################################################" echo "# Deploy run: $RUN_ID ($(date))" echo "# Mode: $(if [[ "$FRESH" == "true" ]]; then echo "fresh"; else echo "continue"; fi)" +echo "# Suite: $SUITE" +echo "# OSAC_DEPLOY_MODE: $OSAC_DEPLOY_MODE" echo "################################################################" echo "" # --- Main pipeline --- +# Same step names as make deploy / deploy-fast; mode selects fresh vs snapshot +# behavior inside deploy-ocp + deploy-osac (see Makefile OSAC_DEPLOY_MODE). STEPS=( setup-infra deploy-infra deploy-ocp deploy-osac - setup-caas - deploy-caas + "$SETUP_FLOW" + "$DEPLOY_FLOW" post-install ) @@ -179,10 +258,15 @@ for step in "${STEPS[@]}"; do echo "" echo "========================================" echo " Running: make $step (attempt $attempt/$MAX_RETRIES)" + echo " SUITE=$SUITE OSAC_DEPLOY_MODE=$OSAC_DEPLOY_MODE" echo "========================================" echo "" - if make "$step" ${EXTRA_VARS:+EXTRA_VARS="${EXTRA_VARS}"} 2>&1 | tee "$STEP_LOG"; then + if make "$step" \ + SUITE="${SUITE}" \ + OSAC_DEPLOY_MODE="${OSAC_DEPLOY_MODE}" \ + ${EXTRA_VARS:+EXTRA_VARS="${EXTRA_VARS}"} \ + 2>&1 | tee "$STEP_LOG"; then echo "$step" >> "$PROGRESS_FILE" status_step "$step" "completed" echo "=== DONE $step ===" @@ -217,6 +301,8 @@ echo "" echo "========================================" echo " Full deploy completed successfully!" echo " Run ID: $RUN_ID" +echo " Suite: $SUITE" +echo " OSAC_DEPLOY_MODE: $OSAC_DEPLOY_MODE" echo " Status: cat $STATUS_FILE | jq '.runs[\"$RUN_ID\"]'" echo "========================================" rm -f "$PROGRESS_FILE" From c001b20de36e4885ebfcbeaf15f3354a8460b4d3 Mon Sep 17 00:00:00 2001 From: Sergio Ruiz Date: Fri, 7 Aug 2026 09:12:48 -0500 Subject: [PATCH 2/4] OSAC-3616: Add per-VM memory overrides for MaaS discovery Keep h01 at 16GiB when growing h02/h03 for real-model headroom, and publish catalog items with ClusterTemplateReference id (OSAC-1330). Signed-off-by: Sergio Ruiz Assisted-by: Cursor Co-authored-by: Cursor --- infra/netris/CLAUDE.md | 2 +- infra/netris/Makefile | 4 +++- infra/netris/README.md | 6 ++++++ infra/netris/inventory/group_vars/all.yml | 2 ++ .../netris/roles/discover-caas/tasks/main.yml | 18 ++++++++++++++---- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/infra/netris/CLAUDE.md b/infra/netris/CLAUDE.md index de794474e..99790b38b 100644 --- a/infra/netris/CLAUDE.md +++ b/infra/netris/CLAUDE.md @@ -92,7 +92,7 @@ All variables in `inventory/group_vars/all.yml`. Key sections: - **OSAC**: `osac_repo/branch`, `osac_namespace`, `osac_values_file` - **Component images**: `osac_operator_image`, `fulfillment_service_image` (empty = defaults) - **Snapshot**: `snapshot_flavor_image`, `snapshot_flavor_dir`, `snapshot_recert_image`, `snapshot_osac_namespace`, `snapshot_osac_values_file` -- **CaaS / MaaS**: `caas_discovery_vm_patterns`, `caas_host_type_id`, `caas_cluster_name`, `caas_agents`, `caas_resource_class_hostnames` (empty = all agents); MaaS overrides via Makefile `MAAS_*` +- **CaaS / MaaS**: `caas_discovery_vm_patterns`, `caas_host_type_id`, `caas_cluster_name`, `caas_agents`, `caas_resource_class_hostnames` (empty = all agents); per-VM sizing via `caas_discovery_vcpu_overrides` / `caas_discovery_memory_mb_overrides`; MaaS Makefile `MAAS_*` (incl. `MAAS_DISCOVERY_MEMORY_MB_OVERRIDES`, default h01→16GiB) ## External Dependencies diff --git a/infra/netris/Makefile b/infra/netris/Makefile index d6f1a82cb..ace2a1065 100644 --- a/infra/netris/Makefile +++ b/infra/netris/Makefile @@ -119,10 +119,11 @@ MAAS_HOST_TYPE_ID ?= g5 MAAS_CLUSTER_TEMPLATE ?= osac.templates.ocp_4_20_ai_maas MAAS_CATALOG_ITEM ?= maas-ocp # Discovery VM sizing for setup-maas only (CaaS keeps inventory defaults: 4 vCPU / 16GiB). -# Default MaaS: 10 vCPU / 20GiB for h02–h03; h01 stays at 4 (unused for MaaS workers). +# Default MaaS: 10 vCPU / 20GiB for h02–h03; h01 stays at 4 vCPU / 16GiB (unused for MaaS workers). MAAS_DISCOVERY_VCPU ?= 10 MAAS_DISCOVERY_MEMORY_MB ?= 20480 MAAS_DISCOVERY_VCPU_OVERRIDES ?= {"hgx-pod00-su0-h01":4} +MAAS_DISCOVERY_MEMORY_MB_OVERRIDES ?= {"hgx-pod00-su0-h01":16384} # Only these inventory hostnames get resource_class=g5 (and approval). Empty = all agents. # Comma-separated. Keeps h01 out of the MaaS claim pool; setup-caas still labels all as ci-worker. MAAS_LABEL_HOSTNAMES ?= hgx-pod00-su0-h02,hgx-pod00-su0-h03 @@ -143,6 +144,7 @@ MAAS_EXTRA = \ -e 'caas_discovery_vcpu=$(MAAS_DISCOVERY_VCPU)' \ -e 'caas_discovery_memory_mb=$(MAAS_DISCOVERY_MEMORY_MB)' \ -e '{"caas_discovery_vcpu_overrides":$(MAAS_DISCOVERY_VCPU_OVERRIDES)}' \ + -e '{"caas_discovery_memory_mb_overrides":$(MAAS_DISCOVERY_MEMORY_MB_OVERRIDES)}' \ -e '{"caas_resource_class_hostnames":$(MAAS_LABEL_HOSTNAMES_JSON)}' setup-maas: diff --git a/infra/netris/README.md b/infra/netris/README.md index e4b299771..12bf1f8f3 100644 --- a/infra/netris/README.md +++ b/infra/netris/README.md @@ -343,6 +343,10 @@ make deploy-osac EXTRA_VARS='{"osac_branch": "feature-x"}' ```bash make deploy-ocp EXTRA_VARS="ocp_version=4.18" make setup-caas EXTRA_VARS="caas_cluster_name=my-cluster caas_discovery_vcpu=8" + +# MaaS: default 10 vCPU / 20GiB for h02–h03; h01 stays 4 vCPU / 16GiB via overrides. +# Grow workers only (e.g. real model headroom) without resizing h01: +make setup-maas MAAS_DISCOVERY_MEMORY_MB=49152 ``` #### Lab & Identity @@ -425,7 +429,9 @@ dns_server: "10.0.0.1" | `caas_cluster_template` | `osac.templates.ocp_ci_small` | Cluster template for CaaS | defaults only | | `caas_host_type_id` | `ci-worker` | Resource class label for CaaS agents | defaults only | | `caas_discovery_vcpu` | `4` | Discovery VM vCPUs | yes (8) | +| `caas_discovery_vcpu_overrides` | `{}` | Per-VM vCPU map (VM name → count); empty = use `caas_discovery_vcpu` | defaults only | | `caas_discovery_memory_mb` | `16384` | Discovery VM memory in MB | yes (32768) | +| `caas_discovery_memory_mb_overrides` | `{}` | Per-VM memory map (VM name → MB); empty = use `caas_discovery_memory_mb` | defaults only | | `caas_discovery_disk_gb` | `100` | Discovery VM disk in GB | yes (150) | | `caas_discovery_vm_patterns` | `[hgx-pod00-su0-h01..03]` | VM names for CaaS discovery | defaults only | diff --git a/infra/netris/inventory/group_vars/all.yml b/infra/netris/inventory/group_vars/all.yml index a956b1217..cd6e295db 100644 --- a/infra/netris/inventory/group_vars/all.yml +++ b/infra/netris/inventory/group_vars/all.yml @@ -97,6 +97,8 @@ caas_discovery_vcpu: 4 # Optional per-VM vCPU overrides (VM name → count). Empty = use caas_discovery_vcpu for all. caas_discovery_vcpu_overrides: {} caas_discovery_memory_mb: 16384 +# Optional per-VM memory overrides (VM name → MB). Empty = use caas_discovery_memory_mb for all. +caas_discovery_memory_mb_overrides: {} caas_discovery_disk_gb: 100 caas_discovery_infraenv_name: "infraenv" caas_discovery_infraenv_namespace: "hardware-inventory" diff --git a/infra/netris/roles/discover-caas/tasks/main.yml b/infra/netris/roles/discover-caas/tasks/main.yml index 312a94e6a..58dd831b4 100644 --- a/infra/netris/roles/discover-caas/tasks/main.yml +++ b/infra/netris/roles/discover-caas/tasks/main.yml @@ -227,8 +227,13 @@ - name: Set maximum memory ansible.builtin.command: - cmd: virsh setmaxmem {{ item.item }} {{ caas_discovery_memory_mb }}M --config - when: item.stdout | int != caas_discovery_memory_mb | int + cmd: >- + virsh setmaxmem {{ item.item }} + {{ caas_discovery_memory_mb_overrides[item.item] | default(caas_discovery_memory_mb) }}M + --config + when: >- + item.stdout | int != + (caas_discovery_memory_mb_overrides[item.item] | default(caas_discovery_memory_mb)) | int loop: "{{ _current_memory.results }}" loop_control: label: "{{ item.item }}" @@ -236,8 +241,13 @@ - name: Set memory ansible.builtin.command: - cmd: virsh setmem {{ item.item }} {{ caas_discovery_memory_mb }}M --config - when: item.stdout | int != caas_discovery_memory_mb | int + cmd: >- + virsh setmem {{ item.item }} + {{ caas_discovery_memory_mb_overrides[item.item] | default(caas_discovery_memory_mb) }}M + --config + when: >- + item.stdout | int != + (caas_discovery_memory_mb_overrides[item.item] | default(caas_discovery_memory_mb)) | int loop: "{{ _current_memory.results }}" loop_control: label: "{{ item.item }}" From c726cdd9fb26b4a7f74b5a231229a9d884c5db64 Mon Sep 17 00:00:00 2001 From: Sergio Ruiz Date: Mon, 10 Aug 2026 15:58:15 -0500 Subject: [PATCH 3/4] OSAC-3616: Restore all-general softgate roles SNAT softgate pairing did not uniquely fix the SoftGate VRF issue; keep Dan's general/general/general/general default. Signed-off-by: Sergio Ruiz Assisted-by: Cursor Co-authored-by: Cursor --- infra/netris/netris-lab/README.md | 2 +- infra/netris/netris-lab/group_vars/all.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/netris/netris-lab/README.md b/infra/netris/netris-lab/README.md index a5988cff7..c78200eca 100644 --- a/infra/netris/netris-lab/README.md +++ b/infra/netris/netris-lab/README.md @@ -75,7 +75,7 @@ All variables are in `group_vars/all.yml`. | `leaf_count` | `2` | NS leaf switches | | `spine_count` | `2` | NS spine switches | | `softgate_count` | `4` | Softgate VMs | -| `softgate_roles` | `'["general","general","snat","snat"]'` | Softgate role assignment | +| `softgate_roles` | `'["general","general","general","general"]'` | Softgate role assignment | | `softgate_node_type` | `"softgate_hs"` | Installer node type (dpdk disabled post-install) | | `oob_leaf_count` | `1` | OOB leaf switches | | `oob_gpu_per_switch` | `4` | GPU servers per OOB leaf | diff --git a/infra/netris/netris-lab/group_vars/all.yml b/infra/netris/netris-lab/group_vars/all.yml index 949780f6d..aacc5239b 100644 --- a/infra/netris/netris-lab/group_vars/all.yml +++ b/infra/netris/netris-lab/group_vars/all.yml @@ -83,7 +83,7 @@ ns_fabric: oob_gpu_per_switch: 4 oob_uplink_into_spine: 1 softgate_count: 4 - softgate_roles: '["general", "general", "snat", "snat"]' + softgate_roles: '["general", "general", "general", "general"]' softgate_leaf_list: "[0, 1]" softgate_node_type: "softgate_hs" # installer requires _hs to find the package; dpdk disabled post-install leaf_to_softgate_start_port: 176 From dae8000a357c35523358191a6834d78242ae459f Mon Sep 17 00:00:00 2001 From: Sergio Ruiz Date: Thu, 20 Aug 2026 07:19:33 -0500 Subject: [PATCH 4/4] OSAC-3616: Fix trailing whitespace in infra/netris/CLAUDE.md Remove trailing spaces on the MaaS workflow line so pre-commit trim-trailing-whitespace passes in CI. Signed-off-by: Sergio Ruiz Assisted-by: Cursor Co-authored-by: Cursor --- infra/netris/CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/netris/CLAUDE.md b/infra/netris/CLAUDE.md index 99790b38b..21f716c01 100644 --- a/infra/netris/CLAUDE.md +++ b/infra/netris/CLAUDE.md @@ -69,7 +69,7 @@ make gather-infra # Gather diagnostic info from the cluster **CaaS:** deploy or deploy-fast → setup-caas → deploy-caas -**MaaS:** deploy or deploy-fast → setup-maas → deploy-maas +**MaaS:** deploy or deploy-fast → setup-maas → deploy-maas (or `make redeploy-fresh SUITE=maas OSAC_DEPLOY_MODE=snapshot`) **VMaaS / BMaaS:** not yet implemented