From 1021a60127ee370140f26099e2424e7232312287 Mon Sep 17 00:00:00 2001 From: abtreece Date: Thu, 30 Apr 2026 14:23:14 -0500 Subject: [PATCH 1/9] feat(ansible): add Molecule + Lima testing for the Hetzner playbook Adds a Molecule scenario that converges the playbook against a Lima VM with all live external dependencies (Azure Key Vault, GCS, GitHub releases) replaced by stubs, so the scenario runs without cloud credentials. Full lifecycle (create + prepare + converge + idempotence + verify + destroy) is green end-to-end. Test seams in prod files (~10 lines, all gated behind molecule_test; prod behavior byte-identical when molecule_test is unset): - templates/caddy-env.j2: skip az keyvault lookups when molecule_test - tasks/caddy.yml: gate query-latest-repo-versions and Caddyfile install - tasks/apiserver-deployer.yml: gate apiserver-deployer.sh install - tasks/unattended-upgrades.yml: gate the GitHub-release deb install Test artifacts live under ansible/files/molecule-test/ (stubs and service-contract fixture) and ansible/molecule/default/ (scenario). Pinned tooling versions in ansible/requirements.txt. Contributor instructions in ansible/README.md. --- ansible/README.md | 95 ++++++++++++++ .../molecule-test/apiserver-fixture/Gemfile | 7 ++ .../molecule-test/apiserver-fixture/config.ru | 9 ++ .../molecule-test/stub-apiserver-deployer | 6 + .../stub-query-latest-repo-versions | 10 ++ ansible/files/molecule-test/test-Caddyfile | 16 +++ ansible/molecule/default/converge.yml | 117 ++++++++++++++++++ ansible/molecule/default/create.yml | 43 +++++++ ansible/molecule/default/destroy.yml | 10 ++ ansible/molecule/default/files | 1 + ansible/molecule/default/lima.yaml | 18 +++ ansible/molecule/default/molecule.yml | 38 ++++++ ansible/molecule/default/prepare.yml | 21 ++++ ansible/molecule/default/templates | 1 + ansible/molecule/default/verify.yml | 53 ++++++++ ansible/requirements.txt | 3 + ansible/tasks/apiserver-deployer.yml | 1 + ansible/tasks/caddy.yml | 2 + ansible/tasks/unattended-upgrades.yml | 1 + ansible/templates/caddy-env.j2 | 5 + 20 files changed, 457 insertions(+) create mode 100644 ansible/README.md create mode 100644 ansible/files/molecule-test/apiserver-fixture/Gemfile create mode 100644 ansible/files/molecule-test/apiserver-fixture/config.ru create mode 100644 ansible/files/molecule-test/stub-apiserver-deployer create mode 100644 ansible/files/molecule-test/stub-query-latest-repo-versions create mode 100644 ansible/files/molecule-test/test-Caddyfile create mode 100644 ansible/molecule/default/converge.yml create mode 100644 ansible/molecule/default/create.yml create mode 100644 ansible/molecule/default/destroy.yml create mode 120000 ansible/molecule/default/files create mode 100644 ansible/molecule/default/lima.yaml create mode 100644 ansible/molecule/default/molecule.yml create mode 100644 ansible/molecule/default/prepare.yml create mode 120000 ansible/molecule/default/templates create mode 100644 ansible/molecule/default/verify.yml create mode 100644 ansible/requirements.txt diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 0000000..ecff392 --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,95 @@ +# `infra/ansible/` — Hetzner VM playbook + +Configures `backend.fullstaqruby.org`. See `../docs/` for operational context. + +## Local testing with Molecule + Lima + +This playbook ships a Molecule scenario that converges the playbook against a +Lima VM, with all live external dependencies (Azure Key Vault, GCS, GitHub +releases) replaced by stubs so the scenario runs offline-friendly without +cloud credentials. + +### One-time setup (macOS) + +``` +brew install lima uv +uv tool install --with ansible-core --with molecule-plugins --python 3.12 molecule +``` + +`uv tool install` puts each tool in its own isolated venv. Molecule shells out +to `ansible`, `ansible-playbook`, `ansible-config`, etc., so the entire tool +venv's `bin/` directory must be on PATH — adding only the `molecule` symlink +isn't enough. Add this to your shell rc (or run per-session): + +``` +export PATH="$HOME/.local/share/uv/tools/molecule/bin:$PATH" +``` + +Install the Ansible collections the playbook depends on: + +``` +ansible-galaxy collection install community.general ansible.posix +``` + +### Daily loop + +From this directory (`infra/ansible/`): + +``` +molecule converge # iterative; keeps VM alive between runs +molecule verify # run assertions against the converged VM +molecule idempotence # second-run check (changed=0) +molecule destroy # tear down the Lima VM +molecule test # full lifecycle: destroy → create → prepare → converge → idempotence → verify → destroy +``` + +Approximate timings on Apple Silicon (arm64 native guest): + +- VM boot: ~20s +- First converge: ~3–5 min (apt installs + bundle install of fixture) +- Subsequent converges: ~30–60s +- Verify: ~10s + +### What the scenario does + +- Boots a Debian 12 cloud-image VM via Lima (`molecule/default/lima.yaml`) +- Bootstraps packages the prod playbook assumes preinstalled (cron, ufw, + acl, rsyslog, ruby-dev, build-essential — see + `molecule/default/prepare.yml`) +- Runs `main.yml`'s task list with `molecule_test: true`, which causes four + prod tasks to skip live external calls +- Stages a service-contract fixture (minimal Sinatra app on Puma) at + `/opt/apiserver/versions/latest` so the apiserver systemd unit can start + successfully +- Verifies that `caddy`, `prometheus`, `fail2ban`, `ssh`, and `apiserver` + are all `systemctl is-active`, that the apiserver Unix socket exists, that + `caddy validate` passes, and that SSH/UFW/unattended-upgrades configuration + matches expectations + +### Inspecting the VM + +``` +limactl shell ansible-molecule # interactive shell on the VM +limactl shell ansible-molecule sudo systemctl status caddy +limactl shell ansible-molecule sudo journalctl -u apiserver +``` + +### Troubleshooting + +- **`No version is set for command molecule`** — your shell is using asdf + shims that shadow the uv-installed binaries. Make sure + `~/.local/share/uv/tools/molecule/bin` is *before* asdf's shim dir on PATH. +- **`Could not find or access '.j2'` or `.sh`** — a `files/` or + `templates/` symlink in `molecule/default/` may be missing or broken; both + must point to `../../files` and `../../templates` respectively. +- **`limactl: command not found`** — `brew install lima`. +- **VM in a wedged state** — `molecule destroy` then re-run. As a last resort: + `limactl delete --force ansible-molecule`. + +### What is *not* tested by this scenario + +- Real ACME/TLS issuance (test Caddyfile uses `auto_https off`) +- Real Azure Key Vault, GCS, or GitHub-release integration +- OIDC JWT verification (covered by `../apiserver/` Ruby tests) +- Live `fail2ban` banning behavior (only verifies the unit starts cleanly) +- AppArmor profile loading (the playbook only installs the package today) diff --git a/ansible/files/molecule-test/apiserver-fixture/Gemfile b/ansible/files/molecule-test/apiserver-fixture/Gemfile new file mode 100644 index 0000000..d0333d1 --- /dev/null +++ b/ansible/files/molecule-test/apiserver-fixture/Gemfile @@ -0,0 +1,7 @@ +source "https://rubygems.org" + +ruby ">= 3.0" + +gem "puma" +gem "sinatra", "~> 4" +gem "rackup" diff --git a/ansible/files/molecule-test/apiserver-fixture/config.ru b/ansible/files/molecule-test/apiserver-fixture/config.ru new file mode 100644 index 0000000..24b775d --- /dev/null +++ b/ansible/files/molecule-test/apiserver-fixture/config.ru @@ -0,0 +1,9 @@ +require "sinatra/base" + +class FixtureApp < Sinatra::Base + get "/admin/health" do + "ok" + end +end + +run FixtureApp diff --git a/ansible/files/molecule-test/stub-apiserver-deployer b/ansible/files/molecule-test/stub-apiserver-deployer new file mode 100644 index 0000000..36d60cd --- /dev/null +++ b/ansible/files/molecule-test/stub-apiserver-deployer @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Test stub for apiserver-deployer. Real script downloads tarballs from +# GitHub releases. Test fixture is staged separately by Molecule converge. +set -eu +echo "[stub-apiserver-deployer] no-op (fixture staged by Molecule)" +exit 0 diff --git a/ansible/files/molecule-test/stub-query-latest-repo-versions b/ansible/files/molecule-test/stub-query-latest-repo-versions new file mode 100644 index 0000000..d69ed38 --- /dev/null +++ b/ansible/files/molecule-test/stub-query-latest-repo-versions @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Test stub for query-latest-repo-versions. Real script reads from GCS. +# Writes static placeholder values into the file Caddy expects. +set -eu +OUT="${1:-/etc/caddy/env-repo-versions}" +cat >"$OUT" < + limactl start --tty=false --name=ansible-molecule + {{ molecule_scenario_directory }}/lima.yaml + when: "'ansible-molecule' not in limactl_list.stdout" + changed_when: true + + - name: Get SSH config from Lima + ansible.builtin.command: limactl show-ssh --format=config ansible-molecule + register: lima_ssh_config + changed_when: false + + - name: Write Lima SSH config to ephemeral dir + ansible.builtin.copy: + content: "{{ lima_ssh_config.stdout }}\n" + dest: "{{ molecule_ephemeral_directory }}/ssh_config" + mode: "0600" + + - name: Update inventory with SSH config + ansible.builtin.copy: + dest: "{{ molecule_ephemeral_directory }}/inventory/hosts.yml" + mode: "0644" + content: | + all: + hosts: + ansible-molecule: + ansible_connection: ssh + ansible_host: lima-ansible-molecule + ansible_user: "{{ lookup('env', 'USER') }}" + ansible_ssh_common_args: "-F {{ molecule_ephemeral_directory }}/ssh_config" + ansible_become: true + ansible_python_interpreter: /usr/bin/python3 diff --git a/ansible/molecule/default/destroy.yml b/ansible/molecule/default/destroy.yml new file mode 100644 index 0000000..faa35d5 --- /dev/null +++ b/ansible/molecule/default/destroy.yml @@ -0,0 +1,10 @@ +- name: Destroy Lima VM + hosts: localhost + connection: local + gather_facts: false + tasks: + - name: Stop and delete Lima instance + ansible.builtin.command: limactl delete --force ansible-molecule + register: result + changed_when: result.rc == 0 + failed_when: false diff --git a/ansible/molecule/default/files b/ansible/molecule/default/files new file mode 120000 index 0000000..81016f4 --- /dev/null +++ b/ansible/molecule/default/files @@ -0,0 +1 @@ +../../files \ No newline at end of file diff --git a/ansible/molecule/default/lima.yaml b/ansible/molecule/default/lima.yaml new file mode 100644 index 0000000..7759382 --- /dev/null +++ b/ansible/molecule/default/lima.yaml @@ -0,0 +1,18 @@ +images: + - location: "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-arm64.qcow2" + arch: "aarch64" + - location: "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2" + arch: "x86_64" +cpus: 2 +memory: "2GiB" +disk: "20GiB" +ssh: + loadDotSSHPubKeys: true +mounts: [] +provision: + - mode: system + script: | + #!/bin/sh + set -eux + apt-get update + apt-get install -y python3 sudo diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml new file mode 100644 index 0000000..8e574e4 --- /dev/null +++ b/ansible/molecule/default/molecule.yml @@ -0,0 +1,38 @@ +dependency: + name: galaxy +driver: + name: default + options: + managed: false + login_cmd_template: "limactl shell {instance}" + ansible_connection_options: + ansible_connection: ssh +platforms: + - name: ansible-molecule +provisioner: + name: ansible + config_options: + defaults: + forks: 5 + inventory: + hosts: + all: + hosts: + ansible-molecule: + ansible_user: lima-ansible-molecule + ansible_host: 127.0.0.1 + ansible_become: true + ansible_python_interpreter: /usr/bin/python3 +verifier: + name: ansible +scenario: + test_sequence: + - dependency + - destroy + - syntax + - create + - prepare + - converge + - idempotence + - verify + - destroy diff --git a/ansible/molecule/default/prepare.yml b/ansible/molecule/default/prepare.yml new file mode 100644 index 0000000..cdc4600 --- /dev/null +++ b/ansible/molecule/default/prepare.yml @@ -0,0 +1,21 @@ +- name: Prepare guest + hosts: all + gather_facts: true + tasks: + - name: Ensure python3 and sudo are present + ansible.builtin.raw: | + which python3 || (apt-get update && apt-get install -y python3 sudo) + changed_when: false + + - name: Install packages prod expects but playbook does not declare + ansible.builtin.apt: + name: + - cron + - ufw + - lsb-release + - acl + - ruby-dev + - build-essential + - rsyslog + update_cache: true + cache_valid_time: 3600 diff --git a/ansible/molecule/default/templates b/ansible/molecule/default/templates new file mode 120000 index 0000000..07531b7 --- /dev/null +++ b/ansible/molecule/default/templates @@ -0,0 +1 @@ +../../templates \ No newline at end of file diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml new file mode 100644 index 0000000..7c85f47 --- /dev/null +++ b/ansible/molecule/default/verify.yml @@ -0,0 +1,53 @@ +- name: Verify + hosts: all + gather_facts: true + tasks: + - name: Core services are active + ansible.builtin.command: systemctl is-active {{ item }} + loop: + - caddy + - prometheus + - fail2ban + - ssh + - apiserver + changed_when: false + + - name: Apiserver socket exists + ansible.builtin.stat: + path: /run/apiserver/server.sock + register: apiserver_socket + failed_when: not apiserver_socket.stat.exists + + - name: Apiserver-deployer one-shot is active (RemainAfterExit) + ansible.builtin.command: systemctl is-active apiserver-deployer + changed_when: false + + - name: Caddy config validates + ansible.builtin.command: caddy validate --config /etc/caddy/Caddyfile --envfile /etc/caddy/env + changed_when: false + + - name: AppArmor package is installed + ansible.builtin.package_facts: {} + + - name: Assert AppArmor is installed + ansible.builtin.assert: + that: "'apparmor' in ansible_facts.packages" + + - name: UFW is active + ansible.builtin.command: ufw status verbose + register: ufw_status + changed_when: false + failed_when: "'Status: active' not in ufw_status.stdout" + + - name: SSH config matches hardening expectations + ansible.builtin.command: sshd -T + register: sshd_config + changed_when: false + + - name: Assert PermitRootLogin without-password + ansible.builtin.assert: + that: "'permitrootlogin without-password' in sshd_config.stdout" + + - name: Unattended-upgrade dry-run succeeds + ansible.builtin.command: unattended-upgrade --dry-run + changed_when: false diff --git a/ansible/requirements.txt b/ansible/requirements.txt new file mode 100644 index 0000000..af047da --- /dev/null +++ b/ansible/requirements.txt @@ -0,0 +1,3 @@ +ansible-core==2.20.5 +molecule==26.4.0 +molecule-plugins==25.8.12 diff --git a/ansible/tasks/apiserver-deployer.yml b/ansible/tasks/apiserver-deployer.yml index 3f42f21..8cca5ff 100644 --- a/ansible/tasks/apiserver-deployer.yml +++ b/ansible/tasks/apiserver-deployer.yml @@ -41,6 +41,7 @@ owner: root group: root mode: 0755 + when: not (molecule_test | default(false)) notify: Restart apiserver-deployer - name: Install apiserver-deployer systemd unit diff --git a/ansible/tasks/caddy.yml b/ansible/tasks/caddy.yml index 4f96114..363ac1d 100644 --- a/ansible/tasks/caddy.yml +++ b/ansible/tasks/caddy.yml @@ -35,6 +35,7 @@ owner: root group: root mode: 0755 + when: not (molecule_test | default(false)) notify: Restart Caddy - name: Create Caddy config dir @@ -87,6 +88,7 @@ owner: caddy group: caddy mode: 0644 + when: not (molecule_test | default(false)) notify: Restart Caddy - name: Create Caddy systemd service diff --git a/ansible/tasks/unattended-upgrades.yml b/ansible/tasks/unattended-upgrades.yml index 3aae092..fa2e8f8 100644 --- a/ansible/tasks/unattended-upgrades.yml +++ b/ansible/tasks/unattended-upgrades.yml @@ -17,3 +17,4 @@ - name: Install unattended-upgrades-prometheus-collector apt: deb: https://github.com/FooBarWidget/unattended-upgrades-prometheus-collector/releases/download/v1.0.0/unattended-upgrades-prometheus-collector_1.0.0_all.deb + when: not (molecule_test | default(false)) diff --git a/ansible/templates/caddy-env.j2 b/ansible/templates/caddy-env.j2 index a72d4b8..c58852b 100644 --- a/ansible/templates/caddy-env.j2 +++ b/ansible/templates/caddy-env.j2 @@ -1,6 +1,11 @@ DOMAIN_NAME={{ domain_name }} AZURE_TENANT_ID={{ azure_tenant_id }} AZURE_SUBSCRIPTION_ID={{ azure_subscription_id }} +{% if molecule_test | default(false) %} +AZURE_DNS_UPDATER_CLIENT_ID=stub-client-id +AZURE_DNS_UPDATER_CLIENT_SECRET=stub-client-secret +{% else %} AZURE_DNS_UPDATER_CLIENT_ID={{ lookup('pipe', keyvault_get_dns_updater_client_id_command) }} AZURE_DNS_UPDATER_CLIENT_SECRET={{ lookup('pipe', keyvault_get_dns_updater_client_secret_command) }} +{% endif %} GCLOUD_BUCKET_PREFIX={{ gcloud_bucket_prefix }} From 8acb27eacc83bd06d2f1fba691c6b8ddd7412105 Mon Sep 17 00:00:00 2001 From: abtreece Date: Thu, 30 Apr 2026 14:54:54 -0500 Subject: [PATCH 2/9] fix(ansible): drop recurse:true from apiserver versions dir creation The "Create apiserver deployment directory" task used recurse:true with mode:0755. The recurse walked into vendor/bundle/ruby/*/bin/ (gem-bin shims) and the latest symlink. Both report mode 0777 from stat() because they are symlinks, and Linux cannot actually chmod a symlink, but Ansible still reports changed:true because requested mode != observed mode. Effect: every ansible-playbook run after the first apiserver release deploy has been reporting at least one task changed, forever. Masked by absence of an idempotence test in CI; surfaced by the new Molecule scenario. Release tarballs are extracted by the deployer running as itself, so ownership inside the tree is already correct. The recurse-chown was over-defensive. --- ansible/tasks/apiserver-deployer.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/tasks/apiserver-deployer.yml b/ansible/tasks/apiserver-deployer.yml index 8cca5ff..758456d 100644 --- a/ansible/tasks/apiserver-deployer.yml +++ b/ansible/tasks/apiserver-deployer.yml @@ -29,7 +29,6 @@ file: path: /opt/apiserver/versions state: directory - recurse: true owner: apiserver-deployer group: apiserver-deployer mode: 0755 From 5ef9e81940e0b3017ae756b78ad957165001b650 Mon Sep 17 00:00:00 2001 From: abtreece Date: Thu, 14 May 2026 23:04:26 -0500 Subject: [PATCH 3/9] fix(ansible): start existing stopped Lima instances in molecule create A previous create-only check on instance presence skipped the start step when 'ansible-molecule' was in any state (including Stopped), causing converge to SSH to a powered-off VM. Gate on status instead: create when absent, resume when present and not Running. --- ansible/molecule/default/create.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/ansible/molecule/default/create.yml b/ansible/molecule/default/create.yml index 23a0711..b97b053 100644 --- a/ansible/molecule/default/create.yml +++ b/ansible/molecule/default/create.yml @@ -3,17 +3,24 @@ connection: local gather_facts: false tasks: - - name: Check if instance already exists - ansible.builtin.command: limactl list --format '{{ "{{.Name}}" }}' ansible-molecule - register: limactl_list + - name: Check Lima instance status + ansible.builtin.command: limactl list --format '{{ "{{.Status}}" }}' ansible-molecule + register: limactl_status changed_when: false failed_when: false - - name: Start Lima instance + - name: Create Lima instance ansible.builtin.command: > limactl start --tty=false --name=ansible-molecule {{ molecule_scenario_directory }}/lima.yaml - when: "'ansible-molecule' not in limactl_list.stdout" + when: limactl_status.stdout | trim == '' + changed_when: true + + - name: Start existing Lima instance + ansible.builtin.command: limactl start --tty=false ansible-molecule + when: + - limactl_status.stdout | trim != '' + - limactl_status.stdout | trim != 'Running' changed_when: true - name: Get SSH config from Lima From 02c377d1fa6ac50c200e87cb2ea46b289e23afc7 Mon Sep 17 00:00:00 2001 From: abtreece Date: Thu, 14 May 2026 23:04:34 -0500 Subject: [PATCH 4/9] fix(ansible): surface real limactl delete errors in molecule destroy A blanket failed_when:false suppressed every limactl delete failure, not just the expected absent-instance case, so Molecule could report success while leaving a wedged VM behind. Gate the delete on existence so real delete failures (permissions, broken state) fail loudly. --- ansible/molecule/default/destroy.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ansible/molecule/default/destroy.yml b/ansible/molecule/default/destroy.yml index faa35d5..674238b 100644 --- a/ansible/molecule/default/destroy.yml +++ b/ansible/molecule/default/destroy.yml @@ -3,8 +3,13 @@ connection: local gather_facts: false tasks: + - name: Check if Lima instance exists + ansible.builtin.command: limactl list --format '{{ "{{.Name}}" }}' ansible-molecule + register: limactl_list + changed_when: false + failed_when: false + - name: Stop and delete Lima instance ansible.builtin.command: limactl delete --force ansible-molecule - register: result - changed_when: result.rc == 0 - failed_when: false + when: "'ansible-molecule' in limactl_list.stdout" + changed_when: true From c4feaddb553bf28d678b33a60312b69931c272a0 Mon Sep 17 00:00:00 2001 From: abtreece Date: Thu, 14 May 2026 23:08:34 -0500 Subject: [PATCH 5/9] fix(ansible): re-stage molecule fixture files on every converge The fixture staging block was gated on /opt/apiserver/versions/latest existing, so iterative 'molecule converge' runs never picked up edits to the fixture Gemfile/config.ru and never reran bundler. Stage the files unconditionally (copy is idempotent), gate bundler on Gemfile or config.ru changes (or missing Gemfile.lock), and gate the recursive chown on bundler having actually run. The local feedback loop now reflects fixture edits without re-creating the VM. --- ansible/molecule/default/converge.yml | 111 ++++++++++++++------------ 1 file changed, 58 insertions(+), 53 deletions(-) diff --git a/ansible/molecule/default/converge.yml b/ansible/molecule/default/converge.yml index 7d12cc9..7e8706b 100644 --- a/ansible/molecule/default/converge.yml +++ b/ansible/molecule/default/converge.yml @@ -47,66 +47,71 @@ - import_tasks: ../../tasks/apiserver.yml - - name: "[test] Check whether apiserver fixture is already staged" - ansible.builtin.stat: - path: /opt/apiserver/versions/latest - register: fixture_latest + - name: "[test] Stage apiserver service-contract fixture dir" + ansible.builtin.file: + path: /opt/apiserver/versions/fixture-1 + state: directory + owner: apiserver-deployer + group: apiserver-deployer + mode: "0755" - - name: "[test] Stage apiserver service-contract fixture (first run only)" - when: not fixture_latest.stat.exists - block: - - name: "[test] Stage apiserver service-contract fixture dir" - ansible.builtin.file: - path: /opt/apiserver/versions/fixture-1 - state: directory - owner: apiserver-deployer - group: apiserver-deployer - mode: "0755" + - name: "[test] Copy fixture Gemfile" + ansible.builtin.copy: + src: molecule-test/apiserver-fixture/Gemfile + dest: /opt/apiserver/versions/fixture-1/Gemfile + owner: apiserver-deployer + group: apiserver-deployer + mode: "0644" + register: fixture_gemfile - - name: "[test] Copy fixture Gemfile" - ansible.builtin.copy: - src: molecule-test/apiserver-fixture/Gemfile - dest: /opt/apiserver/versions/fixture-1/Gemfile - owner: apiserver-deployer - group: apiserver-deployer - mode: "0644" + - name: "[test] Copy fixture config.ru" + ansible.builtin.copy: + src: molecule-test/apiserver-fixture/config.ru + dest: /opt/apiserver/versions/fixture-1/config.ru + owner: apiserver-deployer + group: apiserver-deployer + mode: "0644" + register: fixture_configru - - name: "[test] Copy fixture config.ru" - ansible.builtin.copy: - src: molecule-test/apiserver-fixture/config.ru - dest: /opt/apiserver/versions/fixture-1/config.ru - owner: apiserver-deployer - group: apiserver-deployer - mode: "0644" + - name: "[test] Configure bundler path for fixture" + ansible.builtin.command: + cmd: bundle config set --local path vendor/bundle + chdir: /opt/apiserver/versions/fixture-1 + creates: /opt/apiserver/versions/fixture-1/.bundle/config - - name: "[test] Configure bundler path for fixture" - ansible.builtin.command: - cmd: bundle config set --local path vendor/bundle - chdir: /opt/apiserver/versions/fixture-1 - changed_when: true + - name: "[test] Check for fixture Gemfile.lock" + ansible.builtin.stat: + path: /opt/apiserver/versions/fixture-1/Gemfile.lock + register: fixture_lock - - name: "[test] Bundle install fixture deps as root" - ansible.builtin.command: - cmd: bundle install - chdir: /opt/apiserver/versions/fixture-1 - creates: /opt/apiserver/versions/fixture-1/Gemfile.lock + - name: "[test] Bundle install fixture deps as root" + ansible.builtin.command: + cmd: bundle install + chdir: /opt/apiserver/versions/fixture-1 + register: fixture_bundle + when: > + fixture_gemfile.changed + or fixture_configru.changed + or not fixture_lock.stat.exists + changed_when: true - - name: "[test] Chown fixture tree to apiserver-deployer" - ansible.builtin.file: - path: /opt/apiserver/versions/fixture-1 - state: directory - recurse: true - owner: apiserver-deployer - group: apiserver-deployer + - name: "[test] Chown fixture tree to apiserver-deployer" + ansible.builtin.file: + path: /opt/apiserver/versions/fixture-1 + state: directory + recurse: true + owner: apiserver-deployer + group: apiserver-deployer + when: fixture_bundle.changed - - name: "[test] Activate fixture as latest" - ansible.builtin.file: - src: fixture-1 - dest: /opt/apiserver/versions/latest - state: link - owner: apiserver-deployer - group: apiserver-deployer - force: true + - name: "[test] Activate fixture as latest" + ansible.builtin.file: + src: fixture-1 + dest: /opt/apiserver/versions/latest + state: link + owner: apiserver-deployer + group: apiserver-deployer + force: true handlers: - import_tasks: ../../handlers/ssh.yml From 73dfecf1b7d299e5f51a3f88bb543762ec69113a Mon Sep 17 00:00:00 2001 From: abtreece Date: Thu, 14 May 2026 23:10:12 -0500 Subject: [PATCH 6/9] docs(ansible): pin Molecule setup command to requirements.txt versions The setup command pulled latest Molecule/ansible-core/molecule-plugins, so README users could land on tool versions newer than requirements.txt and hit divergent Molecule behavior. Pin all three packages inline to match requirements.txt and call out the version-sync requirement. --- ansible/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible/README.md b/ansible/README.md index ecff392..f0ca8b2 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -13,9 +13,12 @@ cloud credentials. ``` brew install lima uv -uv tool install --with ansible-core --with molecule-plugins --python 3.12 molecule +uv tool install --with ansible-core==2.20.5 --with molecule-plugins==25.8.12 --python 3.12 molecule==26.4.0 ``` +Versions match `requirements.txt`, which is the canonical pin source for this +scenario. Bump both together when upgrading. + `uv tool install` puts each tool in its own isolated venv. Molecule shells out to `ansible`, `ansible-playbook`, `ansible-config`, etc., so the entire tool venv's `bin/` directory must be on PATH — adding only the `molecule` symlink From cd0e2474e6043d27ee8128f4f365135f1f490158 Mon Sep 17 00:00:00 2001 From: abtreece Date: Thu, 14 May 2026 23:14:35 -0500 Subject: [PATCH 7/9] fix(ansible): commit Gemfile.lock for molecule fixture and copy it onto the VM The fixture Gemfile resolved against rubygems on every fresh VM, so an upstream puma/sinatra/rackup release could break Molecule runs even when the playbook didn't change. Commit a Gemfile.lock locked to linux platforms (x86_64-linux, aarch64-linux), copy it alongside the Gemfile in converge, and re-gate bundle install on lockfile or Gemfile/config.ru changes (or missing vendor/bundle). Regenerate via 'bundle lock' from the fixture directory when intentionally bumping versions. --- .../apiserver-fixture/Gemfile.lock | 42 +++++++++++++++++++ ansible/molecule/default/converge.yml | 18 ++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 ansible/files/molecule-test/apiserver-fixture/Gemfile.lock diff --git a/ansible/files/molecule-test/apiserver-fixture/Gemfile.lock b/ansible/files/molecule-test/apiserver-fixture/Gemfile.lock new file mode 100644 index 0000000..575ecf8 --- /dev/null +++ b/ansible/files/molecule-test/apiserver-fixture/Gemfile.lock @@ -0,0 +1,42 @@ +GEM + remote: https://rubygems.org/ + specs: + base64 (0.3.0) + logger (1.7.0) + mustermann (3.1.1) + nio4r (2.7.5) + puma (8.0.1) + nio4r (~> 2.0) + rack (3.2.6) + rack-protection (4.2.1) + base64 (>= 0.1.0) + logger (>= 1.6.0) + rack (>= 3.0.0, < 4) + rack-session (2.1.2) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rackup (2.3.1) + rack (>= 3) + sinatra (4.2.1) + logger (>= 1.6.0) + mustermann (~> 3.0) + rack (>= 3.0.0, < 4) + rack-protection (= 4.2.1) + rack-session (>= 2.0.0, < 3) + tilt (~> 2.0) + tilt (2.7.0) + +PLATFORMS + aarch64-linux + x86_64-linux + +DEPENDENCIES + puma + rackup + sinatra (~> 4) + +RUBY VERSION + ruby 3.2.3p157 + +BUNDLED WITH + 2.4.19 diff --git a/ansible/molecule/default/converge.yml b/ansible/molecule/default/converge.yml index 7e8706b..82c7ff8 100644 --- a/ansible/molecule/default/converge.yml +++ b/ansible/molecule/default/converge.yml @@ -64,6 +64,15 @@ mode: "0644" register: fixture_gemfile + - name: "[test] Copy fixture Gemfile.lock" + ansible.builtin.copy: + src: molecule-test/apiserver-fixture/Gemfile.lock + dest: /opt/apiserver/versions/fixture-1/Gemfile.lock + owner: apiserver-deployer + group: apiserver-deployer + mode: "0644" + register: fixture_gemfile_lock + - name: "[test] Copy fixture config.ru" ansible.builtin.copy: src: molecule-test/apiserver-fixture/config.ru @@ -79,10 +88,10 @@ chdir: /opt/apiserver/versions/fixture-1 creates: /opt/apiserver/versions/fixture-1/.bundle/config - - name: "[test] Check for fixture Gemfile.lock" + - name: "[test] Check for fixture vendor/bundle" ansible.builtin.stat: - path: /opt/apiserver/versions/fixture-1/Gemfile.lock - register: fixture_lock + path: /opt/apiserver/versions/fixture-1/vendor/bundle + register: fixture_vendor - name: "[test] Bundle install fixture deps as root" ansible.builtin.command: @@ -91,8 +100,9 @@ register: fixture_bundle when: > fixture_gemfile.changed + or fixture_gemfile_lock.changed or fixture_configru.changed - or not fixture_lock.stat.exists + or not fixture_vendor.stat.exists changed_when: true - name: "[test] Chown fixture tree to apiserver-deployer" From f53ff1f728ff1f37fa37d6e2a9f4e2216aee7911 Mon Sep 17 00:00:00 2001 From: abtreece Date: Thu, 14 May 2026 23:16:08 -0500 Subject: [PATCH 8/9] fix(ansible): drop unreachable python3/sudo bootstrap from molecule prepare The raw task could not fulfill its stated purpose: gather_facts:true runs first and requires python3, and ansible_become:true routes the command through sudo - both prerequisites the task claimed to install. Lima's provision script (lima.yaml) already installs python3 and sudo before SSH is up, so the task was redundant in the working path and misleading in the failing path. --- ansible/molecule/default/prepare.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ansible/molecule/default/prepare.yml b/ansible/molecule/default/prepare.yml index cdc4600..93598d7 100644 --- a/ansible/molecule/default/prepare.yml +++ b/ansible/molecule/default/prepare.yml @@ -2,11 +2,6 @@ hosts: all gather_facts: true tasks: - - name: Ensure python3 and sudo are present - ansible.builtin.raw: | - which python3 || (apt-get update && apt-get install -y python3 sudo) - changed_when: false - - name: Install packages prod expects but playbook does not declare ansible.builtin.apt: name: From e272d91c4e890ccd7a84a6d62ce5923e409092c4 Mon Sep 17 00:00:00 2001 From: abtreece Date: Thu, 14 May 2026 23:16:51 -0500 Subject: [PATCH 9/9] feat(ansible): assert Caddy actually proxies /admin/* to apiserver socket Verify previously only confirmed Caddy was active, its config parsed, and the apiserver socket file existed - a permission or proxy regression (caddy user can't read the socket, reverse_proxy misconfigured) would still pass. The fixture exposes GET /admin/health returning 'ok', so add a uri request to http://127.0.0.1:8080/admin/health that exercises the full Caddy -> Unix socket -> apiserver path. --- ansible/molecule/default/verify.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml index 7c85f47..59f01e5 100644 --- a/ansible/molecule/default/verify.yml +++ b/ansible/molecule/default/verify.yml @@ -26,6 +26,14 @@ ansible.builtin.command: caddy validate --config /etc/caddy/Caddyfile --envfile /etc/caddy/env changed_when: false + - name: Caddy proxies /admin/* to the apiserver socket + ansible.builtin.uri: + url: http://127.0.0.1:8080/admin/health + return_content: true + status_code: 200 + register: admin_proxy + failed_when: admin_proxy.content != "ok" + - name: AppArmor package is installed ansible.builtin.package_facts: {}