diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 0000000..f0ca8b2 --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,98 @@ +# `infra/ansible/` — Hetzner VM playbook + +Configures `backend.fullstaqruby.org`. See `../docs/` for operational context. + +## Local testing with Molecule + Lima + +This playbook ships a Molecule scenario that converges the playbook against a +Lima VM, with all live external dependencies (Azure Key Vault, GCS, GitHub +releases) replaced by stubs so the scenario runs offline-friendly without +cloud credentials. + +### One-time setup (macOS) + +``` +brew install lima uv +uv tool install --with ansible-core==2.20.5 --with molecule-plugins==25.8.12 --python 3.12 molecule==26.4.0 +``` + +Versions match `requirements.txt`, which is the canonical pin source for this +scenario. Bump both together when upgrading. + +`uv tool install` puts each tool in its own isolated venv. Molecule shells out +to `ansible`, `ansible-playbook`, `ansible-config`, etc., so the entire tool +venv's `bin/` directory must be on PATH — adding only the `molecule` symlink +isn't enough. Add this to your shell rc (or run per-session): + +``` +export PATH="$HOME/.local/share/uv/tools/molecule/bin:$PATH" +``` + +Install the Ansible collections the playbook depends on: + +``` +ansible-galaxy collection install community.general ansible.posix +``` + +### Daily loop + +From this directory (`infra/ansible/`): + +``` +molecule converge # iterative; keeps VM alive between runs +molecule verify # run assertions against the converged VM +molecule idempotence # second-run check (changed=0) +molecule destroy # tear down the Lima VM +molecule test # full lifecycle: destroy → create → prepare → converge → idempotence → verify → destroy +``` + +Approximate timings on Apple Silicon (arm64 native guest): + +- VM boot: ~20s +- First converge: ~3–5 min (apt installs + bundle install of fixture) +- Subsequent converges: ~30–60s +- Verify: ~10s + +### What the scenario does + +- Boots a Debian 12 cloud-image VM via Lima (`molecule/default/lima.yaml`) +- Bootstraps packages the prod playbook assumes preinstalled (cron, ufw, + acl, rsyslog, ruby-dev, build-essential — see + `molecule/default/prepare.yml`) +- Runs `main.yml`'s task list with `molecule_test: true`, which causes four + prod tasks to skip live external calls +- Stages a service-contract fixture (minimal Sinatra app on Puma) at + `/opt/apiserver/versions/latest` so the apiserver systemd unit can start + successfully +- Verifies that `caddy`, `prometheus`, `fail2ban`, `ssh`, and `apiserver` + are all `systemctl is-active`, that the apiserver Unix socket exists, that + `caddy validate` passes, and that SSH/UFW/unattended-upgrades configuration + matches expectations + +### Inspecting the VM + +``` +limactl shell ansible-molecule # interactive shell on the VM +limactl shell ansible-molecule sudo systemctl status caddy +limactl shell ansible-molecule sudo journalctl -u apiserver +``` + +### Troubleshooting + +- **`No version is set for command molecule`** — your shell is using asdf + shims that shadow the uv-installed binaries. Make sure + `~/.local/share/uv/tools/molecule/bin` is *before* asdf's shim dir on PATH. +- **`Could not find or access '.j2'` or `.sh`** — a `files/` or + `templates/` symlink in `molecule/default/` may be missing or broken; both + must point to `../../files` and `../../templates` respectively. +- **`limactl: command not found`** — `brew install lima`. +- **VM in a wedged state** — `molecule destroy` then re-run. As a last resort: + `limactl delete --force ansible-molecule`. + +### What is *not* tested by this scenario + +- Real ACME/TLS issuance (test Caddyfile uses `auto_https off`) +- Real Azure Key Vault, GCS, or GitHub-release integration +- OIDC JWT verification (covered by `../apiserver/` Ruby tests) +- Live `fail2ban` banning behavior (only verifies the unit starts cleanly) +- AppArmor profile loading (the playbook only installs the package today) diff --git a/ansible/files/molecule-test/apiserver-fixture/Gemfile b/ansible/files/molecule-test/apiserver-fixture/Gemfile new file mode 100644 index 0000000..d0333d1 --- /dev/null +++ b/ansible/files/molecule-test/apiserver-fixture/Gemfile @@ -0,0 +1,7 @@ +source "https://rubygems.org" + +ruby ">= 3.0" + +gem "puma" +gem "sinatra", "~> 4" +gem "rackup" diff --git a/ansible/files/molecule-test/apiserver-fixture/Gemfile.lock b/ansible/files/molecule-test/apiserver-fixture/Gemfile.lock new file mode 100644 index 0000000..575ecf8 --- /dev/null +++ b/ansible/files/molecule-test/apiserver-fixture/Gemfile.lock @@ -0,0 +1,42 @@ +GEM + remote: https://rubygems.org/ + specs: + base64 (0.3.0) + logger (1.7.0) + mustermann (3.1.1) + nio4r (2.7.5) + puma (8.0.1) + nio4r (~> 2.0) + rack (3.2.6) + rack-protection (4.2.1) + base64 (>= 0.1.0) + logger (>= 1.6.0) + rack (>= 3.0.0, < 4) + rack-session (2.1.2) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rackup (2.3.1) + rack (>= 3) + sinatra (4.2.1) + logger (>= 1.6.0) + mustermann (~> 3.0) + rack (>= 3.0.0, < 4) + rack-protection (= 4.2.1) + rack-session (>= 2.0.0, < 3) + tilt (~> 2.0) + tilt (2.7.0) + +PLATFORMS + aarch64-linux + x86_64-linux + +DEPENDENCIES + puma + rackup + sinatra (~> 4) + +RUBY VERSION + ruby 3.2.3p157 + +BUNDLED WITH + 2.4.19 diff --git a/ansible/files/molecule-test/apiserver-fixture/config.ru b/ansible/files/molecule-test/apiserver-fixture/config.ru new file mode 100644 index 0000000..24b775d --- /dev/null +++ b/ansible/files/molecule-test/apiserver-fixture/config.ru @@ -0,0 +1,9 @@ +require "sinatra/base" + +class FixtureApp < Sinatra::Base + get "/admin/health" do + "ok" + end +end + +run FixtureApp diff --git a/ansible/files/molecule-test/stub-apiserver-deployer b/ansible/files/molecule-test/stub-apiserver-deployer new file mode 100644 index 0000000..36d60cd --- /dev/null +++ b/ansible/files/molecule-test/stub-apiserver-deployer @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Test stub for apiserver-deployer. Real script downloads tarballs from +# GitHub releases. Test fixture is staged separately by Molecule converge. +set -eu +echo "[stub-apiserver-deployer] no-op (fixture staged by Molecule)" +exit 0 diff --git a/ansible/files/molecule-test/stub-query-latest-repo-versions b/ansible/files/molecule-test/stub-query-latest-repo-versions new file mode 100644 index 0000000..d69ed38 --- /dev/null +++ b/ansible/files/molecule-test/stub-query-latest-repo-versions @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Test stub for query-latest-repo-versions. Real script reads from GCS. +# Writes static placeholder values into the file Caddy expects. +set -eu +OUT="${1:-/etc/caddy/env-repo-versions}" +cat >"$OUT" < + fixture_gemfile.changed + or fixture_gemfile_lock.changed + or fixture_configru.changed + or not fixture_vendor.stat.exists + changed_when: true + + - name: "[test] Chown fixture tree to apiserver-deployer" + ansible.builtin.file: + path: /opt/apiserver/versions/fixture-1 + state: directory + recurse: true + owner: apiserver-deployer + group: apiserver-deployer + when: fixture_bundle.changed + + - name: "[test] Activate fixture as latest" + ansible.builtin.file: + src: fixture-1 + dest: /opt/apiserver/versions/latest + state: link + owner: apiserver-deployer + group: apiserver-deployer + force: true + + handlers: + - import_tasks: ../../handlers/ssh.yml + - import_tasks: ../../handlers/systemd.yml + - import_tasks: ../../handlers/apiserver.yml + - import_tasks: ../../handlers/apiserver-deployer.yml + - import_tasks: ../../handlers/caddy.yml + - import_tasks: ../../handlers/prometheus.yml diff --git a/ansible/molecule/default/create.yml b/ansible/molecule/default/create.yml new file mode 100644 index 0000000..b97b053 --- /dev/null +++ b/ansible/molecule/default/create.yml @@ -0,0 +1,50 @@ +- name: Create Lima VM + hosts: localhost + connection: local + gather_facts: false + tasks: + - name: Check Lima instance status + ansible.builtin.command: limactl list --format '{{ "{{.Status}}" }}' ansible-molecule + register: limactl_status + changed_when: false + failed_when: false + + - name: Create Lima instance + ansible.builtin.command: > + limactl start --tty=false --name=ansible-molecule + {{ molecule_scenario_directory }}/lima.yaml + when: limactl_status.stdout | trim == '' + changed_when: true + + - name: Start existing Lima instance + ansible.builtin.command: limactl start --tty=false ansible-molecule + when: + - limactl_status.stdout | trim != '' + - limactl_status.stdout | trim != 'Running' + changed_when: true + + - name: Get SSH config from Lima + ansible.builtin.command: limactl show-ssh --format=config ansible-molecule + register: lima_ssh_config + changed_when: false + + - name: Write Lima SSH config to ephemeral dir + ansible.builtin.copy: + content: "{{ lima_ssh_config.stdout }}\n" + dest: "{{ molecule_ephemeral_directory }}/ssh_config" + mode: "0600" + + - name: Update inventory with SSH config + ansible.builtin.copy: + dest: "{{ molecule_ephemeral_directory }}/inventory/hosts.yml" + mode: "0644" + content: | + all: + hosts: + ansible-molecule: + ansible_connection: ssh + ansible_host: lima-ansible-molecule + ansible_user: "{{ lookup('env', 'USER') }}" + ansible_ssh_common_args: "-F {{ molecule_ephemeral_directory }}/ssh_config" + ansible_become: true + ansible_python_interpreter: /usr/bin/python3 diff --git a/ansible/molecule/default/destroy.yml b/ansible/molecule/default/destroy.yml new file mode 100644 index 0000000..674238b --- /dev/null +++ b/ansible/molecule/default/destroy.yml @@ -0,0 +1,15 @@ +- name: Destroy Lima VM + hosts: localhost + connection: local + gather_facts: false + tasks: + - name: Check if Lima instance exists + ansible.builtin.command: limactl list --format '{{ "{{.Name}}" }}' ansible-molecule + register: limactl_list + changed_when: false + failed_when: false + + - name: Stop and delete Lima instance + ansible.builtin.command: limactl delete --force ansible-molecule + when: "'ansible-molecule' in limactl_list.stdout" + changed_when: true diff --git a/ansible/molecule/default/files b/ansible/molecule/default/files new file mode 120000 index 0000000..81016f4 --- /dev/null +++ b/ansible/molecule/default/files @@ -0,0 +1 @@ +../../files \ No newline at end of file diff --git a/ansible/molecule/default/lima.yaml b/ansible/molecule/default/lima.yaml new file mode 100644 index 0000000..7759382 --- /dev/null +++ b/ansible/molecule/default/lima.yaml @@ -0,0 +1,18 @@ +images: + - location: "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-arm64.qcow2" + arch: "aarch64" + - location: "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2" + arch: "x86_64" +cpus: 2 +memory: "2GiB" +disk: "20GiB" +ssh: + loadDotSSHPubKeys: true +mounts: [] +provision: + - mode: system + script: | + #!/bin/sh + set -eux + apt-get update + apt-get install -y python3 sudo diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml new file mode 100644 index 0000000..8e574e4 --- /dev/null +++ b/ansible/molecule/default/molecule.yml @@ -0,0 +1,38 @@ +dependency: + name: galaxy +driver: + name: default + options: + managed: false + login_cmd_template: "limactl shell {instance}" + ansible_connection_options: + ansible_connection: ssh +platforms: + - name: ansible-molecule +provisioner: + name: ansible + config_options: + defaults: + forks: 5 + inventory: + hosts: + all: + hosts: + ansible-molecule: + ansible_user: lima-ansible-molecule + ansible_host: 127.0.0.1 + ansible_become: true + ansible_python_interpreter: /usr/bin/python3 +verifier: + name: ansible +scenario: + test_sequence: + - dependency + - destroy + - syntax + - create + - prepare + - converge + - idempotence + - verify + - destroy diff --git a/ansible/molecule/default/prepare.yml b/ansible/molecule/default/prepare.yml new file mode 100644 index 0000000..93598d7 --- /dev/null +++ b/ansible/molecule/default/prepare.yml @@ -0,0 +1,16 @@ +- name: Prepare guest + hosts: all + gather_facts: true + tasks: + - name: Install packages prod expects but playbook does not declare + ansible.builtin.apt: + name: + - cron + - ufw + - lsb-release + - acl + - ruby-dev + - build-essential + - rsyslog + update_cache: true + cache_valid_time: 3600 diff --git a/ansible/molecule/default/templates b/ansible/molecule/default/templates new file mode 120000 index 0000000..07531b7 --- /dev/null +++ b/ansible/molecule/default/templates @@ -0,0 +1 @@ +../../templates \ No newline at end of file diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml new file mode 100644 index 0000000..59f01e5 --- /dev/null +++ b/ansible/molecule/default/verify.yml @@ -0,0 +1,61 @@ +- name: Verify + hosts: all + gather_facts: true + tasks: + - name: Core services are active + ansible.builtin.command: systemctl is-active {{ item }} + loop: + - caddy + - prometheus + - fail2ban + - ssh + - apiserver + changed_when: false + + - name: Apiserver socket exists + ansible.builtin.stat: + path: /run/apiserver/server.sock + register: apiserver_socket + failed_when: not apiserver_socket.stat.exists + + - name: Apiserver-deployer one-shot is active (RemainAfterExit) + ansible.builtin.command: systemctl is-active apiserver-deployer + changed_when: false + + - name: Caddy config validates + ansible.builtin.command: caddy validate --config /etc/caddy/Caddyfile --envfile /etc/caddy/env + changed_when: false + + - name: Caddy proxies /admin/* to the apiserver socket + ansible.builtin.uri: + url: http://127.0.0.1:8080/admin/health + return_content: true + status_code: 200 + register: admin_proxy + failed_when: admin_proxy.content != "ok" + + - name: AppArmor package is installed + ansible.builtin.package_facts: {} + + - name: Assert AppArmor is installed + ansible.builtin.assert: + that: "'apparmor' in ansible_facts.packages" + + - name: UFW is active + ansible.builtin.command: ufw status verbose + register: ufw_status + changed_when: false + failed_when: "'Status: active' not in ufw_status.stdout" + + - name: SSH config matches hardening expectations + ansible.builtin.command: sshd -T + register: sshd_config + changed_when: false + + - name: Assert PermitRootLogin without-password + ansible.builtin.assert: + that: "'permitrootlogin without-password' in sshd_config.stdout" + + - name: Unattended-upgrade dry-run succeeds + ansible.builtin.command: unattended-upgrade --dry-run + changed_when: false diff --git a/ansible/requirements.txt b/ansible/requirements.txt new file mode 100644 index 0000000..af047da --- /dev/null +++ b/ansible/requirements.txt @@ -0,0 +1,3 @@ +ansible-core==2.20.5 +molecule==26.4.0 +molecule-plugins==25.8.12 diff --git a/ansible/tasks/apiserver-deployer.yml b/ansible/tasks/apiserver-deployer.yml index 3f42f21..758456d 100644 --- a/ansible/tasks/apiserver-deployer.yml +++ b/ansible/tasks/apiserver-deployer.yml @@ -29,7 +29,6 @@ file: path: /opt/apiserver/versions state: directory - recurse: true owner: apiserver-deployer group: apiserver-deployer mode: 0755 @@ -41,6 +40,7 @@ owner: root group: root mode: 0755 + when: not (molecule_test | default(false)) notify: Restart apiserver-deployer - name: Install apiserver-deployer systemd unit diff --git a/ansible/tasks/caddy.yml b/ansible/tasks/caddy.yml index 4f96114..363ac1d 100644 --- a/ansible/tasks/caddy.yml +++ b/ansible/tasks/caddy.yml @@ -35,6 +35,7 @@ owner: root group: root mode: 0755 + when: not (molecule_test | default(false)) notify: Restart Caddy - name: Create Caddy config dir @@ -87,6 +88,7 @@ owner: caddy group: caddy mode: 0644 + when: not (molecule_test | default(false)) notify: Restart Caddy - name: Create Caddy systemd service diff --git a/ansible/tasks/unattended-upgrades.yml b/ansible/tasks/unattended-upgrades.yml index 3aae092..fa2e8f8 100644 --- a/ansible/tasks/unattended-upgrades.yml +++ b/ansible/tasks/unattended-upgrades.yml @@ -17,3 +17,4 @@ - name: Install unattended-upgrades-prometheus-collector apt: deb: https://github.com/FooBarWidget/unattended-upgrades-prometheus-collector/releases/download/v1.0.0/unattended-upgrades-prometheus-collector_1.0.0_all.deb + when: not (molecule_test | default(false)) diff --git a/ansible/templates/caddy-env.j2 b/ansible/templates/caddy-env.j2 index a72d4b8..c58852b 100644 --- a/ansible/templates/caddy-env.j2 +++ b/ansible/templates/caddy-env.j2 @@ -1,6 +1,11 @@ DOMAIN_NAME={{ domain_name }} AZURE_TENANT_ID={{ azure_tenant_id }} AZURE_SUBSCRIPTION_ID={{ azure_subscription_id }} +{% if molecule_test | default(false) %} +AZURE_DNS_UPDATER_CLIENT_ID=stub-client-id +AZURE_DNS_UPDATER_CLIENT_SECRET=stub-client-secret +{% else %} AZURE_DNS_UPDATER_CLIENT_ID={{ lookup('pipe', keyvault_get_dns_updater_client_id_command) }} AZURE_DNS_UPDATER_CLIENT_SECRET={{ lookup('pipe', keyvault_get_dns_updater_client_secret_command) }} +{% endif %} GCLOUD_BUCKET_PREFIX={{ gcloud_bucket_prefix }}