feat(ansible): add Molecule + Lima testing for the Hetzner playbook

ansible/README.md

@@ -0,0 +1,98 @@
+# `infra/ansible/` — Hetzner VM playbook
+
+Configures `backend.fullstaqruby.org`. See `../docs/` for operational context.
+
+## Local testing with Molecule + Lima
+
+This playbook ships a Molecule scenario that converges the playbook against a
+Lima VM, with all live external dependencies (Azure Key Vault, GCS, GitHub
+releases) replaced by stubs so the scenario runs offline-friendly without
+cloud credentials.
+
+### One-time setup (macOS)
+
+```
+brew install lima uv
+uv tool install --with ansible-core==2.20.5 --with molecule-plugins==25.8.12 --python 3.12 molecule==26.4.0
+```
+
+Versions match `requirements.txt`, which is the canonical pin source for this
+scenario. Bump both together when upgrading.
+
+`uv tool install` puts each tool in its own isolated venv. Molecule shells out
+to `ansible`, `ansible-playbook`, `ansible-config`, etc., so the entire tool
+venv's `bin/` directory must be on PATH — adding only the `molecule` symlink
+isn't enough. Add this to your shell rc (or run per-session):
+
+```
+export PATH="$HOME/.local/share/uv/tools/molecule/bin:$PATH"
+```
+
+Install the Ansible collections the playbook depends on:
+
+```
+ansible-galaxy collection install community.general ansible.posix
+```
+
+### Daily loop
+
+From this directory (`infra/ansible/`):
+
+```
+molecule converge        # iterative; keeps VM alive between runs
+molecule verify          # run assertions against the converged VM
+molecule idempotence     # second-run check (changed=0)
+molecule destroy         # tear down the Lima VM
+molecule test            # full lifecycle: destroy → create → prepare → converge → idempotence → verify → destroy
+```
+
+Approximate timings on Apple Silicon (arm64 native guest):
+
+- VM boot: ~20s
+- First converge: ~3–5 min (apt installs + bundle install of fixture)
+- Subsequent converges: ~30–60s
+- Verify: ~10s
+
+### What the scenario does
+
+- Boots a Debian 12 cloud-image VM via Lima (`molecule/default/lima.yaml`)
+- Bootstraps packages the prod playbook assumes preinstalled (cron, ufw,
+  acl, rsyslog, ruby-dev, build-essential — see
+  `molecule/default/prepare.yml`)
+- Runs `main.yml`'s task list with `molecule_test: true`, which causes four
+  prod tasks to skip live external calls
+- Stages a service-contract fixture (minimal Sinatra app on Puma) at
+  `/opt/apiserver/versions/latest` so the apiserver systemd unit can start
+  successfully
+- Verifies that `caddy`, `prometheus`, `fail2ban`, `ssh`, and `apiserver`
+  are all `systemctl is-active`, that the apiserver Unix socket exists, that
+  `caddy validate` passes, and that SSH/UFW/unattended-upgrades configuration
+  matches expectations
+
+### Inspecting the VM
+
+```
+limactl shell ansible-molecule          # interactive shell on the VM
+limactl shell ansible-molecule sudo systemctl status caddy
+limactl shell ansible-molecule sudo journalctl -u apiserver
+```
+
+### Troubleshooting
+
+- **`No version is set for command molecule`** — your shell is using asdf
+  shims that shadow the uv-installed binaries. Make sure
+  `~/.local/share/uv/tools/molecule/bin` is *before* asdf's shim dir on PATH.
+- **`Could not find or access '<file>.j2'` or `<file>.sh`** — a `files/` or
+  `templates/` symlink in `molecule/default/` may be missing or broken; both
+  must point to `../../files` and `../../templates` respectively.
+- **`limactl: command not found`** — `brew install lima`.
+- **VM in a wedged state** — `molecule destroy` then re-run. As a last resort:
+  `limactl delete --force ansible-molecule`.
+
+### What is *not* tested by this scenario
+
+- Real ACME/TLS issuance (test Caddyfile uses `auto_https off`)
+- Real Azure Key Vault, GCS, or GitHub-release integration
+- OIDC JWT verification (covered by `../apiserver/` Ruby tests)
+- Live `fail2ban` banning behavior (only verifies the unit starts cleanly)
+- AppArmor profile loading (the playbook only installs the package today)

ansible/files/molecule-test/apiserver-fixture/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+ruby ">= 3.0"
+
+gem "puma"
+gem "sinatra", "~> 4"
+gem "rackup"

ansible/files/molecule-test/apiserver-fixture/Gemfile.lock

@@ -0,0 +1,42 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    base64 (0.3.0)
+    logger (1.7.0)
+    mustermann (3.1.1)
+    nio4r (2.7.5)
+    puma (8.0.1)
+      nio4r (~> 2.0)
+    rack (3.2.6)
+    rack-protection (4.2.1)
+      base64 (>= 0.1.0)
+      logger (>= 1.6.0)
+      rack (>= 3.0.0, < 4)
+    rack-session (2.1.2)
+      base64 (>= 0.1.0)
+      rack (>= 3.0.0)
+    rackup (2.3.1)
+      rack (>= 3)
+    sinatra (4.2.1)
+      logger (>= 1.6.0)
+      mustermann (~> 3.0)
+      rack (>= 3.0.0, < 4)
+      rack-protection (= 4.2.1)
+      rack-session (>= 2.0.0, < 3)
+      tilt (~> 2.0)
+    tilt (2.7.0)
+
+PLATFORMS
+  aarch64-linux
+  x86_64-linux
+
+DEPENDENCIES
+  puma
+  rackup
+  sinatra (~> 4)
+
+RUBY VERSION
+   ruby 3.2.3p157
+
+BUNDLED WITH
+   2.4.19

ansible/files/molecule-test/apiserver-fixture/config.ru

@@ -0,0 +1,9 @@
+require "sinatra/base"
+
+class FixtureApp < Sinatra::Base
+  get "/admin/health" do
+    "ok"
+  end
+end
+
+run FixtureApp

ansible/files/molecule-test/stub-apiserver-deployer

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# Test stub for apiserver-deployer. Real script downloads tarballs from
+# GitHub releases. Test fixture is staged separately by Molecule converge.
+set -eu
+echo "[stub-apiserver-deployer] no-op (fixture staged by Molecule)"
+exit 0

ansible/files/molecule-test/stub-query-latest-repo-versions

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Test stub for query-latest-repo-versions. Real script reads from GCS.
+# Writes static placeholder values into the file Caddy expects.
+set -eu
+OUT="${1:-/etc/caddy/env-repo-versions}"
+cat >"$OUT" <<EOF
+APT_LATEST_VERSION=stub-apt-1
+YUM_LATEST_VERSION=stub-yum-1
+REPO_QUERY_TIME=1970-01-01T00:00:00Z
+EOF

ansible/files/molecule-test/test-Caddyfile

@@ -0,0 +1,16 @@
+{
+	admin off
+	auto_https off
+	log {
+		format console
+	}
+}
+
+:8080 {
+	handle /admin/* {
+		reverse_proxy unix//run/apiserver/server.sock
+	}
+	handle {
+		respond "fullstaq-ruby molecule test"
+	}
+}

ansible/molecule/default/converge.yml

@@ -0,0 +1,132 @@
+- name: Converge
+  hosts: all
+  vars:
+    molecule_test: true
+  vars_files:
+    - ../../vars/main.yml
+    - ../../vars/terraform.yml
+  tasks:
+    - import_tasks: ../../tasks/essentials.yml
+    - import_tasks: ../../tasks/ssh.yml
+    - import_tasks: ../../tasks/fail2ban.yml
+    - import_tasks: ../../tasks/apparmor.yml
+    - import_tasks: ../../tasks/autoreboot.yml
+    - import_tasks: ../../tasks/unattended-upgrades.yml
+    - import_tasks: ../../tasks/ufw.yml
+    - import_tasks: ../../tasks/prometheus.yml
+    - import_tasks: ../../tasks/caddy.yml
+
+    - name: "[test] Install stub query-latest-repo-versions"
+      ansible.builtin.copy:
+        src: molecule-test/stub-query-latest-repo-versions
+        dest: /usr/local/bin/query-latest-repo-versions
+        owner: root
+        group: root
+        mode: "0755"
+      notify: Restart Caddy
+
+    - name: "[test] Install test Caddyfile"
+      ansible.builtin.copy:
+        src: molecule-test/test-Caddyfile
+        dest: /etc/caddy/Caddyfile
+        owner: caddy
+        group: caddy
+        mode: "0644"
+      notify: Restart Caddy
+
+    - import_tasks: ../../tasks/apiserver-deployer.yml
+
+    - name: "[test] Install stub apiserver-deployer"
+      ansible.builtin.copy:
+        src: molecule-test/stub-apiserver-deployer
+        dest: /usr/local/bin/apiserver-deployer
+        owner: root
+        group: root
+        mode: "0755"
+      notify: Restart apiserver-deployer
+
+    - import_tasks: ../../tasks/apiserver.yml
+
+    - name: "[test] Stage apiserver service-contract fixture dir"
+      ansible.builtin.file:
+        path: /opt/apiserver/versions/fixture-1
+        state: directory
+        owner: apiserver-deployer
+        group: apiserver-deployer
+        mode: "0755"
+
+    - name: "[test] Copy fixture Gemfile"
+      ansible.builtin.copy:
+        src: molecule-test/apiserver-fixture/Gemfile
+        dest: /opt/apiserver/versions/fixture-1/Gemfile
+        owner: apiserver-deployer
+        group: apiserver-deployer
+        mode: "0644"
+      register: fixture_gemfile
+
+    - name: "[test] Copy fixture Gemfile.lock"
+      ansible.builtin.copy:
+        src: molecule-test/apiserver-fixture/Gemfile.lock
+        dest: /opt/apiserver/versions/fixture-1/Gemfile.lock
+        owner: apiserver-deployer
+        group: apiserver-deployer
+        mode: "0644"
+      register: fixture_gemfile_lock
+
+    - name: "[test] Copy fixture config.ru"
+      ansible.builtin.copy:
+        src: molecule-test/apiserver-fixture/config.ru
+        dest: /opt/apiserver/versions/fixture-1/config.ru
+        owner: apiserver-deployer
+        group: apiserver-deployer
+        mode: "0644"
+      register: fixture_configru
+
+    - name: "[test] Configure bundler path for fixture"
+      ansible.builtin.command:
+        cmd: bundle config set --local path vendor/bundle
+        chdir: /opt/apiserver/versions/fixture-1
+        creates: /opt/apiserver/versions/fixture-1/.bundle/config
+
+    - name: "[test] Check for fixture vendor/bundle"
+      ansible.builtin.stat:
+        path: /opt/apiserver/versions/fixture-1/vendor/bundle
+      register: fixture_vendor
+
+    - name: "[test] Bundle install fixture deps as root"
+      ansible.builtin.command:
+        cmd: bundle install
+        chdir: /opt/apiserver/versions/fixture-1
+      register: fixture_bundle
+      when: >
+        fixture_gemfile.changed
+        or fixture_gemfile_lock.changed
+        or fixture_configru.changed
+        or not fixture_vendor.stat.exists
+      changed_when: true
+
+    - name: "[test] Chown fixture tree to apiserver-deployer"
+      ansible.builtin.file:
+        path: /opt/apiserver/versions/fixture-1
+        state: directory
+        recurse: true
+        owner: apiserver-deployer
+        group: apiserver-deployer
+      when: fixture_bundle.changed
+
+    - name: "[test] Activate fixture as latest"
+      ansible.builtin.file:
+        src: fixture-1
+        dest: /opt/apiserver/versions/latest
+        state: link
+        owner: apiserver-deployer
+        group: apiserver-deployer
+        force: true
+
+  handlers:
+    - import_tasks: ../../handlers/ssh.yml
+    - import_tasks: ../../handlers/systemd.yml
+    - import_tasks: ../../handlers/apiserver.yml
+    - import_tasks: ../../handlers/apiserver-deployer.yml
+    - import_tasks: ../../handlers/caddy.yml
+    - import_tasks: ../../handlers/prometheus.yml

ansible/molecule/default/create.yml

@@ -0,0 +1,50 @@
+- name: Create Lima VM
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+    - name: Check Lima instance status
+      ansible.builtin.command: limactl list --format '{{ "{{.Status}}" }}' ansible-molecule
+      register: limactl_status
+      changed_when: false
+      failed_when: false
+
+    - name: Create Lima instance
+      ansible.builtin.command: >
+        limactl start --tty=false --name=ansible-molecule
+        {{ molecule_scenario_directory }}/lima.yaml
+      when: limactl_status.stdout | trim == ''
+      changed_when: true
+
+    - name: Start existing Lima instance
+      ansible.builtin.command: limactl start --tty=false ansible-molecule
+      when:
+        - limactl_status.stdout | trim != ''
+        - limactl_status.stdout | trim != 'Running'
+      changed_when: true
+
+    - name: Get SSH config from Lima
+      ansible.builtin.command: limactl show-ssh --format=config ansible-molecule
+      register: lima_ssh_config
+      changed_when: false
+
+    - name: Write Lima SSH config to ephemeral dir
+      ansible.builtin.copy:
+        content: "{{ lima_ssh_config.stdout }}\n"
+        dest: "{{ molecule_ephemeral_directory }}/ssh_config"
+        mode: "0600"
+
+    - name: Update inventory with SSH config
+      ansible.builtin.copy:
+        dest: "{{ molecule_ephemeral_directory }}/inventory/hosts.yml"
+        mode: "0644"
+        content: |
+          all:
+            hosts:
+              ansible-molecule:
+                ansible_connection: ssh
+                ansible_host: lima-ansible-molecule
+                ansible_user: "{{ lookup('env', 'USER') }}"
+                ansible_ssh_common_args: "-F {{ molecule_ephemeral_directory }}/ssh_config"
+                ansible_become: true
+                ansible_python_interpreter: /usr/bin/python3

ansible/molecule/default/destroy.yml

@@ -0,0 +1,15 @@
+- name: Destroy Lima VM
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  tasks:
+    - name: Check if Lima instance exists
+      ansible.builtin.command: limactl list --format '{{ "{{.Name}}" }}' ansible-molecule
+      register: limactl_list
+      changed_when: false
+      failed_when: false
+
+    - name: Stop and delete Lima instance
+      ansible.builtin.command: limactl delete --force ansible-molecule
+      when: "'ansible-molecule' in limactl_list.stdout"
+      changed_when: true

ansible/molecule/default/files

@@ -0,0 +1 @@
+../../files