diff --git a/.env.example b/.env.example index c2116b8..cbae2e6 100644 --- a/.env.example +++ b/.env.example @@ -1,10 +1,23 @@ -# Domain Configuration -DOMAIN=ssh.example.com +# Domain Configuration: the regional tunnel endpoint this box serves, +# e.g. uswest1.tunnels.cde.glueopshosted.com +DOMAIN=uswest1.tunnels.cde.glueopshosted.com + +# Source-IP allowlist for TUNNEL CREATION, comma-separated CIDRs (single +# IPs as /32). Enforced by the authenticator on SSH auth only — browsers +# and CloudFront (:80/:443) are never affected. Default: all RFC 1918 +# private ranges plus the CGNAT range 100.64.0.0/10 (Tailscale), i.e. only +# VMs on private networks or the tailnet can register/connect tunnels. If +# VMs arrive from public addresses, add those. Empty value = no source +# restriction. A malformed CIDR fails the authenticator at startup rather +# than silently allowing. +TUNNEL_ALLOWED_CIDRS=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/10 # Let's Encrypt ACME Email (required for certificate notifications) ACME_EMAIL=admin@example.com # AWS IAM Credentials for Route53 DNS01 Challenge # Required permissions: route53:ListHostedZones, route53:GetChange, route53:ChangeResourceRecordSets +# Use the acme-dns01-cde.glueopshosted.com user — the cde_acme_* outputs of +# glueops-opentofu-workspaces/aws-cloud-development-environment-assets-production. AWS_ACCESS_KEY_ID=your_access_key_id AWS_SECRET_ACCESS_KEY=your_secret_access_key diff --git a/MIGRATION.md b/MIGRATION.md deleted file mode 100644 index 8bd352d..0000000 --- a/MIGRATION.md +++ /dev/null @@ -1,127 +0,0 @@ -# Migrating an existing sish host to this repo - -This is a one-time runbook for moving a server that runs the **old -`glueops/sish` fork** stack (cloned as `~/sish/glueops-docker-compose`) onto this -repo's upstream-image stack. - -The migration is really just **copy runtime state into the new repo, then swap -stacks**. Nothing in the running setup is in git — the parts that matter are: - -- `sish_users/` — the trust-on-first-use key database. **Irreplaceable**: losing - it locks every user out until they re-register. -- `.env` — domain, ACME email, AWS credentials. -- `ssl/` — the current live certificate sish serves. -- the `letsencrypt` Docker volume — certbot's ACME account + renewal state. - -> [!WARNING] -> Never run `docker compose down -v` against the old stack — `-v` deletes the -> `letsencrypt` volume. Plain `down` keeps all volumes. - -Run everything on the server. Adjust paths if your old checkout lives elsewhere. - -## 0. Back up the irreplaceable data first - -```bash -cd ~ -tar czf sish-backup-$(date +%F-%H%M).tgz \ - -C ~/sish/glueops-docker-compose sish_users ssl .env backup-keys -ls -lh sish-backup-*.tgz -``` - -## 1. Stop the old stack (keeps data + volumes) - -```bash -cd ~/sish/glueops-docker-compose -docker compose down # frees ports 80/443/2222; keeps sish_users, ssl, volume -``` - -## 2. Clone this repo - -```bash -cd ~ && git clone https://github.com/GlueOps/cde-sish-tunnels.git -``` - -## 3. Copy runtime state into the new repo - -```bash -OLD=~/sish/glueops-docker-compose -NEW=~/cde-sish-tunnels - -cp "$OLD/.env" "$NEW/.env" -cp -a "$OLD/sish_users/." "$NEW/sish_users/" # the TOFU key DB — critical -cp -a "$OLD/ssl/." "$NEW/ssl/" # current live certs -cp -a "$OLD/backup-keys" "$NEW/" # not used by the stack; kept for safety - -# sanity: same number of users carried over, env vars present -echo "old: $(ls "$OLD/sish_users" | wc -l) new: $(ls "$NEW/sish_users" | wc -l)" -grep -E 'DOMAIN|ACME_EMAIL|AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY' "$NEW/.env" -``` - -The new compose uses the same env-var names, so `.env` drops in unchanged. - -## 4. Migrate the certbot volume (recommended) - -Carries over the ACME account and renewal state so the new stack does **not** -request a brand-new certificate on first boot. - -```bash -docker volume ls | grep letsencrypt # confirm source (expected: glueops-docker-compose_letsencrypt) - -docker volume create cde-sish-tunnels_letsencrypt -docker run --rm \ - -v glueops-docker-compose_letsencrypt:/from:ro \ - -v cde-sish-tunnels_letsencrypt:/to \ - alpine sh -c 'cp -a /from/. /to/' -``` - -Skippable: if omitted, certbot just requests a fresh cert via DNS-01 on first -boot. That works, but counts against Let's Encrypt rate limits if you redeploy -repeatedly. Either way tunnels stay up, because `ssl/` already holds a valid cert -that sish serves immediately. - -## 5. Start the new stack - -```bash -cd ~/cde-sish-tunnels -docker compose up -d --build -``` - -## 6. Verify - -```bash -docker compose ps # expect 3 services up -docker compose logs sish | tail -30 # binds :2222/:80/:443, loads certs from /ssl -docker compose logs authenticator | tail -20 -docker compose images sish # confirm it's ghcr.io/antoniomika/sish now - -# cert check (replace the domain) -echo | openssl s_client -connect :443 -servername x. 2>/dev/null \ - | openssl x509 -noout -subject -dates -``` - -Then prove the TOFU DB migrated: have an **existing** user connect with their -existing key — they should be allowed without re-registering. - -```bash -# from a client machine -ssh -p 2222 -R test:80:localhost:8080 -``` - -## 7. Rollback - -The old checkout is untouched, so reverting is instant: - -```bash -cd ~/cde-sish-tunnels && docker compose down -cd ~/sish/glueops-docker-compose && docker compose up -d -``` - -> Never run both stacks at once — they bind the same host ports. Always `down` -> one before `up` the other. - -## 8. Clean up (only once confident, e.g. the next day) - -```bash -docker volume rm glueops-docker-compose_letsencrypt # old certbot volume -# rm -rf ~/sish # old repo + fork, if desired -``` diff --git a/README.md b/README.md index 7644c6c..3fffe01 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,42 @@ sish watches that directory and reloads automatically. Renewal is checked every The IAM credentials need: `route53:ListHostedZones`, `route53:GetChange`, `route53:ChangeResourceRecordSets`. +## Deployment model: one box per datacenter region + +This repo deploys **regional** tunnel servers only: one instance per +datacenter, each serving `DOMAIN=.tunnels.cde.glueopshosted.com`. +Subdomains are bound literally (`--force-requested-subdomains`): a codespace +VM binds its own hostname, so URLs are +`https://..tunnels.cde.glueopshosted.com` with no prefix, +and a taken name fails the bind rather than silently going random. AWS creds +come from the `acme-dns01-cde.glueopshosted.com` IAM user (the `cde_acme_*` +outputs of the CDE assets workspace). + +### Restricting who can create tunnels + +`TUNNEL_ALLOWED_CIDRS` (comma-separated CIDRs, default: all RFC 1918 private +ranges plus `100.64.0.0/10` — the CGNAT range Tailscale uses) is enforced by +the **authenticator** on every SSH auth attempt using the client address sish +reports — so it restricts tunnel creation only. Browsers and CloudFront +(:80/:443) never touch the authenticator and are unaffected. Outside-the-list +clients get a logged 403 (`denied (source ... not allowlisted)`) and can +never register or connect a tunnel. Empty value disables the restriction; a +malformed CIDR fails the authenticator at startup. + +Deliberately NOT used: sish's own `--whitelisted-ips`, which is global across +SSH *and* HTTP/S and would block public browsers. The :2222 publish is +IPv4-only because docker-proxy rewrites IPv6 clients' sources to the bridge +gateway (a 172.x address inside the allowlist) — an unqualified publish would +let external IPv6 clients through the check. + +Regional boxes sit behind a per-region CloudFront distribution: browsers hit +`https://..tunnels.cde.glueopshosted.com` via the CDN, which +origin-fetches this box as `origin..tunnels.cde.glueopshosted.com` — +covered by the same wildcard cert, no extra config here. The box's 443 stays +directly reachable on purpose (CloudFront is an accelerator, not the security +boundary). SSH (:2222) is always direct, never through the CDN. The full +region rollout runbook lives in GlueOps/slackbot-developer-workspaces#499. + ## Quick start 1. Configure environment: @@ -73,28 +109,28 @@ The IAM credentials need: `route53:ListHostedZones`, `route53:GetChange`, ``` ``` - DOMAIN=ssh.example.com + DOMAIN=uswest1.tunnels.cde.glueopshosted.com ACME_EMAIL=admin@example.com AWS_ACCESS_KEY_ID=your_access_key_id AWS_SECRET_ACCESS_KEY=your_secret_access_key ``` -2. Start the stack: +2. Start the stack (the `cde.glueopshosted.com` NS delegation must be live + for certbot's DNS-01 validation to succeed): ```bash docker compose up -d ``` -3. Create a tunnel from a developer machine: +3. Smoke-test the tunnel and naming mode from any machine: ```bash - ssh -p 2222 -R myapp:80:localhost:3000 ssh.example.com - # → https://-myapp.ssh.example.com + ssh -p 2222 -R smoketest:80:localhost:3000 uswest1.tunnels.cde.glueopshosted.com + # → https://smoketest.uswest1.tunnels.cde.glueopshosted.com + # (no - prefix in the URL sish prints — if there is one, the wrong + # stack version is deployed) ``` -> Migrating a server from the old `glueops/sish` fork stack? See -> [`MIGRATION.md`](./MIGRATION.md). - ## Upgrading sish We pin the upstream image by digest in [`docker-compose.yml`](./docker-compose.yml). diff --git a/auth/Dockerfile b/auth/Dockerfile index 30e8972..b479028 100644 --- a/auth/Dockerfile +++ b/auth/Dockerfile @@ -1,4 +1,8 @@ -FROM python:3.12-alpine@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df +FROM python:3.14-alpine@sha256:26730869004e2b9c4b9ad09cab8625e81d256d1ce97e72df5520e806b1709f92 + +# Worker print() to a non-tty stdout is block-buffered without this — the +# auth audit lines would arrive late or vanish if a worker dies. +ENV PYTHONUNBUFFERED=1 WORKDIR /app diff --git a/auth/auth.py b/auth/auth.py index 56bea20..adea399 100644 --- a/auth/auth.py +++ b/auth/auth.py @@ -1,3 +1,7 @@ +import ipaddress +import os +import sys +import tempfile from pathlib import Path from flask import Flask, request @@ -6,39 +10,133 @@ app = Flask(__name__) DATA_DIR = Path("/data") +# Source-IP allowlist for tunnel creation: sish reports each SSH client's +# address in the auth payload, so enforcing here scopes the restriction to +# SSH only — HTTP(S) never touches this service. Empty/unset = allow all. +# Parsed at import so a malformed CIDR crashes the worker at startup +# (visible) instead of silently allowing at request time. +ALLOWED_NETWORKS = [ + ipaddress.ip_network(cidr.strip()) + for cidr in os.environ.get("TUNNEL_ALLOWED_CIDRS", "").split(",") + if cidr.strip() +] or None + + +def source_ip(remote_addr): + """IP from sish's remote_addr ("1.2.3.4:56789", "[::1]:2222"), else None.""" + host = remote_addr.rsplit(":", 1)[0].strip("[]") + try: + return ipaddress.ip_address(host) + except ValueError: + return None + + +def audit(message): + # One write syscall per line: with 4 workers sharing stdout, print()'s + # separate message+newline writes can interleave mid-line. + sys.stdout.write(message + "\n") + +# sish auth payloads are tiny; anything bigger is not sish. +app.config["MAX_CONTENT_LENGTH"] = 16 * 1024 + + +def read_key(user_file): + """Stored key, or None if the username is unregistered. Any other OSError + (weird path, unreadable file) propagates — callers must deny, not 500.""" + try: + return user_file.read_text().strip() + except FileNotFoundError: + return None + @app.route("/", methods=["POST"]) def auth(): - if not request.is_json: + data = request.get_json(silent=True) + if not isinstance(data, dict): return "Invalid JSON", 400 - data = request.json - username = secure_filename(data.get("user", "")) + username = data.get("user") key = data.get("auth_key") + remote = data.get("remote_addr") + remote = remote if isinstance(remote, str) else "?" + if not isinstance(username, str) or not isinstance(key, str): + return "Missing data", 403 + key = key.strip() if not username or not key: return "Missing data", 403 + # Tunnel-creation allowlist: an unparseable source (including a caller + # that sent no remote_addr) fails closed while the list is active. + if ALLOWED_NETWORKS is not None: + ip = source_ip(remote) + if ip is None or not any(ip in net for net in ALLOWED_NETWORKS): + audit(f"auth: {username!r} denied (source {remote} not allowlisted)") + return "Forbidden", 403 + + # sish usernames are hostnames (63-char DNS label cap) but arrive as + # attacker-chosen SSH usernames; without this cap an oversized name + # reaches the filesystem and ENAMETOOLONG turns every retry into a 500. + if len(username) > 63: + return "Forbidden", 403 + + # Reject any name the sanitizer would have to touch: distinct raw + # usernames must never collapse onto one key file ("vm a" vs "vm_a" + # both sanitize to vm_a). VM usernames are machine-generated hostnames + # and always pass unchanged; !r keeps the rejected raw value from + # injecting into the log line. + if secure_filename(username) != username: + audit(f"auth: {username!r} denied (unsafe username) from {remote}") + return "Forbidden", 403 + user_file = DATA_DIR / username - if user_file.exists(): - stored_key = user_file.read_text().strip() + try: + stored_key = read_key(user_file) + except OSError: + audit(f"auth: {username} denied (stored key unreadable) from {remote}") + return "Forbidden", 403 + if stored_key is not None: if stored_key == key: - print(f"auth: {username} allowed") + audit(f"auth: {username} allowed from {remote}") return "OK", 200 - print(f"auth: {username} denied (key mismatch)") + audit(f"auth: {username} denied (key mismatch) from {remote}") return "Forbidden", 403 + # First-use registration, atomically: write the key to a temp file and + # hard-link it into place, so the key file appears fully written or not + # at all. A create-then-write would expose an empty file to concurrent + # workers — or leave one behind on a crash, permanently locking the + # username out. Usernames can't start with "." (secure_filename strips + # leading dots), so the temp prefix can never collide with a user file. + fd, tmp_path = tempfile.mkstemp(dir=DATA_DIR, prefix=".tmp-") try: - with open(user_file, "x") as f: + with os.fdopen(fd, "w") as f: f.write(key) - print(f"auth: {username} registered") - return "OK", 200 + os.link(tmp_path, user_file) except FileExistsError: - print(f"auth: {username} denied (race)") + # Lost the race. If the winner stored the same key (same VM + # connecting twice at boot), it's still an allow. + try: + winner = read_key(user_file) + except OSError: + winner = None + if winner == key: + audit(f"auth: {username} allowed from {remote}") + return "OK", 200 + # winner None means the existing path couldn't be read (e.g. a + # dangling symlink someone left in sish_users/) — that's an operator + # problem, not a registration race; don't mislabel it in the audit. + reason = "race" if winner is not None else "stored key unreadable" + audit(f"auth: {username} denied ({reason}) from {remote}") return "Forbidden", 403 + finally: + os.unlink(tmp_path) + + audit(f"auth: {username} registered from {remote}") + return "OK", 200 if __name__ == "__main__": DATA_DIR.mkdir(exist_ok=True) - app.run(host="0.0.0.0", port=5000) \ No newline at end of file + app.run(host="0.0.0.0", port=5000) diff --git a/auth/requirements.txt b/auth/requirements.txt index dfce2bb..175d899 100644 --- a/auth/requirements.txt +++ b/auth/requirements.txt @@ -1,5 +1,5 @@ blinker==1.9.0 -click==8.4.1 +click==8.4.2 Flask==3.1.3 gunicorn==26.0.0 itsdangerous==2.2.0 diff --git a/docker-compose.yml b/docker-compose.yml index 18a8d34..1b14942 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,11 +35,23 @@ services: container_name: sish restart: always ports: - - "2222:2222" + # :2222 is IPv4-only ON PURPOSE: an unqualified publish also binds + # [::], and IPv6 connections reach the IPv4-only compose network via + # docker-proxy, which rewrites their source to the bridge gateway + # (172.x) — laundering external clients INSIDE the private-range + # tunnel allowlist. Tunnel DNS is A-record-only, so nothing + # legitimate connects over v6. + - "0.0.0.0:2222:2222" - "80:80" - "443:443" volumes: - ./ssl:/ssl + # /empty-pubkeys: sish accepts any key found in its authentication-keys + # directory WITHOUT consulting the auth webhook. The image's default dir + # ships (effectively) empty, but pointing the flag at a tmpfs makes the + # bypass impossible by construction — never mount keys into this stack. + tmpfs: + - /empty-pubkeys command: > --domain ${DOMAIN} --http-address :80 @@ -47,13 +59,13 @@ services: --ssh-address :2222 --https-certificate-directory=/ssl --authentication=true + --authentication-keys-directory=/empty-pubkeys --https=true --bind-random-subdomains=false --authentication-key-request-url="http://authenticator:5000/" - --authentication-key-request-timeout=15s + --authentication-key-request-timeout=3s --cleanup-unauthed-timeout=30s - --append-user-to-subdomain-separator="-" - --append-user-to-subdomain + --force-requested-subdomains --cleanup-unbound # --debug depends_on: @@ -65,6 +77,13 @@ services: build: ./auth container_name: sish-auth restart: always + environment: + # Source-IP allowlist for TUNNEL CREATION only (SSH auth): sish passes + # each SSH client's address here, and connections outside these CIDRs + # are denied. HTTP/HTTPS (browsers, CloudFront) never touch this + # service and are unaffected. Default: all RFC 1918 private ranges + # plus 100.64.0.0/10 (CGNAT, used by Tailscale). + - TUNNEL_ALLOWED_CIDRS=${TUNNEL_ALLOWED_CIDRS:-10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/10} volumes: # This folder will appear on your host machine containing the user files - ./sish_users:/data