From 11ca9b2a130b7959d79d4f86ddeb849a0bc68bcc Mon Sep 17 00:00:00 2001 From: Venkat Date: Sat, 1 Aug 2026 17:58:33 +0000 Subject: [PATCH 01/10] feat: support regional tunnel deployments via SISH_SUBDOMAIN_FLAGS One stack, two roles: unset keeps the legacy central behavior (append-user naming, argv-identical default), regional boxes set --force-requested-subdomains so VMs bind their bare hostname (..tunnels.cde.glueopshosted.com). Documents the role matrix, CloudFront origin relationship, and credential sources. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ErBUiAYTosnpbF9hvUj3Dn --- .env.example | 13 +++++++++++++ README.md | 26 ++++++++++++++++++++++++++ docker-compose.yml | 3 +-- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index c2116b8..fe7e642 100644 --- a/.env.example +++ b/.env.example @@ -1,10 +1,23 @@ # Domain Configuration +# Legacy central instance: tunnels.glueopshosted.com +# Regional instance: .tunnels.cde.glueopshosted.com (e.g. uswest1.tunnels.cde.glueopshosted.com) DOMAIN=ssh.example.com +# Subdomain naming mode. UNSET = legacy behavior (append the SSH username to +# the requested bind: VM binds "cde" + user -> cde-.DOMAIN). +# Regional instances MUST set the line below instead: the VM binds its bare +# hostname (-> .DOMAIN) and a taken name fails the bind rather than +# silently going random. The codespaces image and the slackbot derive naming +# from the endpoint value, so this must match the instance's role. +#SISH_SUBDOMAIN_FLAGS=--force-requested-subdomains + # Let's Encrypt ACME Email (required for certificate notifications) ACME_EMAIL=admin@example.com # AWS IAM Credentials for Route53 DNS01 Challenge # Required permissions: route53:ListHostedZones, route53:GetChange, route53:ChangeResourceRecordSets +# Legacy central: the acme-dns01-tunnels.glueopshosted.com user (aws-dns-production). +# Regional: the acme-dns01-cde.glueopshosted.com user — cde_acme_* outputs of +# glueops-opentofu-workspaces/aws-cloud-development-environment-assets-production. AWS_ACCESS_KEY_ID=your_access_key_id AWS_SECRET_ACCESS_KEY=your_secret_access_key diff --git a/README.md b/README.md index 7644c6c..50452c7 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,32 @@ sish watches that directory and reloads automatically. Renewal is checked every The IAM credentials need: `route53:ListHostedZones`, `route53:GetChange`, `route53:ChangeResourceRecordSets`. +## Deployment roles: legacy central vs regional + +The same stack deploys both generations of tunnel server; the difference is +`DOMAIN` plus one env var. + +| | Legacy central (one box) | Regional (one box per datacenter) | +|---|---|---| +| `DOMAIN` | `tunnels.glueopshosted.com` | `.tunnels.cde.glueopshosted.com` | +| `SISH_SUBDOMAIN_FLAGS` | unset (defaults to append-user) | `--force-requested-subdomains` | +| VM bind → URL | binds `cde` → `cde-.DOMAIN` | binds `` → `.DOMAIN` | +| AWS creds | `acme-dns01-tunnels...` user (aws-dns-production) | `acme-dns01-cde...` user (CDE assets workspace) | + +Regional boxes sit behind a per-region CloudFront distribution: browsers hit +`https://..tunnels.cde.glueopshosted.com` via the CDN, which +origin-fetches this box as `origin..tunnels.cde.glueopshosted.com` — +covered by the same wildcard cert, no extra config here. The box's 443 stays +directly reachable on purpose (CloudFront is an accelerator, not the security +boundary). SSH (:2222) is always direct, never through the CDN. + +Naming mode is not cosmetic: the codespaces image and the slackbot both derive +the bind/URL from the endpoint value (legacy endpoint → append-user naming, +anything else → bare hostname), so a regional box MUST set +`SISH_SUBDOMAIN_FLAGS=--force-requested-subdomains` or every access URL it +serves will be wrong. The full region rollout runbook lives in +GlueOps/slackbot-developer-workspaces#499. + ## Quick start 1. Configure environment: diff --git a/docker-compose.yml b/docker-compose.yml index 18a8d34..56a38b2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,8 +52,7 @@ services: --authentication-key-request-url="http://authenticator:5000/" --authentication-key-request-timeout=15s --cleanup-unauthed-timeout=30s - --append-user-to-subdomain-separator="-" - --append-user-to-subdomain + ${SISH_SUBDOMAIN_FLAGS:---append-user-to-subdomain-separator=- --append-user-to-subdomain} --cleanup-unbound # --debug depends_on: From 64475e65770cb7c21e1f818f883a13e901d05295 Mon Sep 17 00:00:00 2001 From: Venkat Date: Sun, 2 Aug 2026 08:25:27 +0000 Subject: [PATCH 02/10] =?UTF-8?q?feat!:=20regional-only=20deployment=20?= =?UTF-8?q?=E2=80=94=20drop=20legacy=20append-user=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stack now deploys only per-datacenter regional tunnel servers: --force-requested-subdomains is hardcoded (bare-hostname binds, taken names fail the bind), the SISH_SUBDOMAIN_FLAGS toggle is gone, and the fork-migration runbook is removed. The legacy central box must stay pinned to its current checkout (pre-0.4) — repo tip no longer carries the append-user naming it depends on. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ErBUiAYTosnpbF9hvUj3Dn --- .env.example | 18 ++----- MIGRATION.md | 127 --------------------------------------------- README.md | 54 +++++++++---------- docker-compose.yml | 2 +- 4 files changed, 32 insertions(+), 169 deletions(-) delete mode 100644 MIGRATION.md diff --git a/.env.example b/.env.example index fe7e642..d665370 100644 --- a/.env.example +++ b/.env.example @@ -1,23 +1,13 @@ -# Domain Configuration -# Legacy central instance: tunnels.glueopshosted.com -# Regional instance: .tunnels.cde.glueopshosted.com (e.g. uswest1.tunnels.cde.glueopshosted.com) -DOMAIN=ssh.example.com - -# Subdomain naming mode. UNSET = legacy behavior (append the SSH username to -# the requested bind: VM binds "cde" + user -> cde-.DOMAIN). -# Regional instances MUST set the line below instead: the VM binds its bare -# hostname (-> .DOMAIN) and a taken name fails the bind rather than -# silently going random. The codespaces image and the slackbot derive naming -# from the endpoint value, so this must match the instance's role. -#SISH_SUBDOMAIN_FLAGS=--force-requested-subdomains +# Domain Configuration: the regional tunnel endpoint this box serves, +# e.g. uswest1.tunnels.cde.glueopshosted.com +DOMAIN=uswest1.tunnels.cde.glueopshosted.com # Let's Encrypt ACME Email (required for certificate notifications) ACME_EMAIL=admin@example.com # AWS IAM Credentials for Route53 DNS01 Challenge # Required permissions: route53:ListHostedZones, route53:GetChange, route53:ChangeResourceRecordSets -# Legacy central: the acme-dns01-tunnels.glueopshosted.com user (aws-dns-production). -# Regional: the acme-dns01-cde.glueopshosted.com user — cde_acme_* outputs of +# Use the acme-dns01-cde.glueopshosted.com user — the cde_acme_* outputs of # glueops-opentofu-workspaces/aws-cloud-development-environment-assets-production. AWS_ACCESS_KEY_ID=your_access_key_id AWS_SECRET_ACCESS_KEY=your_secret_access_key diff --git a/MIGRATION.md b/MIGRATION.md deleted file mode 100644 index 8bd352d..0000000 --- a/MIGRATION.md +++ /dev/null @@ -1,127 +0,0 @@ -# Migrating an existing sish host to this repo - -This is a one-time runbook for moving a server that runs the **old -`glueops/sish` fork** stack (cloned as `~/sish/glueops-docker-compose`) onto this -repo's upstream-image stack. - -The migration is really just **copy runtime state into the new repo, then swap -stacks**. Nothing in the running setup is in git — the parts that matter are: - -- `sish_users/` — the trust-on-first-use key database. **Irreplaceable**: losing - it locks every user out until they re-register. -- `.env` — domain, ACME email, AWS credentials. -- `ssl/` — the current live certificate sish serves. -- the `letsencrypt` Docker volume — certbot's ACME account + renewal state. - -> [!WARNING] -> Never run `docker compose down -v` against the old stack — `-v` deletes the -> `letsencrypt` volume. Plain `down` keeps all volumes. - -Run everything on the server. Adjust paths if your old checkout lives elsewhere. - -## 0. Back up the irreplaceable data first - -```bash -cd ~ -tar czf sish-backup-$(date +%F-%H%M).tgz \ - -C ~/sish/glueops-docker-compose sish_users ssl .env backup-keys -ls -lh sish-backup-*.tgz -``` - -## 1. Stop the old stack (keeps data + volumes) - -```bash -cd ~/sish/glueops-docker-compose -docker compose down # frees ports 80/443/2222; keeps sish_users, ssl, volume -``` - -## 2. Clone this repo - -```bash -cd ~ && git clone https://github.com/GlueOps/cde-sish-tunnels.git -``` - -## 3. Copy runtime state into the new repo - -```bash -OLD=~/sish/glueops-docker-compose -NEW=~/cde-sish-tunnels - -cp "$OLD/.env" "$NEW/.env" -cp -a "$OLD/sish_users/." "$NEW/sish_users/" # the TOFU key DB — critical -cp -a "$OLD/ssl/." "$NEW/ssl/" # current live certs -cp -a "$OLD/backup-keys" "$NEW/" # not used by the stack; kept for safety - -# sanity: same number of users carried over, env vars present -echo "old: $(ls "$OLD/sish_users" | wc -l) new: $(ls "$NEW/sish_users" | wc -l)" -grep -E 'DOMAIN|ACME_EMAIL|AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY' "$NEW/.env" -``` - -The new compose uses the same env-var names, so `.env` drops in unchanged. - -## 4. Migrate the certbot volume (recommended) - -Carries over the ACME account and renewal state so the new stack does **not** -request a brand-new certificate on first boot. - -```bash -docker volume ls | grep letsencrypt # confirm source (expected: glueops-docker-compose_letsencrypt) - -docker volume create cde-sish-tunnels_letsencrypt -docker run --rm \ - -v glueops-docker-compose_letsencrypt:/from:ro \ - -v cde-sish-tunnels_letsencrypt:/to \ - alpine sh -c 'cp -a /from/. /to/' -``` - -Skippable: if omitted, certbot just requests a fresh cert via DNS-01 on first -boot. That works, but counts against Let's Encrypt rate limits if you redeploy -repeatedly. Either way tunnels stay up, because `ssl/` already holds a valid cert -that sish serves immediately. - -## 5. Start the new stack - -```bash -cd ~/cde-sish-tunnels -docker compose up -d --build -``` - -## 6. Verify - -```bash -docker compose ps # expect 3 services up -docker compose logs sish | tail -30 # binds :2222/:80/:443, loads certs from /ssl -docker compose logs authenticator | tail -20 -docker compose images sish # confirm it's ghcr.io/antoniomika/sish now - -# cert check (replace the domain) -echo | openssl s_client -connect :443 -servername x. 2>/dev/null \ - | openssl x509 -noout -subject -dates -``` - -Then prove the TOFU DB migrated: have an **existing** user connect with their -existing key — they should be allowed without re-registering. - -```bash -# from a client machine -ssh -p 2222 -R test:80:localhost:8080 -``` - -## 7. Rollback - -The old checkout is untouched, so reverting is instant: - -```bash -cd ~/cde-sish-tunnels && docker compose down -cd ~/sish/glueops-docker-compose && docker compose up -d -``` - -> Never run both stacks at once — they bind the same host ports. Always `down` -> one before `up` the other. - -## 8. Clean up (only once confident, e.g. the next day) - -```bash -docker volume rm glueops-docker-compose_letsencrypt # old certbot volume -# rm -rf ~/sish # old repo + fork, if desired -``` diff --git a/README.md b/README.md index 50452c7..4725453 100644 --- a/README.md +++ b/README.md @@ -64,31 +64,31 @@ sish watches that directory and reloads automatically. Renewal is checked every The IAM credentials need: `route53:ListHostedZones`, `route53:GetChange`, `route53:ChangeResourceRecordSets`. -## Deployment roles: legacy central vs regional - -The same stack deploys both generations of tunnel server; the difference is -`DOMAIN` plus one env var. - -| | Legacy central (one box) | Regional (one box per datacenter) | -|---|---|---| -| `DOMAIN` | `tunnels.glueopshosted.com` | `.tunnels.cde.glueopshosted.com` | -| `SISH_SUBDOMAIN_FLAGS` | unset (defaults to append-user) | `--force-requested-subdomains` | -| VM bind → URL | binds `cde` → `cde-.DOMAIN` | binds `` → `.DOMAIN` | -| AWS creds | `acme-dns01-tunnels...` user (aws-dns-production) | `acme-dns01-cde...` user (CDE assets workspace) | +## Deployment model: one box per datacenter region + +This repo deploys **regional** tunnel servers only: one instance per +datacenter, each serving `DOMAIN=.tunnels.cde.glueopshosted.com`. +Subdomains are bound literally (`--force-requested-subdomains`): a codespace +VM binds its own hostname, so URLs are +`https://..tunnels.cde.glueopshosted.com` with no prefix, +and a taken name fails the bind rather than silently going random. AWS creds +come from the `acme-dns01-cde.glueopshosted.com` IAM user (the `cde_acme_*` +outputs of the CDE assets workspace). + +> [!WARNING] +> The legacy central box (`tunnels.glueopshosted.com`) is NOT deployed from +> this version. It depends on the append-user naming this repo no longer +> carries (`--append-user-to-subdomain`, removed in 0.4): keep that box pinned +> to its current checkout and never pull repo tip onto it. It serves existing +> VMs until they drain, then retires. Regional boxes sit behind a per-region CloudFront distribution: browsers hit `https://..tunnels.cde.glueopshosted.com` via the CDN, which origin-fetches this box as `origin..tunnels.cde.glueopshosted.com` — covered by the same wildcard cert, no extra config here. The box's 443 stays directly reachable on purpose (CloudFront is an accelerator, not the security -boundary). SSH (:2222) is always direct, never through the CDN. - -Naming mode is not cosmetic: the codespaces image and the slackbot both derive -the bind/URL from the endpoint value (legacy endpoint → append-user naming, -anything else → bare hostname), so a regional box MUST set -`SISH_SUBDOMAIN_FLAGS=--force-requested-subdomains` or every access URL it -serves will be wrong. The full region rollout runbook lives in -GlueOps/slackbot-developer-workspaces#499. +boundary). SSH (:2222) is always direct, never through the CDN. The full +region rollout runbook lives in GlueOps/slackbot-developer-workspaces#499. ## Quick start @@ -99,28 +99,28 @@ GlueOps/slackbot-developer-workspaces#499. ``` ``` - DOMAIN=ssh.example.com + DOMAIN=uswest1.tunnels.cde.glueopshosted.com ACME_EMAIL=admin@example.com AWS_ACCESS_KEY_ID=your_access_key_id AWS_SECRET_ACCESS_KEY=your_secret_access_key ``` -2. Start the stack: +2. Start the stack (the `cde.glueopshosted.com` NS delegation must be live + for certbot's DNS-01 validation to succeed): ```bash docker compose up -d ``` -3. Create a tunnel from a developer machine: +3. Smoke-test the tunnel and naming mode from any machine: ```bash - ssh -p 2222 -R myapp:80:localhost:3000 ssh.example.com - # → https://-myapp.ssh.example.com + ssh -p 2222 -R smoketest:80:localhost:3000 uswest1.tunnels.cde.glueopshosted.com + # → https://smoketest.uswest1.tunnels.cde.glueopshosted.com + # (no - prefix in the URL sish prints — if there is one, the wrong + # stack version is deployed) ``` -> Migrating a server from the old `glueops/sish` fork stack? See -> [`MIGRATION.md`](./MIGRATION.md). - ## Upgrading sish We pin the upstream image by digest in [`docker-compose.yml`](./docker-compose.yml). diff --git a/docker-compose.yml b/docker-compose.yml index 56a38b2..2640c0d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,7 +52,7 @@ services: --authentication-key-request-url="http://authenticator:5000/" --authentication-key-request-timeout=15s --cleanup-unauthed-timeout=30s - ${SISH_SUBDOMAIN_FLAGS:---append-user-to-subdomain-separator=- --append-user-to-subdomain} + --force-requested-subdomains --cleanup-unbound # --debug depends_on: From 5a21008a14cf986b26ed9209a7389e987e9e9e54 Mon Sep 17 00:00:00 2001 From: Venkat Date: Sun, 2 Aug 2026 08:30:00 +0000 Subject: [PATCH 03/10] docs: drop the legacy pin warning from the README Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ErBUiAYTosnpbF9hvUj3Dn --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index 4725453..33ee071 100644 --- a/README.md +++ b/README.md @@ -75,13 +75,6 @@ and a taken name fails the bind rather than silently going random. AWS creds come from the `acme-dns01-cde.glueopshosted.com` IAM user (the `cde_acme_*` outputs of the CDE assets workspace). -> [!WARNING] -> The legacy central box (`tunnels.glueopshosted.com`) is NOT deployed from -> this version. It depends on the append-user naming this repo no longer -> carries (`--append-user-to-subdomain`, removed in 0.4): keep that box pinned -> to its current checkout and never pull repo tip onto it. It serves existing -> VMs until they drain, then retires. - Regional boxes sit behind a per-region CloudFront distribution: browsers hit `https://..tunnels.cde.glueopshosted.com` via the CDN, which origin-fetches this box as `origin..tunnels.cde.glueopshosted.com` — From 8f11ac508ad875df501091b5ce3ddf69cd154557 Mon Sep 17 00:00:00 2001 From: Venkat Date: Sun, 2 Aug 2026 08:40:24 +0000 Subject: [PATCH 04/10] =?UTF-8?q?chore(deps):=20apply=20renovate=20updates?= =?UTF-8?q?=20=E2=80=94=20python=203.14-alpine,=20click=208.4.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Supersedes renovate PRs #28/#29/#30 (taking 3.14 over 3.13). All pins verified on the pinned digest: cp314 musllinux wheels present (MarkupSafe C speedups active), Flask/Werkzeug CI-test 3.14 upstream; gunicorn 26.0.0 lacks official 3.14 metadata but the sync-worker path is exercised and passing. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ErBUiAYTosnpbF9hvUj3Dn --- auth/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth/requirements.txt b/auth/requirements.txt index dfce2bb..175d899 100644 --- a/auth/requirements.txt +++ b/auth/requirements.txt @@ -1,5 +1,5 @@ blinker==1.9.0 -click==8.4.1 +click==8.4.2 Flask==3.1.3 gunicorn==26.0.0 itsdangerous==2.2.0 From fc7371d51bf175a42f80c289bff8a4fb5876a5ef Mon Sep 17 00:00:00 2001 From: Venkat Date: Sun, 2 Aug 2026 08:40:24 +0000 Subject: [PATCH 05/10] fix(auth): close poisoning, collision, and partial-write defects Expert review of the TOFU authorizer found and reproduced: a non-string auth_key left an empty key file behind, permanently locking the username out; distinct raw usernames collapsing to one key file via secure_filename ("vm 1" vs "vm_1"); and a create-then-write window exposing empty files to concurrent workers. Registration now validates JSON shape/types, rejects any username the sanitizer would alter, and links a fully-written temp file into place atomically (same-key race losers still allowed). Also: 16KB request cap, remote_addr in audit lines, PYTHONUNBUFFERED so those lines actually flush. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ErBUiAYTosnpbF9hvUj3Dn --- auth/Dockerfile | 6 ++++- auth/auth.py | 65 +++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/auth/Dockerfile b/auth/Dockerfile index 30e8972..b479028 100644 --- a/auth/Dockerfile +++ b/auth/Dockerfile @@ -1,4 +1,8 @@ -FROM python:3.12-alpine@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df +FROM python:3.14-alpine@sha256:26730869004e2b9c4b9ad09cab8625e81d256d1ce97e72df5520e806b1709f92 + +# Worker print() to a non-tty stdout is block-buffered without this — the +# auth audit lines would arrive late or vanish if a worker dies. +ENV PYTHONUNBUFFERED=1 WORKDIR /app diff --git a/auth/auth.py b/auth/auth.py index 56bea20..264b107 100644 --- a/auth/auth.py +++ b/auth/auth.py @@ -1,3 +1,5 @@ +import os +import tempfile from pathlib import Path from flask import Flask, request @@ -6,39 +8,76 @@ app = Flask(__name__) DATA_DIR = Path("/data") +# sish auth payloads are tiny; anything bigger is not sish. +app.config["MAX_CONTENT_LENGTH"] = 16 * 1024 + + +def read_key(user_file): + try: + return user_file.read_text().strip() + except FileNotFoundError: + return None + @app.route("/", methods=["POST"]) def auth(): - if not request.is_json: + data = request.get_json(silent=True) + if not isinstance(data, dict): return "Invalid JSON", 400 - data = request.json - username = secure_filename(data.get("user", "")) + username = data.get("user") key = data.get("auth_key") + remote = data.get("remote_addr") + remote = remote if isinstance(remote, str) else "?" - if not username or not key: + if not isinstance(username, str) or not isinstance(key, str) or not username or not key: return "Missing data", 403 + # Reject any name the sanitizer would have to touch: distinct raw + # usernames must never collapse onto one key file ("vm a" vs "vm_a" + # both sanitize to vm_a). VM usernames are machine-generated hostnames + # and always pass unchanged; !r keeps the rejected raw value from + # injecting into the log line. + if secure_filename(username) != username: + print(f"auth: {username!r} denied (unsafe username) from {remote}") + return "Forbidden", 403 + user_file = DATA_DIR / username - if user_file.exists(): - stored_key = user_file.read_text().strip() + stored_key = read_key(user_file) + if stored_key is not None: if stored_key == key: - print(f"auth: {username} allowed") + print(f"auth: {username} allowed from {remote}") return "OK", 200 - print(f"auth: {username} denied (key mismatch)") + print(f"auth: {username} denied (key mismatch) from {remote}") return "Forbidden", 403 + # First-use registration, atomically: write the key to a temp file and + # hard-link it into place, so the key file appears fully written or not + # at all. A create-then-write would expose an empty file to concurrent + # workers — or leave one behind on a crash, permanently locking the + # username out. Usernames can't start with "." (secure_filename strips + # leading dots), so the temp prefix can never collide with a user file. + fd, tmp_path = tempfile.mkstemp(dir=DATA_DIR, prefix=".tmp-") try: - with open(user_file, "x") as f: + with os.fdopen(fd, "w") as f: f.write(key) - print(f"auth: {username} registered") - return "OK", 200 + os.link(tmp_path, user_file) except FileExistsError: - print(f"auth: {username} denied (race)") + # Lost the race. If the winner stored the same key (same VM + # connecting twice at boot), it's still an allow. + if read_key(user_file) == key: + print(f"auth: {username} allowed from {remote}") + return "OK", 200 + print(f"auth: {username} denied (race) from {remote}") return "Forbidden", 403 + finally: + os.unlink(tmp_path) + + print(f"auth: {username} registered from {remote}") + return "OK", 200 if __name__ == "__main__": DATA_DIR.mkdir(exist_ok=True) - app.run(host="0.0.0.0", port=5000) \ No newline at end of file + app.run(host="0.0.0.0", port=5000) From 271f6dc442d9c2d55dc373bd3f00fc983a30a61d Mon Sep 17 00:00:00 2001 From: Venkat Date: Sun, 2 Aug 2026 08:52:22 +0000 Subject: [PATCH 06/10] fix: close remotely-triggerable 500s and tighten sish auth coupling Second expert pass (Python + sish v2.23.0 source review): cap usernames at the 63-char hostname label limit (oversized attacker-chosen SSH usernames reached the filesystem and turned every retry into a traceback-spamming 500 from the public SSH port), deny on any OSError reading a stored key instead of 500ing, and strip/reject whitespace-only keys. Compose: authentication-key-request-timeout 15s -> 3s (sish holds a global lock across the webhook call; a hung authenticator serialized all auth on the instance) and point authentication-keys-directory at a tmpfs so the webhook-bypass path is empty by construction. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ErBUiAYTosnpbF9hvUj3Dn --- auth/auth.py | 25 ++++++++++++++++++++++--- docker-compose.yml | 9 ++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/auth/auth.py b/auth/auth.py index 264b107..a75ca03 100644 --- a/auth/auth.py +++ b/auth/auth.py @@ -13,6 +13,8 @@ def read_key(user_file): + """Stored key, or None if the username is unregistered. Any other OSError + (weird path, unreadable file) propagates — callers must deny, not 500.""" try: return user_file.read_text().strip() except FileNotFoundError: @@ -30,8 +32,17 @@ def auth(): remote = data.get("remote_addr") remote = remote if isinstance(remote, str) else "?" - if not isinstance(username, str) or not isinstance(key, str) or not username or not key: + if not isinstance(username, str) or not isinstance(key, str): return "Missing data", 403 + key = key.strip() + if not username or not key: + return "Missing data", 403 + + # sish usernames are hostnames (63-char DNS label cap) but arrive as + # attacker-chosen SSH usernames; without this cap an oversized name + # reaches the filesystem and ENAMETOOLONG turns every retry into a 500. + if len(username) > 63: + return "Forbidden", 403 # Reject any name the sanitizer would have to touch: distinct raw # usernames must never collapse onto one key file ("vm a" vs "vm_a" @@ -44,7 +55,11 @@ def auth(): user_file = DATA_DIR / username - stored_key = read_key(user_file) + try: + stored_key = read_key(user_file) + except OSError: + print(f"auth: {username} denied (stored key unreadable) from {remote}") + return "Forbidden", 403 if stored_key is not None: if stored_key == key: print(f"auth: {username} allowed from {remote}") @@ -66,7 +81,11 @@ def auth(): except FileExistsError: # Lost the race. If the winner stored the same key (same VM # connecting twice at boot), it's still an allow. - if read_key(user_file) == key: + try: + winner = read_key(user_file) + except OSError: + winner = None + if winner == key: print(f"auth: {username} allowed from {remote}") return "OK", 200 print(f"auth: {username} denied (race) from {remote}") diff --git a/docker-compose.yml b/docker-compose.yml index 2640c0d..ad26449 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,12 @@ services: - "443:443" volumes: - ./ssl:/ssl + # /empty-pubkeys: sish accepts any key found in its authentication-keys + # directory WITHOUT consulting the auth webhook. The image's default dir + # ships (effectively) empty, but pointing the flag at a tmpfs makes the + # bypass impossible by construction — never mount keys into this stack. + tmpfs: + - /empty-pubkeys command: > --domain ${DOMAIN} --http-address :80 @@ -47,10 +53,11 @@ services: --ssh-address :2222 --https-certificate-directory=/ssl --authentication=true + --authentication-keys-directory=/empty-pubkeys --https=true --bind-random-subdomains=false --authentication-key-request-url="http://authenticator:5000/" - --authentication-key-request-timeout=15s + --authentication-key-request-timeout=3s --cleanup-unauthed-timeout=30s --force-requested-subdomains --cleanup-unbound From 4a582ced7b02a6a2784eacbb1426a0e8ac60ea58 Mon Sep 17 00:00:00 2001 From: Venkat Date: Sun, 2 Aug 2026 09:02:35 +0000 Subject: [PATCH 07/10] fix(auth): tear-free audit lines and honest race-path labeling Each audit line is now a single write syscall (4 workers share stdout; print()'s separate message+newline writes could interleave mid-line), and a registration collision against an unreadable existing path (e.g. a dangling symlink in sish_users/) audits "stored key unreadable" instead of mislabeling it a race. Non-root container and a TOFU registration cap were considered and deliberately skipped (compose runs as root; bind-mount ownership would break registrations). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ErBUiAYTosnpbF9hvUj3Dn --- auth/auth.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/auth/auth.py b/auth/auth.py index a75ca03..7f8cbe7 100644 --- a/auth/auth.py +++ b/auth/auth.py @@ -1,4 +1,5 @@ import os +import sys import tempfile from pathlib import Path @@ -8,6 +9,12 @@ app = Flask(__name__) DATA_DIR = Path("/data") + +def audit(message): + # One write syscall per line: with 4 workers sharing stdout, print()'s + # separate message+newline writes can interleave mid-line. + sys.stdout.write(message + "\n") + # sish auth payloads are tiny; anything bigger is not sish. app.config["MAX_CONTENT_LENGTH"] = 16 * 1024 @@ -50,7 +57,7 @@ def auth(): # and always pass unchanged; !r keeps the rejected raw value from # injecting into the log line. if secure_filename(username) != username: - print(f"auth: {username!r} denied (unsafe username) from {remote}") + audit(f"auth: {username!r} denied (unsafe username) from {remote}") return "Forbidden", 403 user_file = DATA_DIR / username @@ -58,13 +65,13 @@ def auth(): try: stored_key = read_key(user_file) except OSError: - print(f"auth: {username} denied (stored key unreadable) from {remote}") + audit(f"auth: {username} denied (stored key unreadable) from {remote}") return "Forbidden", 403 if stored_key is not None: if stored_key == key: - print(f"auth: {username} allowed from {remote}") + audit(f"auth: {username} allowed from {remote}") return "OK", 200 - print(f"auth: {username} denied (key mismatch) from {remote}") + audit(f"auth: {username} denied (key mismatch) from {remote}") return "Forbidden", 403 # First-use registration, atomically: write the key to a temp file and @@ -86,14 +93,18 @@ def auth(): except OSError: winner = None if winner == key: - print(f"auth: {username} allowed from {remote}") + audit(f"auth: {username} allowed from {remote}") return "OK", 200 - print(f"auth: {username} denied (race) from {remote}") + # winner None means the existing path couldn't be read (e.g. a + # dangling symlink someone left in sish_users/) — that's an operator + # problem, not a registration race; don't mislabel it in the audit. + reason = "race" if winner is not None else "stored key unreadable" + audit(f"auth: {username} denied ({reason}) from {remote}") return "Forbidden", 403 finally: os.unlink(tmp_path) - print(f"auth: {username} registered from {remote}") + audit(f"auth: {username} registered from {remote}") return "OK", 200 From 09fa290a7eade2d02b94ecd7c8a769b87bd7a4a3 Mon Sep 17 00:00:00 2001 From: Venkat Date: Sun, 2 Aug 2026 09:08:03 +0000 Subject: [PATCH 08/10] feat: restrict tunnel creation by source IP, default RFC 1918 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TUNNEL_ALLOWED_CIDRS (default: all private ranges) is enforced by the authenticator using the client address sish reports on every SSH auth — so it gates tunnel registration/connection only, and browsers/CloudFront (:80/:443) are unaffected. Unparseable or missing sources fail closed while the list is active; empty value disables the check; a malformed CIDR crashes the authenticator at startup. sish's own --whitelisted-ips was deliberately not used: it is global across SSH and HTTP/S and would have blocked public web access. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ErBUiAYTosnpbF9hvUj3Dn --- .env.example | 10 ++++++++++ README.md | 18 ++++++++++++++++++ auth/auth.py | 29 +++++++++++++++++++++++++++++ docker-compose.yml | 6 ++++++ 4 files changed, 63 insertions(+) diff --git a/.env.example b/.env.example index d665370..074d542 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,16 @@ # e.g. uswest1.tunnels.cde.glueopshosted.com DOMAIN=uswest1.tunnels.cde.glueopshosted.com +# Source-IP allowlist for TUNNEL CREATION, comma-separated CIDRs (single +# IPs as /32). Enforced by the authenticator on SSH auth only — browsers +# and CloudFront (:80/:443) are never affected. Default: all RFC 1918 +# private ranges, i.e. only VMs on private networks can register/connect +# tunnels. If VMs reach this box over Tailscale, add the CGNAT range +# 100.64.0.0/10 (not part of RFC 1918); if they arrive from public +# addresses, add those. Empty value = no source restriction. A malformed +# CIDR fails the authenticator at startup rather than silently allowing. +TUNNEL_ALLOWED_CIDRS=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 + # Let's Encrypt ACME Email (required for certificate notifications) ACME_EMAIL=admin@example.com diff --git a/README.md b/README.md index 33ee071..7f7564e 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,24 @@ and a taken name fails the bind rather than silently going random. AWS creds come from the `acme-dns01-cde.glueopshosted.com` IAM user (the `cde_acme_*` outputs of the CDE assets workspace). +### Restricting who can create tunnels + +`TUNNEL_ALLOWED_CIDRS` (comma-separated CIDRs, default: all RFC 1918 private +ranges) is enforced by the **authenticator** on every SSH auth attempt using +the client address sish reports — so it restricts tunnel creation only. +Browsers and CloudFront (:80/:443) never touch the authenticator and are +unaffected. Outside-the-list clients get a logged 403 +(`denied (source ... not allowlisted)`) and can never register or connect a +tunnel. Empty value disables the restriction; a malformed CIDR fails the +authenticator at startup. Tailscale-sourced VMs need `100.64.0.0/10` (CGNAT, +not part of RFC 1918) added. + +Deliberately NOT used: sish's own `--whitelisted-ips`, which is global across +SSH *and* HTTP/S and would block public browsers. For packet-level dropping +(before the SSH handshake), add host-firewall rules on :2222 in the +`DOCKER-USER` iptables chain (published ports bypass normal INPUT rules) — +that composes fine with this setting. + Regional boxes sit behind a per-region CloudFront distribution: browsers hit `https://..tunnels.cde.glueopshosted.com` via the CDN, which origin-fetches this box as `origin..tunnels.cde.glueopshosted.com` — diff --git a/auth/auth.py b/auth/auth.py index 7f8cbe7..adea399 100644 --- a/auth/auth.py +++ b/auth/auth.py @@ -1,3 +1,4 @@ +import ipaddress import os import sys import tempfile @@ -9,6 +10,26 @@ app = Flask(__name__) DATA_DIR = Path("/data") +# Source-IP allowlist for tunnel creation: sish reports each SSH client's +# address in the auth payload, so enforcing here scopes the restriction to +# SSH only — HTTP(S) never touches this service. Empty/unset = allow all. +# Parsed at import so a malformed CIDR crashes the worker at startup +# (visible) instead of silently allowing at request time. +ALLOWED_NETWORKS = [ + ipaddress.ip_network(cidr.strip()) + for cidr in os.environ.get("TUNNEL_ALLOWED_CIDRS", "").split(",") + if cidr.strip() +] or None + + +def source_ip(remote_addr): + """IP from sish's remote_addr ("1.2.3.4:56789", "[::1]:2222"), else None.""" + host = remote_addr.rsplit(":", 1)[0].strip("[]") + try: + return ipaddress.ip_address(host) + except ValueError: + return None + def audit(message): # One write syscall per line: with 4 workers sharing stdout, print()'s @@ -45,6 +66,14 @@ def auth(): if not username or not key: return "Missing data", 403 + # Tunnel-creation allowlist: an unparseable source (including a caller + # that sent no remote_addr) fails closed while the list is active. + if ALLOWED_NETWORKS is not None: + ip = source_ip(remote) + if ip is None or not any(ip in net for net in ALLOWED_NETWORKS): + audit(f"auth: {username!r} denied (source {remote} not allowlisted)") + return "Forbidden", 403 + # sish usernames are hostnames (63-char DNS label cap) but arrive as # attacker-chosen SSH usernames; without this cap an oversized name # reaches the filesystem and ENAMETOOLONG turns every retry into a 500. diff --git a/docker-compose.yml b/docker-compose.yml index ad26449..f71d743 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,6 +71,12 @@ services: build: ./auth container_name: sish-auth restart: always + environment: + # Source-IP allowlist for TUNNEL CREATION only (SSH auth): sish passes + # each SSH client's address here, and connections outside these CIDRs + # are denied. HTTP/HTTPS (browsers, CloudFront) never touch this + # service and are unaffected. Default: all RFC 1918 private ranges. + - TUNNEL_ALLOWED_CIDRS=${TUNNEL_ALLOWED_CIDRS:-10.0.0.0/8,172.16.0.0/12,192.168.0.0/16} volumes: # This folder will appear on your host machine containing the user files - ./sish_users:/data From 0587cd064935330a368787479d0562470b8c685a Mon Sep 17 00:00:00 2001 From: Venkat Date: Sun, 2 Aug 2026 09:14:35 +0000 Subject: [PATCH 09/10] feat: include the Tailscale CGNAT range in the default tunnel allowlist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 100.64.0.0/10 joins the RFC 1918 defaults for TUNNEL_ALLOWED_CIDRS — codespace VMs join the tailnet at boot, so tailnet-sourced tunnel connections are first-class. Also drops the host-firewall documentation. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ErBUiAYTosnpbF9hvUj3Dn --- .env.example | 12 ++++++------ README.md | 20 ++++++++------------ docker-compose.yml | 5 +++-- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.env.example b/.env.example index 074d542..cbae2e6 100644 --- a/.env.example +++ b/.env.example @@ -5,12 +5,12 @@ DOMAIN=uswest1.tunnels.cde.glueopshosted.com # Source-IP allowlist for TUNNEL CREATION, comma-separated CIDRs (single # IPs as /32). Enforced by the authenticator on SSH auth only — browsers # and CloudFront (:80/:443) are never affected. Default: all RFC 1918 -# private ranges, i.e. only VMs on private networks can register/connect -# tunnels. If VMs reach this box over Tailscale, add the CGNAT range -# 100.64.0.0/10 (not part of RFC 1918); if they arrive from public -# addresses, add those. Empty value = no source restriction. A malformed -# CIDR fails the authenticator at startup rather than silently allowing. -TUNNEL_ALLOWED_CIDRS=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 +# private ranges plus the CGNAT range 100.64.0.0/10 (Tailscale), i.e. only +# VMs on private networks or the tailnet can register/connect tunnels. If +# VMs arrive from public addresses, add those. Empty value = no source +# restriction. A malformed CIDR fails the authenticator at startup rather +# than silently allowing. +TUNNEL_ALLOWED_CIDRS=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/10 # Let's Encrypt ACME Email (required for certificate notifications) ACME_EMAIL=admin@example.com diff --git a/README.md b/README.md index 7f7564e..8ea2676 100644 --- a/README.md +++ b/README.md @@ -78,20 +78,16 @@ outputs of the CDE assets workspace). ### Restricting who can create tunnels `TUNNEL_ALLOWED_CIDRS` (comma-separated CIDRs, default: all RFC 1918 private -ranges) is enforced by the **authenticator** on every SSH auth attempt using -the client address sish reports — so it restricts tunnel creation only. -Browsers and CloudFront (:80/:443) never touch the authenticator and are -unaffected. Outside-the-list clients get a logged 403 -(`denied (source ... not allowlisted)`) and can never register or connect a -tunnel. Empty value disables the restriction; a malformed CIDR fails the -authenticator at startup. Tailscale-sourced VMs need `100.64.0.0/10` (CGNAT, -not part of RFC 1918) added. +ranges plus `100.64.0.0/10` — the CGNAT range Tailscale uses) is enforced by +the **authenticator** on every SSH auth attempt using the client address sish +reports — so it restricts tunnel creation only. Browsers and CloudFront +(:80/:443) never touch the authenticator and are unaffected. Outside-the-list +clients get a logged 403 (`denied (source ... not allowlisted)`) and can +never register or connect a tunnel. Empty value disables the restriction; a +malformed CIDR fails the authenticator at startup. Deliberately NOT used: sish's own `--whitelisted-ips`, which is global across -SSH *and* HTTP/S and would block public browsers. For packet-level dropping -(before the SSH handshake), add host-firewall rules on :2222 in the -`DOCKER-USER` iptables chain (published ports bypass normal INPUT rules) — -that composes fine with this setting. +SSH *and* HTTP/S and would block public browsers. Regional boxes sit behind a per-region CloudFront distribution: browsers hit `https://..tunnels.cde.glueopshosted.com` via the CDN, which diff --git a/docker-compose.yml b/docker-compose.yml index f71d743..fee4fa5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,8 +75,9 @@ services: # Source-IP allowlist for TUNNEL CREATION only (SSH auth): sish passes # each SSH client's address here, and connections outside these CIDRs # are denied. HTTP/HTTPS (browsers, CloudFront) never touch this - # service and are unaffected. Default: all RFC 1918 private ranges. - - TUNNEL_ALLOWED_CIDRS=${TUNNEL_ALLOWED_CIDRS:-10.0.0.0/8,172.16.0.0/12,192.168.0.0/16} + # service and are unaffected. Default: all RFC 1918 private ranges + # plus 100.64.0.0/10 (CGNAT, used by Tailscale). + - TUNNEL_ALLOWED_CIDRS=${TUNNEL_ALLOWED_CIDRS:-10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,100.64.0.0/10} volumes: # This folder will appear on your host machine containing the user files - ./sish_users:/data From 0391d1ddd30ed18a577b07c51bc47d65b04dae60 Mon Sep 17 00:00:00 2001 From: Venkat Date: Sun, 2 Aug 2026 09:24:29 +0000 Subject: [PATCH 10/10] fix: publish :2222 IPv4-only to keep the tunnel allowlist sound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sish v2.23.0 source review confirmed the allowlist gates every tunnel-creation path — but an unqualified port publish also binds [::], and docker-proxy relays IPv6 clients into the IPv4-only compose network with their source rewritten to the bridge gateway, a 172.x address inside the private-range allowlist. Binding 0.0.0.0 closes the laundering path; tunnel DNS is A-record-only so no legitimate client connects over v6. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ErBUiAYTosnpbF9hvUj3Dn --- README.md | 5 ++++- docker-compose.yml | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8ea2676..3fffe01 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,10 @@ never register or connect a tunnel. Empty value disables the restriction; a malformed CIDR fails the authenticator at startup. Deliberately NOT used: sish's own `--whitelisted-ips`, which is global across -SSH *and* HTTP/S and would block public browsers. +SSH *and* HTTP/S and would block public browsers. The :2222 publish is +IPv4-only because docker-proxy rewrites IPv6 clients' sources to the bridge +gateway (a 172.x address inside the allowlist) — an unqualified publish would +let external IPv6 clients through the check. Regional boxes sit behind a per-region CloudFront distribution: browsers hit `https://..tunnels.cde.glueopshosted.com` via the CDN, which diff --git a/docker-compose.yml b/docker-compose.yml index fee4fa5..1b14942 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,13 @@ services: container_name: sish restart: always ports: - - "2222:2222" + # :2222 is IPv4-only ON PURPOSE: an unqualified publish also binds + # [::], and IPv6 connections reach the IPv4-only compose network via + # docker-proxy, which rewrites their source to the bridge gateway + # (172.x) — laundering external clients INSIDE the private-range + # tunnel allowlist. Tunnel DNS is A-record-only, so nothing + # legitimate connects over v6. + - "0.0.0.0:2222:2222" - "80:80" - "443:443" volumes: