diff --git a/.github/workflows/bayn-ci.yml b/.github/workflows/bayn-ci.yml index e43b61f63c..6e419e0c21 100644 --- a/.github/workflows/bayn-ci.yml +++ b/.github/workflows/bayn-ci.yml @@ -36,6 +36,8 @@ jobs: case "${path}" in services/bayn/* | \ packages/scripts/src/bayn/* | \ + scripts/bayn-post-deploy-verify.ts | \ + scripts/bayn-post-deploy-verify.test.ts | \ argocd/applications/bayn/* | \ argocd/applications/torghut/clickhouse/* | \ argocd/applicationsets/product.yaml | \ @@ -108,10 +110,34 @@ jobs: bun-version: 1.3.14 node-version: 24.11.1 install-command: bun install --frozen-lockfile --ignore-scripts --filter @proompteng/source --filter @proompteng/bayn --filter @proompteng/scripts - lint-command: bunx oxfmt --check services/bayn argocd/applications/bayn argocd/applications/torghut/clickhouse/bayn-sealed-secret.yaml argocd/applications/torghut/clickhouse/clickhouse-cluster.yaml - oxlint-command: bun run --cwd services/bayn lint:oxlint + lint-command: >- + bunx oxfmt --check + services/bayn + scripts/bayn-post-deploy-verify.ts + scripts/bayn-post-deploy-verify.test.ts + argocd/applications/bayn + argocd/applications/torghut/clickhouse/bayn-sealed-secret.yaml + argocd/applications/torghut/clickhouse/clickhouse-cluster.yaml + oxlint-command: >- + bun run --cwd services/bayn lint:oxlint && + bunx oxlint --config .oxlintrc.json + scripts/bayn-post-deploy-verify.ts + scripts/bayn-post-deploy-verify.test.ts oxlint-type-command: bun run --cwd services/bayn lint:oxlint:type - test-command: bun run --cwd services/bayn tsc && bun run --cwd services/bayn lint:effect && bun run --cwd services/bayn test && bun test packages/scripts/src/bayn + test-command: >- + bun run --cwd services/bayn tsc && + bun run --cwd services/bayn lint:effect && + bun run --cwd services/bayn test && + bun test packages/scripts/src/bayn scripts/bayn-post-deploy-verify.test.ts && + bun node_modules/typescript/bin/tsc --ignoreConfig --noEmit + --module esnext + --moduleResolution bundler + --target esnext + --lib esnext,dom + --types bun-types + --skipLibCheck + scripts/bayn-post-deploy-verify.ts + scripts/bayn-post-deploy-verify.test.ts build-command: bun run --cwd services/bayn build secrets: inherit diff --git a/.github/workflows/bayn-post-deploy-verify.yml b/.github/workflows/bayn-post-deploy-verify.yml new file mode 100644 index 0000000000..84cdd76ec9 --- /dev/null +++ b/.github/workflows/bayn-post-deploy-verify.yml @@ -0,0 +1,76 @@ +name: bayn-post-deploy-verify + +on: + push: + branches: + - main + paths: + - 'argocd/applications/bayn/deployment.yaml' + - 'argocd/applications/bayn/kustomization.yaml' + +concurrency: + group: bayn-post-deploy-verify-main + cancel-in-progress: true + +jobs: + verify: + runs-on: arc-amd64 + timeout-minutes: 20 + permissions: + contents: read + steps: + - name: Checkout exact promoted main revision + uses: actions/checkout@v5 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + + - name: Set up Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.14 + + - name: Set up kubectl + uses: azure/setup-kubectl@v5 + with: + version: v1.35.0 + + - name: Configure in-cluster kubeconfig + shell: bash + run: | + set -euo pipefail + if kubectl config current-context >/dev/null 2>&1; then + exit 0 + fi + test -n "${KUBERNETES_SERVICE_HOST:-}" + test -n "${KUBERNETES_SERVICE_PORT:-}" + test -r /var/run/secrets/kubernetes.io/serviceaccount/token + test -r /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + + kubeconfig="${RUNNER_TEMP}/kubeconfig" + echo "KUBECONFIG=${kubeconfig}" >> "${GITHUB_ENV}" + kubectl config set-cluster in-cluster \ + --server="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}" \ + --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \ + --embed-certs=true \ + --kubeconfig="${kubeconfig}" + kubectl config set-credentials gha-runner \ + --token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \ + --kubeconfig="${kubeconfig}" + kubectl config set-context in-cluster \ + --cluster=in-cluster \ + --user=gha-runner \ + --namespace=bayn \ + --kubeconfig="${kubeconfig}" + kubectl config use-context in-cluster --kubeconfig="${kubeconfig}" + + - name: Verify exact Bayn production promotion + shell: bash + env: + EXPECTED_ARGO_REVISION: ${{ github.sha }} + run: | + set -euo pipefail + bun scripts/bayn-post-deploy-verify.ts \ + --expected-revision "${EXPECTED_ARGO_REVISION}" \ + --deadline-seconds 900 \ + --interval-seconds 10 diff --git a/scripts/bayn-post-deploy-verify.test.ts b/scripts/bayn-post-deploy-verify.test.ts new file mode 100644 index 0000000000..3382f5ef9b --- /dev/null +++ b/scripts/bayn-post-deploy-verify.test.ts @@ -0,0 +1,1071 @@ +import { describe, expect, test } from 'bun:test' +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' + +import { + parseExpectedPromotion, + readArgoSyncRevision, + redactSensitive, + runCommand, + runWithinDeadline, + retryVerification, + verifyArgoRevision, + validateReadOnlyPermissions as validateProductionReadOnlyPermissions, + validateSnapshot as validateProductionSnapshot, + VerificationFailure, + type ExpectedPromotion, + type RunCommand, + type VerificationSnapshot, +} from './bayn-post-deploy-verify' + +const expectedArgoRevision = '27f279697152319abacac0d1ea806a210671ca8c' +const descendantArgoRevision = '5fa8e184d6001ff868093d50d110af4be0639695' +const sourceRevision = 'c7794ce4892ae7d9d6a7b38480a02fe1b39399b0' +const oldSourceRevision = '6be9f985e866d70748bc11d21ca92731c81a5736' +const digest = 'sha256:13579bcffc30f6f2eaa4ba2054347b96888117236b286e27ac63374d9ea1db53' +const oldDigest = 'sha256:5b46c9b7ed1ca3f7617c9357f0fedca6226badd037ea4fbed2daf78fa8aa5564' +const repository = 'registry.ide-newton.ts.net/lab/bayn' +const tag = `sha-${sourceRevision}` +const imageReference = `${repository}:${tag}@${digest}` +const verificationNowMs = Date.parse('2026-07-30T06:52:30.000Z') + +const kustomization = ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: + - name: ${repository} + newName: ${repository} + newTag: "${tag}" + digest: ${digest} +` + +const deploymentManifest = ` +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - name: bayn + image: ${repository} + env: + - name: BAYN_CODE_REVISION + value: ${sourceRevision} + - name: BAYN_IMAGE_REPOSITORY + value: ${repository} + - name: BAYN_IMAGE_DIGEST + value: ${digest} + - name: BAYN_BROKER_ENVIRONMENT + value: sandbox + - name: BAYN_MAXIMUM_AUTHORITY + value: OBSERVE + - name: BAYN_ALPACA_ACCOUNT_ID + valueFrom: + secretKeyRef: + name: bayn-alpaca-auth + key: account-id + volumes: + - name: postgres-ca + secret: + secretName: bayn-db-ca +` + +const expected = parseExpectedPromotion(kustomization, deploymentManifest) +const readOnlyAuthorizationRules = [ + 'selfsubjectreviews.authentication.k8s.io [] [] [create]', + 'selfsubjectaccessreviews.authorization.k8s.io [] [] [create]', + 'selfsubjectrulesreviews.authorization.k8s.io [] [] [create]', + 'deployments.apps [] [] [get list watch]', +].join('\n') + +const validateReadOnlyPermissions = ( + run: RunCommand, + signal: AbortSignal, + secretNames: readonly string[] = [], +): Promise => + validateProductionReadOnlyPermissions( + async (command, commandSignal) => + command.includes('--list') + ? { stdout: readOnlyAuthorizationRules, stderr: '', exitCode: 0 } + : run(command, commandSignal), + signal, + secretNames, + ) + +const validateSnapshot = ( + snapshot: VerificationSnapshot, + reconciledRevision: string, + expectedPromotion: ExpectedPromotion, + promotionRevision = reconciledRevision, +): void => { + validateProductionSnapshot(snapshot, reconciledRevision, expectedPromotion, promotionRevision, verificationNowMs) +} + +const dependency = () => ({ status: 'AVAILABLE', checkedAt: '2026-07-30T06:52:25.210Z', error: null }) + +const baseSnapshot = (): VerificationSnapshot => ({ + application: { + metadata: { name: 'bayn' }, + spec: { + destination: { namespace: 'bayn' }, + source: { + path: 'argocd/applications/bayn', + repoURL: 'https://github.com/proompteng/lab.git', + targetRevision: 'main', + }, + }, + status: { + sync: { status: 'Synced', revision: expectedArgoRevision }, + health: { status: 'Healthy' }, + operationState: { + phase: 'Succeeded', + syncResult: { + revision: expectedArgoRevision, + resources: [ + { + group: 'apps', + kind: 'Deployment', + name: 'bayn', + namespace: 'bayn', + status: 'Synced', + }, + ], + }, + }, + summary: { images: [imageReference] }, + resources: [{ group: 'apps', kind: 'Deployment', name: 'bayn', namespace: 'bayn', status: 'Synced' }], + }, + }, + deployment: { + metadata: { name: 'bayn', generation: 101 }, + spec: { + replicas: 1, + template: { + spec: { + containers: [ + { + name: 'bayn', + image: imageReference, + env: [ + { name: 'BAYN_CODE_REVISION', value: sourceRevision }, + { name: 'BAYN_IMAGE_DIGEST', value: digest }, + ], + }, + ], + }, + }, + }, + status: { + observedGeneration: 101, + replicas: 1, + updatedReplicas: 1, + readyReplicas: 1, + availableReplicas: 1, + unavailableReplicas: 0, + terminatingReplicas: 0, + }, + }, + pods: { + items: [ + { + metadata: { name: 'bayn-current' }, + spec: { + containers: [ + { + name: 'bayn', + image: imageReference, + env: [ + { name: 'BAYN_CODE_REVISION', value: sourceRevision }, + { name: 'BAYN_IMAGE_DIGEST', value: digest }, + ], + }, + ], + }, + status: { + phase: 'Running', + conditions: [{ type: 'Ready', status: 'True' }], + containerStatuses: [ + { + name: 'bayn', + ready: true, + started: true, + restartCount: 0, + imageID: `${repository}@${digest}`, + state: { running: { startedAt: '2026-07-30T06:49:41Z' } }, + lastState: {}, + }, + ], + }, + }, + ], + }, + readiness: { + ready: true, + status: 'READY', + checkedAt: '2026-07-30T06:52:25.210Z', + probeSequence: 6, + failedDependencies: [], + }, + metrics: ` +# HELP bayn_reconciliation_stale_threshold_seconds Configured reconciliation staleness threshold. +# TYPE bayn_reconciliation_stale_threshold_seconds gauge +bayn_reconciliation_stale_threshold_seconds 120 +`, + status: { + service: 'bayn', + operational: { + status: 'READY', + ready: true, + probeSequence: 6, + checkedAt: '2026-07-30T06:52:25.210Z', + }, + dependencies: { + postgresql: dependency(), + signal: dependency(), + tigerBeetle: dependency(), + evidence: dependency(), + cycle: dependency(), + cycleRunner: dependency(), + }, + autonomousCycleLoop: { + configured: true, + startedAt: '2026-07-30T06:49:45.348Z', + lastPass: { + result: 'SUCCESS', + observedAt: '2026-07-30T06:52:27.464Z', + outcome: 'NOT_DUE', + }, + }, + broker: { + configured: true, + accountBound: true, + readAvailable: true, + checkedAt: '2026-07-30T06:52:25.210Z', + executionEligible: false, + executionDisabledReason: 'BROKER_ACCESS_READ_ONLY', + reasonCode: null, + error: null, + }, + authority: { + brokerEnvironment: 'sandbox', + brokerAccess: 'read-only', + capitalAuthority: 'none', + durable: { + available: true, + configured: true, + maximum: 'observe', + effective: 'observe', + kill: 'clear', + reason: null, + updatedAt: '2026-07-28T06:49:28.305Z', + }, + brokerOrders: false, + capitalPromotion: false, + }, + cycle: { + observationAvailable: true, + condition: 'WAITING', + reason: 'NO_CYCLE_RECORDED', + checkedAt: '2026-07-30T06:52:25.210Z', + reconciliation: { + reconciliationId: 'a4cbd66fe988e2fc1c9e9d0e29fe10cfb00c6e64c6ea248a5659342c3139ca23', + status: 'EXACT', + discrepancyCount: 0, + reconciledAt: '2026-07-30T06:51:55.079Z', + coversLatestMutation: true, + }, + reconciliationCoversLatestMutation: true, + reconciliationAgeMs: 30_131, + mutations: { + eventCount: 0, + unresolvedCount: 0, + oldestUnresolvedAt: null, + latestOccurredAt: null, + }, + zeroMutation: true, + alerts: { + cycleStalled: false, + cycleFailed: false, + unknownMutationStale: false, + reconciliationBlocked: false, + killActive: false, + authorityIncoherent: false, + }, + error: null, + }, + build: { + sourceRevision, + image: { repository, digest }, + verification: 'embedded', + }, + error: null, + }, +}) + +const clone = (): VerificationSnapshot => structuredClone(baseSnapshot()) + +const captureFailure = (operation: () => void): VerificationFailure => { + try { + operation() + } catch (error) { + expect(error).toBeInstanceOf(VerificationFailure) + return error as VerificationFailure + } + throw new Error('expected verifier failure') +} + +describe('manifest contract', () => { + test('reads the exact full source, tag, and digest', () => { + expect(expected).toEqual({ + sourceRevision, + tag, + digest, + repository, + imageReference, + secretNames: ['bayn-alpaca-auth', 'bayn-db-ca'], + }) + }) + + test('rejects a deployment without production Secret references', () => { + const withoutSecrets = deploymentManifest + .replace(/\n - name: BAYN_ALPACA_ACCOUNT_ID[\s\S]*? key: account-id/, '') + .replace(/\n volumes:[\s\S]*? secretName: bayn-db-ca/, '') + expect(captureFailure(() => parseExpectedPromotion(kustomization, withoutSecrets)).code).toBe('INVALID_MANIFEST') + }) + + test('rejects truncated or mutable tags', () => { + const failure = captureFailure(() => + parseExpectedPromotion(kustomization.replace(tag, 'latest'), deploymentManifest), + ) + expect(failure.code).toBe('INVALID_MANIFEST') + const truncated = captureFailure(() => + parseExpectedPromotion(kustomization.replace(tag, `sha-${sourceRevision.slice(0, 12)}`), deploymentManifest), + ) + expect(truncated.code).toBe('INVALID_MANIFEST') + }) +}) + +describe('production snapshot', () => { + test('accepts exact source and digest with a successful idle NOT_DUE pass', () => { + expect(() => validateSnapshot(baseSnapshot(), expectedArgoRevision, expected)).not.toThrow() + }) + + test('accepts a verified descendant Argo revision with unchanged manifests and promotion history', () => { + const snapshot = clone() + const application = snapshot.application as any + application.status.sync.revision = descendantArgoRevision + application.status.operationState.syncResult.revision = descendantArgoRevision + application.status.history = [ + { + revision: expectedArgoRevision, + deployedAt: '2026-07-30T06:49:31Z', + source: { + path: 'argocd/applications/bayn', + repoURL: 'https://github.com/proompteng/lab.git', + targetRevision: 'main', + }, + }, + ] + expect(readArgoSyncRevision(snapshot.application)).toBe(descendantArgoRevision) + expect(() => validateSnapshot(snapshot, descendantArgoRevision, expected, expectedArgoRevision)).not.toThrow() + }) + + test('accepts a verified descendant when Argo skipped the intermediate promotion history', () => { + const snapshot = clone() + const application = snapshot.application as any + application.status.sync.revision = descendantArgoRevision + application.status.operationState.syncResult.revision = descendantArgoRevision + delete application.status.history + + expect(() => validateSnapshot(snapshot, descendantArgoRevision, expected, expectedArgoRevision)).not.toThrow() + }) + + test('accepts a descendant selective sync that omits the unchanged Bayn Deployment', () => { + const snapshot = clone() + const application = snapshot.application as any + application.status.sync.revision = descendantArgoRevision + application.status.operationState.syncResult.revision = descendantArgoRevision + application.status.operationState.syncResult.resources = [ + { + group: '', + kind: 'ConfigMap', + name: 'bayn-egress-proxy-selective-sync', + namespace: 'bayn', + status: 'Synced', + }, + ] + application.status.history = [ + { + revision: expectedArgoRevision, + deployedAt: '2026-07-30T06:49:31Z', + source: { + path: 'argocd/applications/bayn', + repoURL: 'https://github.com/proompteng/lab.git', + targetRevision: 'main', + }, + }, + ] + + expect(() => validateSnapshot(snapshot, descendantArgoRevision, expected, expectedArgoRevision)).not.toThrow() + }) + + test('accepts an exact selective sync that omits an unchanged Bayn Deployment', () => { + const snapshot = clone() + ;(snapshot.application as any).status.operationState.syncResult.resources = [] + expect(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).not.toThrow() + }) + + test('rejects a Bayn Deployment entry that is present but not Synced', () => { + const snapshot = clone() + ;(snapshot.application as any).status.operationState.syncResult.resources[0].status = 'OutOfSync' + expect(captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).code).toBe( + 'ARGO_NOT_CONVERGED', + ) + }) + + test('rejects the known Argo-new-revision old-image false positive', () => { + const snapshot = clone() + const deployment = snapshot.deployment as any + const pod = (snapshot.pods as any).items[0] + const oldImage = `${repository}:sha-${oldSourceRevision}@${oldDigest}` + deployment.spec.template.spec.containers[0].image = oldImage + deployment.spec.template.spec.containers[0].env[0].value = oldSourceRevision + deployment.spec.template.spec.containers[0].env[1].value = oldDigest + pod.spec.containers[0].image = oldImage + pod.spec.containers[0].env[0].value = oldSourceRevision + pod.spec.containers[0].env[1].value = oldDigest + pod.status.containerStatuses[0].imageID = `${repository}@${oldDigest}` + expect(captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).code).toBe( + 'DEPLOYMENT_NOT_CONVERGED', + ) + }) + + test('rejects stale Argo reconciliation', () => { + const snapshot = clone() + ;(snapshot.application as any).status.sync.revision = oldSourceRevision + expect(captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).code).toBe( + 'ARGO_NOT_CONVERGED', + ) + }) + + test('rejects any container restart or termination evidence', () => { + const snapshot = clone() + ;(snapshot.pods as any).items[0].status.containerStatuses[0].restartCount = 1 + expect(captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).code).toBe( + 'PRODUCTION_CONTRACT_VIOLATION', + ) + }) + + test('rejects unhealthy readiness and dependencies', () => { + const readiness = clone() + ;(readiness.readiness as any).ready = false + expect(captureFailure(() => validateSnapshot(readiness, expectedArgoRevision, expected)).code).toBe( + 'ENDPOINT_UNAVAILABLE', + ) + + const dependencyFailure = clone() + ;(dependencyFailure.status as any).dependencies.signal.status = 'UNAVAILABLE' + expect(captureFailure(() => validateSnapshot(dependencyFailure, expectedArgoRevision, expected)).code).toBe( + 'ENDPOINT_UNAVAILABLE', + ) + + const futureDependencyFailure = clone() + ;(futureDependencyFailure.status as any).dependencies.futureSafetyRead = { + status: 'UNAVAILABLE', + checkedAt: '2026-07-30T06:52:25.210Z', + error: 'redacted', + } + expect(captureFailure(() => validateSnapshot(futureDependencyFailure, expectedArgoRevision, expected)).code).toBe( + 'ENDPOINT_UNAVAILABLE', + ) + }) + + test.each([ + [ + 'broker access', + (snapshot: VerificationSnapshot) => ((snapshot.status as any).authority.brokerAccess = 'mutation'), + ], + [ + 'capital authority', + (snapshot: VerificationSnapshot) => ((snapshot.status as any).authority.capitalAuthority = 'paper'), + ], + ['broker orders', (snapshot: VerificationSnapshot) => ((snapshot.status as any).authority.brokerOrders = true)], + [ + 'capital promotion', + (snapshot: VerificationSnapshot) => ((snapshot.status as any).authority.capitalPromotion = true), + ], + [ + 'durable maximum', + (snapshot: VerificationSnapshot) => ((snapshot.status as any).authority.durable.maximum = 'paper'), + ], + ])('rejects authority escalation: %s', (_name, mutate) => { + const snapshot = clone() + mutate(snapshot) + expect(captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).code).toBe( + 'PRODUCTION_CONTRACT_VIOLATION', + ) + }) + + test('rejects any broker/account mutation evidence', () => { + const snapshot = clone() + ;(snapshot.status as any).cycle.mutations.eventCount = 1 + ;(snapshot.status as any).cycle.zeroMutation = false + expect(captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).code).toBe( + 'PRODUCTION_CONTRACT_VIOLATION', + ) + }) + + test('rejects missing required reconciliation without forcing a DUE cycle', () => { + const snapshot = clone() + ;(snapshot.status as any).cycle.reconciliation = null + expect(captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).code).toBe( + 'ENDPOINT_UNAVAILABLE', + ) + expect((snapshot.status as any).autonomousCycleLoop.lastPass.outcome).toBe('NOT_DUE') + }) + + test('rejects reconciliation at or beyond the live stale threshold', () => { + const snapshot = clone() + ;(snapshot.status as any).cycle.reconciliationAgeMs = 120_000 + expect(captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).code).toBe( + 'ENDPOINT_UNAVAILABLE', + ) + }) + + test('rejects missing or ambiguous reconciliation threshold metrics', () => { + const missing = clone() + ;(missing as any).metrics = '# no reconciliation threshold\n' + expect(captureFailure(() => validateSnapshot(missing, expectedArgoRevision, expected)).code).toBe( + 'ENDPOINT_UNAVAILABLE', + ) + + const duplicate = clone() + ;(duplicate as any).metrics = [ + 'bayn_reconciliation_stale_threshold_seconds 120', + 'bayn_reconciliation_stale_threshold_seconds 121', + ].join('\n') + expect(captureFailure(() => validateSnapshot(duplicate, expectedArgoRevision, expected)).code).toBe( + 'ENDPOINT_UNAVAILABLE', + ) + }) + + test.each([ + [ + 'readiness probe', + (snapshot: VerificationSnapshot, staleAt: string) => ((snapshot.readiness as any).checkedAt = staleAt), + ], + [ + 'operational health probe', + (snapshot: VerificationSnapshot, staleAt: string) => ((snapshot.status as any).operational.checkedAt = staleAt), + ], + [ + 'dependency health probe', + (snapshot: VerificationSnapshot, staleAt: string) => + ((snapshot.status as any).dependencies.postgresql.checkedAt = staleAt), + ], + [ + 'autonomous loop pass', + (snapshot: VerificationSnapshot, staleAt: string) => + ((snapshot.status as any).autonomousCycleLoop.lastPass.observedAt = staleAt), + ], + [ + 'broker probe', + (snapshot: VerificationSnapshot, staleAt: string) => ((snapshot.status as any).broker.checkedAt = staleAt), + ], + [ + 'cycle projection', + (snapshot: VerificationSnapshot, staleAt: string) => ((snapshot.status as any).cycle.checkedAt = staleAt), + ], + [ + 'reconciliation receipt', + (snapshot: VerificationSnapshot, staleAt: string) => + ((snapshot.status as any).cycle.reconciliation.reconciledAt = staleAt), + ], + ])('rejects stale cached %s evidence', (_name, mutate) => { + const snapshot = clone() + mutate(snapshot, '2026-07-30T06:50:29.999Z') + ;(snapshot.status as any).cycle.reconciliationAgeMs = 1 + expect(captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).code).toBe( + 'ENDPOINT_UNAVAILABLE', + ) + }) + + test('rejects timestamps beyond the bounded clock-skew allowance', () => { + const snapshot = clone() + ;(snapshot.status as any).operational.checkedAt = '2026-07-30T06:52:35.001Z' + expect(captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).code).toBe( + 'ENDPOINT_UNAVAILABLE', + ) + }) + + test('rejects missing required status fields', () => { + const snapshot = clone() + delete (snapshot.status as any).cycle.alerts.killActive + expect(captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).code).toBe( + 'PRODUCTION_CONTRACT_VIOLATION', + ) + }) + + test('rejects any newly added active alert', () => { + const snapshot = clone() + ;(snapshot.status as any).cycle.alerts.futureSafetyAlert = true + expect(captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)).code).toBe( + 'PRODUCTION_CONTRACT_VIOLATION', + ) + }) + + test('rejects raw broker identity fields without leaking the value', () => { + const snapshot = clone() + ;(snapshot.status as any).broker.accountId = 'raw-broker-account-123' + const failure = captureFailure(() => validateSnapshot(snapshot, expectedArgoRevision, expected)) + expect(failure.message).not.toContain('raw-broker-account-123') + expect(failure.code).toBe('PRODUCTION_CONTRACT_VIOLATION') + }) +}) + +describe('bounded execution', () => { + test('times out with the last precise blocker', async () => { + let now = 0 + let attempts = 0 + const controller = new AbortController() + await expect( + retryVerification( + async () => { + attempts += 1 + throw new VerificationFailure('ARGO_NOT_CONVERGED', 'stale revision', true) + }, + controller.signal, + { + deadlineMs: 25, + intervalMs: 10, + now: () => now, + sleep: async (milliseconds) => { + now += milliseconds + }, + }, + ), + ).rejects.toMatchObject({ code: 'VERIFICATION_TIMEOUT' }) + expect(attempts).toBe(3) + }) + + test('fails closed when interrupted', async () => { + const controller = new AbortController() + controller.abort() + await expect( + retryVerification(async () => undefined, controller.signal, { deadlineMs: 100, intervalMs: 10 }), + ).rejects.toMatchObject({ code: 'VERIFICATION_INTERRUPTED' }) + }) + + test('rejects an operation that reports success after the configured deadline', async () => { + let now = 0 + const controller = new AbortController() + await expect( + runWithinDeadline( + async () => { + now = 11 + }, + controller.signal, + 10, + () => now, + ), + ).rejects.toMatchObject({ code: 'VERIFICATION_TIMEOUT' }) + }) + + test('cancels a stalled subprocess when the remaining deadline expires', async () => { + const controller = new AbortController() + const startedAt = Date.now() + await expect( + retryVerification( + async (signal) => { + await runCommand( + ['bun', '-e', "process.on('SIGTERM', () => process.exit(0)); setInterval(() => undefined, 1_000)"], + signal, + ) + }, + controller.signal, + { deadlineMs: 50, intervalMs: 10 }, + ), + ).rejects.toMatchObject({ code: 'VERIFICATION_TIMEOUT' }) + expect(Date.now() - startedAt).toBeLessThan(1_000) + }, 2_000) +}) + +describe('Argo revision lineage', () => { + test('accepts the exact promotion revision without Git reads', async () => { + const controller = new AbortController() + let calls = 0 + const run = async () => { + calls += 1 + return { stdout: '', stderr: '', exitCode: 0 } + } + await expect( + verifyArgoRevision(run, controller.signal, '/repo', expectedArgoRevision, expectedArgoRevision), + ).resolves.toBeUndefined() + expect(calls).toBe(0) + }) + + test('accepts a current-main descendant only when Bayn manifests are unchanged', async () => { + const controller = new AbortController() + const commands: readonly string[][] = [] as string[][] + const run = async (command: readonly string[]) => { + ;(commands as string[][]).push([...command]) + return { stdout: '', stderr: '', exitCode: 0 } + } + await expect( + verifyArgoRevision(run, controller.signal, '/repo', expectedArgoRevision, descendantArgoRevision), + ).resolves.toBeUndefined() + expect(commands).toHaveLength(4) + expect(commands[3]).toContain('argocd/applications/bayn/deployment.yaml') + expect(commands[3]).toContain('argocd/applications/bayn/kustomization.yaml') + }) + + test('rejects a stale or divergent Argo revision', async () => { + const controller = new AbortController() + let call = 0 + const run = async () => { + call += 1 + return { stdout: '', stderr: '', exitCode: call === 3 ? 1 : 0 } + } + await expect( + verifyArgoRevision(run, controller.signal, '/repo', expectedArgoRevision, descendantArgoRevision), + ).rejects.toMatchObject({ code: 'ARGO_NOT_CONVERGED' }) + }) + + test('rejects a descendant that superseded the Bayn manifests', async () => { + const controller = new AbortController() + let call = 0 + const run = async () => { + call += 1 + return { stdout: '', stderr: '', exitCode: call === 4 ? 1 : 0 } + } + await expect( + verifyArgoRevision(run, controller.signal, '/repo', expectedArgoRevision, descendantArgoRevision), + ).rejects.toMatchObject({ code: 'PRODUCTION_CONTRACT_VIOLATION' }) + }) +}) + +describe('redaction and permissions', () => { + test('redacts tokens, credentials, and raw account identities', () => { + const redacted = redactSensitive( + 'Authorization: Bearer abc.def token=secret-value accountId=broker-123 password=hunter2', + ) + expect(redacted).not.toContain('abc.def') + expect(redacted).not.toContain('secret-value') + expect(redacted).not.toContain('broker-123') + expect(redacted).not.toContain('hunter2') + }) + + test('requires reads and rejects inherited write authority', async () => { + const controller = new AbortController() + const readOnly = async (command: readonly string[]) => { + const allowedRead = ['get', 'list'].includes(command[3] ?? '') && command[4] !== 'secrets' + return { stdout: allowedRead ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + await expect(validateReadOnlyPermissions(readOnly, controller.signal)).resolves.toBeUndefined() + + const writable = async () => ({ stdout: 'yes\n', stderr: '', exitCode: 0 }) + await expect(validateReadOnlyPermissions(writable, controller.signal)).rejects.toMatchObject({ + code: 'RBAC_DENIED', + }) + }) + + test('fails closed when a required read probe is indeterminate', async () => { + const controller = new AbortController() + let firstProbe = true + const run = async (command: readonly string[]) => { + if (firstProbe) { + firstProbe = false + return { stdout: 'yes\n', stderr: 'authorization warning\n', exitCode: 0 } + } + const verb = command[3] ?? '' + const resource = command[4] ?? '' + const requiredRead = ['get', 'list'].includes(verb) && resource !== 'secrets' + return { stdout: requiredRead ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + + await expect(validateReadOnlyPermissions(run, controller.signal)).rejects.toMatchObject({ + code: 'RBAC_DENIED', + }) + }) + + test.each([ + ['update', 'applications.argoproj.io', 'bayn'], + ['patch', 'deployments.apps', 'bayn'], + ['delete', 'services', 'bayn'], + ['patch', 'endpoints', 'bayn'], + ['get', 'secrets', 'bayn-alpaca-auth'], + ['update', 'secrets', 'bayn-db-ca'], + ])('rejects resourceNames-scoped %s authority for %s/%s', async (grantedVerb, grantedResource, grantedName) => { + const controller = new AbortController() + const run = async (command: readonly string[]) => { + const verb = command[3] ?? '' + const resource = command[4] ?? '' + const resourceNameAt = command.indexOf('--resource-name') + const resourceName = resourceNameAt === -1 ? undefined : command[resourceNameAt + 1] + if (verb === grantedVerb && resource === grantedResource && resourceName === grantedName) { + return { stdout: 'yes\n', stderr: '', exitCode: 0 } + } + const requiredRead = + (verb === 'get' && ['applications.argoproj.io', 'deployments.apps', 'services/proxy'].includes(resource)) || + (verb === 'list' && resource === 'pods') + return { stdout: requiredRead ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + + await expect(validateReadOnlyPermissions(run, controller.signal, expected.secretNames)).rejects.toMatchObject({ + code: 'RBAC_DENIED', + }) + }) + + test('probes parent Service and endpoint-routing resources', async () => { + const controller = new AbortController() + const commands: string[][] = [] + const run = async (command: readonly string[]) => { + commands.push([...command]) + const verb = command[3] ?? '' + const resource = command[4] ?? '' + const requiredRead = + (verb === 'get' && ['applications.argoproj.io', 'deployments.apps', 'services/proxy'].includes(resource)) || + (verb === 'list' && resource === 'pods') + return { stdout: requiredRead ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + + await expect(validateReadOnlyPermissions(run, controller.signal, expected.secretNames)).resolves.toBeUndefined() + for (const resource of ['services', 'services/status', 'endpoints', 'endpointslices.discovery.k8s.io']) { + expect(commands.some((command) => command[4] === resource)).toBe(true) + } + expect( + commands.some( + (command) => command[4] === 'services' && command.includes('--resource-name') && command.includes('bayn'), + ), + ).toBe(true) + }) + + test('fails closed on resourceNames-scoped mutation exposed by the full rules audit', async () => { + const controller = new AbortController() + const readOnlyRules = [ + 'selfsubjectaccessreviews.authorization.k8s.io [] [] [create]', + 'deployments.apps [] [] [get list watch]', + ].join('\n') + const run = async (command: readonly string[]) => { + if (command.includes('--list')) { + const namespace = command[command.indexOf('-n') + 1] + return { + stdout: + namespace === 'bayn' + ? `${readOnlyRules}\nendpointslices.discovery.k8s.io [] [bayn-abcde] [patch]\n` + : `${readOnlyRules}\n`, + stderr: '', + exitCode: 0, + } + } + const verb = command[3] ?? '' + const resource = command[4] ?? '' + const requiredRead = + (verb === 'get' && ['applications.argoproj.io', 'deployments.apps', 'services/proxy'].includes(resource)) || + (verb === 'list' && resource === 'pods') + return { stdout: requiredRead ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + + await expect( + validateProductionReadOnlyPermissions(run, controller.signal, expected.secretNames), + ).rejects.toMatchObject({ code: 'RBAC_DENIED' }) + }) + + test.each([ + ['broad', 'secrets [] [] [get]'], + ['resourceNames-scoped', 'secrets [] [operator-created-secret] [get]'], + ['wildcard resource', '* [] [operator-created-secret] [get]'], + ['API-group wildcard resource', '*.* [] [operator-created-secret] [get]'], + ['Secret resource in wildcard group', 'secrets.* [] [operator-created-secret] [get]'], + ['combined resource', 'configmaps,secrets [] [operator-created-secret] [get]'], + ])('rejects %s Secret disclosure in the full rules audit', async (_name, secretRule) => { + const controller = new AbortController() + const run = async (command: readonly string[]) => { + if (command.includes('--list')) { + return { + stdout: `${readOnlyAuthorizationRules}\n${secretRule}\n`, + stderr: '', + exitCode: 0, + } + } + const verb = command[3] ?? '' + const resource = command[4] ?? '' + const requiredRead = + (verb === 'get' && ['applications.argoproj.io', 'deployments.apps', 'services/proxy'].includes(resource)) || + (verb === 'list' && resource === 'pods') + return { stdout: requiredRead ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + + await expect( + validateProductionReadOnlyPermissions(run, controller.signal, expected.secretNames), + ).rejects.toMatchObject({ code: 'RBAC_DENIED' }) + }) + + test('accepts only read-only namespace rule inventories plus self-review creation', async () => { + const controller = new AbortController() + const run = async (command: readonly string[]) => { + if (command.includes('--list')) { + return { + stdout: [ + 'selfsubjectreviews.authentication.k8s.io [] [] [create]', + 'selfsubjectaccessreviews.authorization.k8s.io [] [] [create]', + 'selfsubjectrulesreviews.authorization.k8s.io [] [] [create]', + 'deployments.apps [] [] [get list watch]', + '*.apps [] [] [get list watch]', + 'secrets.apps [] [] [get]', + ' [/healthz] [] [get]', + ].join('\n'), + stderr: '', + exitCode: 0, + } + } + const verb = command[3] ?? '' + const resource = command[4] ?? '' + const requiredRead = + (verb === 'get' && ['applications.argoproj.io', 'deployments.apps', 'services/proxy'].includes(resource)) || + (verb === 'list' && resource === 'pods') + return { stdout: requiredRead ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + + await expect( + validateProductionReadOnlyPermissions(run, controller.signal, expected.secretNames), + ).resolves.toBeUndefined() + }) + + test.each([ + ['API failure', { stdout: 'no\n', stderr: 'server unavailable\n', exitCode: 1 }], + ['unexpected output', { stdout: 'unknown\n', stderr: '', exitCode: 0 }], + ['stderr with no decision', { stdout: 'no\n', stderr: 'warning\n', exitCode: 0 }], + ])('fails closed when a forbidden write probe is indeterminate: %s', async (_name, indeterminate) => { + const controller = new AbortController() + let forbiddenProbeSeen = false + const run = async (command: readonly string[]) => { + const verb = command[3] ?? '' + const resource = command[4] ?? '' + const requiredRead = ['get', 'list'].includes(verb) && resource !== 'secrets' + if (requiredRead) return { stdout: 'yes\n', stderr: '', exitCode: 0 } + if (!forbiddenProbeSeen) { + forbiddenProbeSeen = true + return indeterminate + } + return { stdout: 'no\n', stderr: '', exitCode: 0 } + } + await expect(validateReadOnlyPermissions(run, controller.signal)).rejects.toMatchObject({ + code: 'RBAC_DENIED', + }) + }) + + test('fails closed when the forbidden secret-read probe is indeterminate', async () => { + const controller = new AbortController() + const run = async (command: readonly string[]) => { + const verb = command[3] ?? '' + const resource = command[4] ?? '' + if (resource === 'secrets') return { stdout: '', stderr: 'authorization API unavailable\n', exitCode: 1 } + const requiredRead = ['get', 'list'].includes(verb) + return { stdout: requiredRead ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + await expect(validateReadOnlyPermissions(run, controller.signal)).rejects.toMatchObject({ + code: 'RBAC_DENIED', + }) + }) + + test.each(['get', 'list', 'watch'])('rejects %s access to Secrets', async (grantedVerb) => { + const controller = new AbortController() + const run = async (command: readonly string[]) => { + const verb = command[3] ?? '' + const resource = command[4] ?? '' + if (resource === 'secrets') { + return { stdout: verb === grantedVerb ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + const requiredRead = ['get', 'list'].includes(verb) + return { stdout: requiredRead ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + + await expect(validateReadOnlyPermissions(run, controller.signal)).rejects.toMatchObject({ + code: 'RBAC_DENIED', + }) + }) + + test.each(['create', 'update', 'patch', 'delete', 'deletecollection'])( + 'rejects %s write access to Secrets', + async (grantedVerb) => { + const controller = new AbortController() + const run = async (command: readonly string[]) => { + const verb = command[3] ?? '' + const resource = command[4] ?? '' + if (resource === 'secrets') { + return { stdout: verb === grantedVerb ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + const requiredRead = ['get', 'list'].includes(verb) + return { stdout: requiredRead ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + + await expect(validateReadOnlyPermissions(run, controller.signal)).rejects.toMatchObject({ + code: 'RBAC_DENIED', + }) + }, + ) + + test('rejects deletecollection authority independently of delete', async () => { + const controller = new AbortController() + const run = async (command: readonly string[]) => { + const verb = command[3] ?? '' + const resource = command[4] ?? '' + if (resource === 'secrets') return { stdout: 'no\n', stderr: '', exitCode: 0 } + if (verb === 'deletecollection') return { stdout: 'yes\n', stderr: '', exitCode: 0 } + const requiredRead = ['get', 'list'].includes(verb) + return { stdout: requiredRead ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + + await expect(validateReadOnlyPermissions(run, controller.signal)).rejects.toMatchObject({ + code: 'RBAC_DENIED', + }) + }) + + test.each([ + ['create', 'services/proxy'], + ['update', 'deployments.apps/scale'], + ['create', 'pods/eviction'], + ])('rejects %s authority on destructive subresource %s', async (grantedVerb, grantedResource) => { + const controller = new AbortController() + const run = async (command: readonly string[]) => { + const verb = command[3] ?? '' + const resource = command[4] ?? '' + if (resource === 'secrets') return { stdout: 'no\n', stderr: '', exitCode: 0 } + if (verb === grantedVerb && resource === grantedResource) { + return { stdout: 'yes\n', stderr: '', exitCode: 0 } + } + const requiredRead = ['get', 'list'].includes(verb) + return { stdout: requiredRead ? 'yes\n' : 'no\n', stderr: '', exitCode: 0 } + } + + await expect(validateReadOnlyPermissions(run, controller.signal)).rejects.toMatchObject({ + code: 'RBAC_DENIED', + }) + }) + + test('workflow is main-only, path-scoped, and GitHub read-only', async () => { + const workflow = await readFile( + join(import.meta.dir, '..', '.github/workflows/bayn-post-deploy-verify.yml'), + 'utf8', + ) + expect(workflow).toContain("- 'argocd/applications/bayn/deployment.yaml'") + expect(workflow).toContain("- 'argocd/applications/bayn/kustomization.yaml'") + expect(workflow).not.toContain('workflow_dispatch:') + expect(workflow).toContain('permissions:\n contents: read') + expect(workflow).not.toMatch(/contents:\s*write/) + expect(workflow).not.toMatch(/pull-requests:\s*write/) + expect(workflow).not.toMatch(/kubectl\s+(apply|create|delete|patch|replace|rollout|set)/) + expect(workflow).toContain('bun scripts/bayn-post-deploy-verify.ts') + }) + + test('Bayn CI detects, formats, lints, typechecks, and executes the verifier suite', async () => { + const workflow = await readFile(join(import.meta.dir, '..', '.github/workflows/bayn-ci.yml'), 'utf8') + expect(workflow).toContain('scripts/bayn-post-deploy-verify.ts') + expect(workflow).toContain('scripts/bayn-post-deploy-verify.test.ts') + expect(workflow).toContain('bunx oxfmt --check') + expect(workflow).toContain('bunx oxlint --config .oxlintrc.json') + expect(workflow).toContain('bun node_modules/typescript/bin/tsc --ignoreConfig --noEmit') + expect(workflow).toContain('bun test packages/scripts/src/bayn scripts/bayn-post-deploy-verify.test.ts') + }) +}) diff --git a/scripts/bayn-post-deploy-verify.ts b/scripts/bayn-post-deploy-verify.ts new file mode 100644 index 0000000000..90e65ee5b9 --- /dev/null +++ b/scripts/bayn-post-deploy-verify.ts @@ -0,0 +1,1192 @@ +#!/usr/bin/env bun + +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' + +const SOURCE_PATTERN = /^[0-9a-f]{40}$/ +const TAG_PATTERN = /^sha-([0-9a-f]{40})$/ +const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/ +const IMAGE_REPOSITORY = 'registry.ide-newton.ts.net/lab/bayn' +const MAXIMUM_CLOCK_SKEW_MS = 5_000 +const REQUIRED_DEPENDENCIES = ['postgresql', 'signal', 'tigerBeetle', 'evidence', 'cycle', 'cycleRunner'] as const +const REQUIRED_ALERTS = [ + 'cycleStalled', + 'cycleFailed', + 'unknownMutationStale', + 'reconciliationBlocked', + 'killActive', + 'authorityIncoherent', +] as const +const SENSITIVE_KEY_PATTERN = /(?:account.?id|broker.?identity|authorization|credential|key.?id|password|secret|token)/i + +type JsonRecord = Record + +export type ExpectedPromotion = { + readonly sourceRevision: string + readonly tag: string + readonly digest: string + readonly repository: string + readonly imageReference: string + readonly secretNames: readonly string[] +} + +export type VerificationSnapshot = { + readonly application: unknown + readonly deployment: unknown + readonly pods: unknown + readonly readiness: unknown + readonly status: unknown + readonly metrics: unknown +} + +export type VerificationFailureCode = + | 'ARGO_NOT_CONVERGED' + | 'DEPLOYMENT_NOT_CONVERGED' + | 'ENDPOINT_UNAVAILABLE' + | 'INVALID_MANIFEST' + | 'POD_NOT_CONVERGED' + | 'PRODUCTION_CONTRACT_VIOLATION' + | 'RBAC_DENIED' + | 'VERIFICATION_INTERRUPTED' + | 'VERIFICATION_TIMEOUT' + +export class VerificationFailure extends Error { + constructor( + readonly code: VerificationFailureCode, + message: string, + readonly retryable: boolean, + ) { + super(`${code}: ${message}`) + this.name = 'VerificationFailure' + } +} + +const record = (value: unknown, path: string): JsonRecord => { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new VerificationFailure('PRODUCTION_CONTRACT_VIOLATION', `${path} must be an object`, false) + } + return value as JsonRecord +} + +const array = (value: unknown, path: string): readonly unknown[] => { + if (!Array.isArray(value)) { + throw new VerificationFailure('PRODUCTION_CONTRACT_VIOLATION', `${path} must be an array`, false) + } + return value +} + +const string = (value: unknown, path: string): string => { + if (typeof value !== 'string' || value.length === 0) { + throw new VerificationFailure('PRODUCTION_CONTRACT_VIOLATION', `${path} must be a non-empty string`, false) + } + return value +} + +const integer = (value: unknown, path: string): number => { + if (typeof value !== 'number' || !Number.isInteger(value)) { + throw new VerificationFailure('PRODUCTION_CONTRACT_VIOLATION', `${path} must be an integer`, false) + } + return value +} + +const optionalInteger = (value: unknown, path: string): number => (value === undefined ? 0 : integer(value, path)) + +const fail = (code: VerificationFailureCode, message: string, retryable: boolean): never => { + throw new VerificationFailure(code, message, retryable) +} + +const requireEqual = ( + actual: unknown, + expected: string | number | boolean | null, + path: string, + code: VerificationFailureCode, + retryable: boolean, +): void => { + if (actual !== expected) fail(code, `${path} did not match the required value`, retryable) +} + +const requireFreshInstant = (value: unknown, path: string, nowMs: number, maximumAgeMs: number): number => { + const source = string(value, path) + const observedAtMs = Date.parse(source) + if (!Number.isFinite(observedAtMs)) { + fail('ENDPOINT_UNAVAILABLE', `${path} is not a valid instant`, true) + } + if (observedAtMs > nowMs + MAXIMUM_CLOCK_SKEW_MS) { + fail('ENDPOINT_UNAVAILABLE', `${path} is unexpectedly in the future`, true) + } + if (nowMs - observedAtMs >= maximumAgeMs) { + fail('ENDPOINT_UNAVAILABLE', `${path} is stale`, true) + } + return observedAtMs +} + +const parseYaml = (source: string, path: string): JsonRecord => { + try { + return record(Bun.YAML.parse(source), path) + } catch (error) { + if (error instanceof VerificationFailure) throw error + return fail('INVALID_MANIFEST', `${path} is not valid YAML`, false) + } +} + +const envValues = (container: JsonRecord, path: string): ReadonlyMap => { + const entries = array(container.env, `${path}.env`) + const values = new Map() + for (const [index, entry] of entries.entries()) { + const item = record(entry, `${path}.env[${index}]`) + const name = string(item.name, `${path}.env[${index}].name`) + if (typeof item.value === 'string') values.set(name, item.value) + } + return values +} + +const collectSecretNames = (value: unknown, names = new Set()): ReadonlySet => { + if (Array.isArray(value)) { + for (const item of value) collectSecretNames(item, names) + return names + } + if (typeof value !== 'object' || value === null) return names + for (const [key, child] of Object.entries(value)) { + if ((key === 'secretKeyRef' || key === 'secretRef') && typeof child === 'object' && child !== null) { + const name = (child as JsonRecord).name + if (typeof name === 'string' && name.length > 0) names.add(name) + } else if (key === 'secretName' && typeof child === 'string' && child.length > 0) { + names.add(child) + } + collectSecretNames(child, names) + } + return names +} + +const namedContainer = (containers: unknown, name: string, path: string): JsonRecord => { + const matches = array(containers, path).filter((candidate) => { + const item = record(candidate, path) + return item.name === name + }) + if (matches.length !== 1) + fail('PRODUCTION_CONTRACT_VIOLATION', `${path} must contain exactly one ${name} container`, false) + return record(matches[0], `${path}.${name}`) +} + +export const parseExpectedPromotion = (kustomizationSource: string, deploymentSource: string): ExpectedPromotion => { + const kustomization = parseYaml(kustomizationSource, 'kustomization') + const images = array(kustomization.images, 'kustomization.images') + if (images.length !== 1) fail('INVALID_MANIFEST', 'kustomization.images must contain exactly one image', false) + const image = record(images[0], 'kustomization.images[0]') + const repository = string(image.newName, 'kustomization.images[0].newName') + const tag = string(image.newTag, 'kustomization.images[0].newTag') + const digest = string(image.digest, 'kustomization.images[0].digest') + requireEqual(image.name, IMAGE_REPOSITORY, 'kustomization.images[0].name', 'INVALID_MANIFEST', false) + requireEqual(repository, IMAGE_REPOSITORY, 'kustomization.images[0].newName', 'INVALID_MANIFEST', false) + const tagMatch = TAG_PATTERN.exec(tag) + if (tagMatch === null) + return fail('INVALID_MANIFEST', 'image tag must contain the full immutable source revision', false) + const sourceFromTag = tagMatch?.[1] + if (sourceFromTag === undefined) return fail('INVALID_MANIFEST', 'image tag is missing the source revision', false) + if (!DIGEST_PATTERN.test(digest)) fail('INVALID_MANIFEST', 'image digest must be a full sha256 digest', false) + + const deployment = parseYaml(deploymentSource, 'deployment') + const spec = record(deployment.spec, 'deployment.spec') + const template = record(spec.template, 'deployment.spec.template') + const podSpec = record(template.spec, 'deployment.spec.template.spec') + const container = namedContainer(podSpec.containers, 'bayn', 'deployment.spec.template.spec.containers') + requireEqual(container.image, repository, 'deployment container image base', 'INVALID_MANIFEST', false) + const env = envValues(container, 'deployment.spec.template.spec.containers.bayn') + const sourceRevision = env.get('BAYN_CODE_REVISION') + if (sourceRevision === undefined || !SOURCE_PATTERN.test(sourceRevision)) { + return fail('INVALID_MANIFEST', 'BAYN_CODE_REVISION must be a full source revision', false) + } + requireEqual(sourceRevision, sourceFromTag, 'BAYN_CODE_REVISION', 'INVALID_MANIFEST', false) + requireEqual(env.get('BAYN_IMAGE_REPOSITORY'), repository, 'BAYN_IMAGE_REPOSITORY', 'INVALID_MANIFEST', false) + requireEqual(env.get('BAYN_IMAGE_DIGEST'), digest, 'BAYN_IMAGE_DIGEST', 'INVALID_MANIFEST', false) + requireEqual(env.get('BAYN_BROKER_ENVIRONMENT'), 'sandbox', 'BAYN_BROKER_ENVIRONMENT', 'INVALID_MANIFEST', false) + requireEqual(env.get('BAYN_MAXIMUM_AUTHORITY'), 'OBSERVE', 'BAYN_MAXIMUM_AUTHORITY', 'INVALID_MANIFEST', false) + const secretNames = [...collectSecretNames(podSpec)].sort() + if (secretNames.length === 0) fail('INVALID_MANIFEST', 'Bayn deployment must reference its production Secrets', false) + + return { + sourceRevision, + tag, + digest, + repository, + imageReference: `${repository}:${tag}@${digest}`, + secretNames, + } +} + +const validateArgo = ( + applicationValue: unknown, + reconciledRevision: string, + promotionRevision: string, + expected: ExpectedPromotion, +): void => { + const application = record(applicationValue, 'application') + requireEqual( + record(application.metadata, 'application.metadata').name, + 'bayn', + 'application.metadata.name', + 'ARGO_NOT_CONVERGED', + true, + ) + const spec = record(application.spec, 'application.spec') + const source = record(spec.source, 'application.spec.source') + requireEqual(source.path, 'argocd/applications/bayn', 'application.spec.source.path', 'ARGO_NOT_CONVERGED', true) + requireEqual( + source.repoURL, + 'https://github.com/proompteng/lab.git', + 'application.spec.source.repoURL', + 'ARGO_NOT_CONVERGED', + true, + ) + requireEqual(source.targetRevision, 'main', 'application.spec.source.targetRevision', 'ARGO_NOT_CONVERGED', true) + requireEqual( + record(spec.destination, 'application.spec.destination').namespace, + 'bayn', + 'application.spec.destination.namespace', + 'ARGO_NOT_CONVERGED', + true, + ) + const status = record(application.status, 'application.status') + const sync = record(status.sync, 'application.status.sync') + requireEqual(sync.status, 'Synced', 'application.status.sync.status', 'ARGO_NOT_CONVERGED', true) + requireEqual(sync.revision, reconciledRevision, 'application.status.sync.revision', 'ARGO_NOT_CONVERGED', true) + requireEqual( + record(status.health, 'application.status.health').status, + 'Healthy', + 'application.status.health.status', + 'ARGO_NOT_CONVERGED', + true, + ) + const operation = record(status.operationState, 'application.status.operationState') + requireEqual(operation.phase, 'Succeeded', 'application.status.operationState.phase', 'ARGO_NOT_CONVERGED', true) + const syncResult = record(operation.syncResult, 'application.status.operationState.syncResult') + const operationRevision = string(syncResult.revision, 'application.status.operationState.syncResult.revision') + if (operationRevision !== promotionRevision && operationRevision !== reconciledRevision) { + fail('ARGO_NOT_CONVERGED', 'Argo operation does not prove the promotion or reconciled revision', true) + } + const operationDeployment = array( + syncResult.resources, + 'application.status.operationState.syncResult.resources', + ).find((candidate) => { + const resource = record(candidate, 'application.status.operationState.syncResult.resources[]') + return ( + resource.group === 'apps' && + resource.kind === 'Deployment' && + resource.name === 'bayn' && + resource.namespace === 'bayn' + ) + }) + if (operationDeployment !== undefined) { + const operationDeploymentRecord = record( + operationDeployment, + 'application.status.operationState.syncResult.resources.bayn', + ) + requireEqual( + operationDeploymentRecord.status, + 'Synced', + 'application.status.operationState.syncResult.resources.bayn.status', + 'ARGO_NOT_CONVERGED', + true, + ) + } + const images = array(record(status.summary, 'application.status.summary').images, 'application.status.summary.images') + if (!images.includes(expected.imageReference)) { + fail('ARGO_NOT_CONVERGED', 'Argo summary does not contain the exact promoted image', true) + } + const deploymentResource = array(status.resources, 'application.status.resources').find((candidate) => { + const resource = record(candidate, 'application.status.resources[]') + return ( + resource.group === 'apps' && + resource.kind === 'Deployment' && + resource.name === 'bayn' && + resource.namespace === 'bayn' + ) + }) + if (deploymentResource === undefined) { + fail('ARGO_NOT_CONVERGED', 'Argo does not report the Bayn Deployment resource', true) + } + requireEqual( + record(deploymentResource, 'application.status.resources.bayn').status, + 'Synced', + 'application.status.resources.bayn.status', + 'ARGO_NOT_CONVERGED', + true, + ) +} + +const validateDeployment = (deploymentValue: unknown, expected: ExpectedPromotion): void => { + const deployment = record(deploymentValue, 'deployment') + const metadata = record(deployment.metadata, 'deployment.metadata') + requireEqual(metadata.name, 'bayn', 'deployment.metadata.name', 'DEPLOYMENT_NOT_CONVERGED', true) + const spec = record(deployment.spec, 'deployment.spec') + requireEqual(spec.replicas, 1, 'deployment.spec.replicas', 'DEPLOYMENT_NOT_CONVERGED', true) + const template = record(spec.template, 'deployment.spec.template') + const podSpec = record(template.spec, 'deployment.spec.template.spec') + const container = namedContainer(podSpec.containers, 'bayn', 'deployment.spec.template.spec.containers') + requireEqual(container.image, expected.imageReference, 'deployment container image', 'DEPLOYMENT_NOT_CONVERGED', true) + const env = envValues(container, 'deployment.spec.template.spec.containers.bayn') + requireEqual( + env.get('BAYN_CODE_REVISION'), + expected.sourceRevision, + 'deployment source revision', + 'DEPLOYMENT_NOT_CONVERGED', + true, + ) + requireEqual( + env.get('BAYN_IMAGE_DIGEST'), + expected.digest, + 'deployment image digest', + 'DEPLOYMENT_NOT_CONVERGED', + true, + ) + const status = record(deployment.status, 'deployment.status') + requireEqual( + status.observedGeneration, + integer(metadata.generation, 'deployment.metadata.generation'), + 'deployment observedGeneration', + 'DEPLOYMENT_NOT_CONVERGED', + true, + ) + for (const key of ['replicas', 'updatedReplicas', 'readyReplicas', 'availableReplicas'] as const) { + requireEqual(status[key], 1, `deployment.status.${key}`, 'DEPLOYMENT_NOT_CONVERGED', true) + } + requireEqual( + optionalInteger(status.unavailableReplicas, 'deployment.status.unavailableReplicas'), + 0, + 'deployment unavailable replicas', + 'DEPLOYMENT_NOT_CONVERGED', + true, + ) + requireEqual( + optionalInteger(status.terminatingReplicas, 'deployment.status.terminatingReplicas'), + 0, + 'deployment terminating replicas', + 'DEPLOYMENT_NOT_CONVERGED', + true, + ) +} + +const validatePod = (podsValue: unknown, expected: ExpectedPromotion): void => { + const pods = record(podsValue, 'pods') + const items = array(pods.items, 'pods.items') + if (items.length !== 1) fail('POD_NOT_CONVERGED', 'exactly one Bayn pod must exist', true) + const pod = record(items[0], 'pods.items[0]') + const metadata = record(pod.metadata, 'pod.metadata') + if (metadata.deletionTimestamp !== undefined) fail('POD_NOT_CONVERGED', 'Bayn pod is terminating', true) + const spec = record(pod.spec, 'pod.spec') + const container = namedContainer(spec.containers, 'bayn', 'pod.spec.containers') + requireEqual(container.image, expected.imageReference, 'pod container image', 'POD_NOT_CONVERGED', true) + const env = envValues(container, 'pod.spec.containers.bayn') + requireEqual(env.get('BAYN_CODE_REVISION'), expected.sourceRevision, 'pod source revision', 'POD_NOT_CONVERGED', true) + requireEqual(env.get('BAYN_IMAGE_DIGEST'), expected.digest, 'pod image digest', 'POD_NOT_CONVERGED', true) + const status = record(pod.status, 'pod.status') + requireEqual(status.phase, 'Running', 'pod.status.phase', 'POD_NOT_CONVERGED', true) + const ready = array(status.conditions, 'pod.status.conditions').some((condition) => { + const item = record(condition, 'pod.status.conditions[]') + return item.type === 'Ready' && item.status === 'True' + }) + if (!ready) fail('POD_NOT_CONVERGED', 'Bayn pod is not Ready', true) + const containerStatus = namedContainer(status.containerStatuses, 'bayn', 'pod.status.containerStatuses') + requireEqual(containerStatus.ready, true, 'pod container ready', 'POD_NOT_CONVERGED', true) + requireEqual(containerStatus.started, true, 'pod container started', 'POD_NOT_CONVERGED', true) + requireEqual(containerStatus.restartCount, 0, 'pod container restartCount', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual( + containerStatus.imageID, + `${expected.repository}@${expected.digest}`, + 'pod container imageID', + 'POD_NOT_CONVERGED', + true, + ) + const state = record(containerStatus.state, 'pod container state') + if (state.running === undefined || state.waiting !== undefined || state.terminated !== undefined) { + fail('PRODUCTION_CONTRACT_VIOLATION', 'Bayn container is not in an uninterrupted running state', false) + } + const lastState = record(containerStatus.lastState ?? {}, 'pod container lastState') + if (Object.keys(lastState).length !== 0) { + fail('PRODUCTION_CONTRACT_VIOLATION', 'Bayn container has prior termination evidence', false) + } +} + +const validateReadiness = (readinessValue: unknown, nowMs: number, maximumEvidenceAgeMs: number): void => { + const readiness = record(readinessValue, 'readiness') + requireEqual(readiness.ready, true, 'readiness.ready', 'ENDPOINT_UNAVAILABLE', true) + requireEqual(readiness.status, 'READY', 'readiness.status', 'ENDPOINT_UNAVAILABLE', true) + if (array(readiness.failedDependencies, 'readiness.failedDependencies').length !== 0) { + fail('ENDPOINT_UNAVAILABLE', 'readiness reports failed dependencies', true) + } + if (integer(readiness.probeSequence, 'readiness.probeSequence') < 1) { + fail('ENDPOINT_UNAVAILABLE', 'readiness probe sequence has not advanced', true) + } + requireFreshInstant(readiness.checkedAt, 'readiness.checkedAt', nowMs, maximumEvidenceAgeMs) +} + +const reconciliationStaleThresholdMs = (metricsValue: unknown): number => { + if (typeof metricsValue !== 'string') { + return fail('ENDPOINT_UNAVAILABLE', 'Bayn metrics response must be text', true) + } + const samples = metricsValue + .split('\n') + .map((line: string) => line.trim()) + .filter((line: string) => line.startsWith('bayn_reconciliation_stale_threshold_seconds ')) + if (samples.length !== 1) { + return fail('ENDPOINT_UNAVAILABLE', 'Bayn reconciliation stale threshold metric must have exactly one sample', true) + } + const sample = samples[0] + if (sample === undefined) { + return fail('ENDPOINT_UNAVAILABLE', 'Bayn reconciliation stale threshold metric is missing', true) + } + const match = /^bayn_reconciliation_stale_threshold_seconds ([0-9]+(?:\.[0-9]+)?)$/.exec(sample) + if (match === null) { + return fail('ENDPOINT_UNAVAILABLE', 'Bayn reconciliation stale threshold metric is invalid', true) + } + const thresholdValue = match[1] + if (thresholdValue === undefined) { + return fail('ENDPOINT_UNAVAILABLE', 'Bayn reconciliation stale threshold metric has no value', true) + } + const thresholdSeconds = Number(thresholdValue) + const thresholdMs = thresholdSeconds * 1_000 + if (!Number.isFinite(thresholdMs) || thresholdMs <= 0) { + fail('ENDPOINT_UNAVAILABLE', 'Bayn reconciliation stale threshold metric must be positive', true) + } + return thresholdMs +} + +const assertNoSensitiveFields = (value: unknown, path = 'status'): void => { + if (Array.isArray(value)) { + value.forEach((item, index) => assertNoSensitiveFields(item, `${path}[${index}]`)) + return + } + if (typeof value !== 'object' || value === null) return + for (const [key, child] of Object.entries(value)) { + if (SENSITIVE_KEY_PATTERN.test(key)) { + fail('PRODUCTION_CONTRACT_VIOLATION', `${path} contains forbidden sensitive identity field`, false) + } + assertNoSensitiveFields(child, `${path}.${key}`) + } +} + +const validateAuthority = (authorityValue: unknown): void => { + const authority = record(authorityValue, 'status.authority') + requireEqual( + authority.brokerEnvironment, + 'sandbox', + 'authority.brokerEnvironment', + 'PRODUCTION_CONTRACT_VIOLATION', + false, + ) + requireEqual(authority.brokerAccess, 'read-only', 'authority.brokerAccess', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual(authority.capitalAuthority, 'none', 'authority.capitalAuthority', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual(authority.brokerOrders, false, 'authority.brokerOrders', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual(authority.capitalPromotion, false, 'authority.capitalPromotion', 'PRODUCTION_CONTRACT_VIOLATION', false) + const durable = record(authority.durable, 'status.authority.durable') + requireEqual(durable.available, true, 'authority.durable.available', 'PRODUCTION_CONTRACT_VIOLATION', false) + if (durable.configured === false) { + for (const field of ['maximum', 'effective', 'kill', 'reason', 'updatedAt'] as const) { + requireEqual(durable[field], null, `authority.durable.${field}`, 'PRODUCTION_CONTRACT_VIOLATION', false) + } + return + } + requireEqual(durable.configured, true, 'authority.durable.configured', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual(durable.maximum, 'observe', 'authority.durable.maximum', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual(durable.effective, 'observe', 'authority.durable.effective', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual(durable.kill, 'clear', 'authority.durable.kill', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual(durable.reason, null, 'authority.durable.reason', 'PRODUCTION_CONTRACT_VIOLATION', false) + string(durable.updatedAt, 'authority.durable.updatedAt') +} + +const validateStatus = ( + statusValue: unknown, + expected: ExpectedPromotion, + reconciliationStaleThresholdMs: number, + nowMs: number, +): void => { + assertNoSensitiveFields(statusValue) + const status = record(statusValue, 'status') + requireEqual(status.service, 'bayn', 'status.service', 'PRODUCTION_CONTRACT_VIOLATION', false) + const operational = record(status.operational, 'status.operational') + requireEqual(operational.status, 'READY', 'status.operational.status', 'ENDPOINT_UNAVAILABLE', true) + requireEqual(operational.ready, true, 'status.operational.ready', 'ENDPOINT_UNAVAILABLE', true) + if (integer(operational.probeSequence, 'status.operational.probeSequence') < 1) { + fail('ENDPOINT_UNAVAILABLE', 'status operational probe sequence has not advanced', true) + } + requireFreshInstant(operational.checkedAt, 'status.operational.checkedAt', nowMs, reconciliationStaleThresholdMs) + const dependencies = record(status.dependencies, 'status.dependencies') + for (const name of REQUIRED_DEPENDENCIES) { + const dependency = record(dependencies[name], `status.dependencies.${name}`) + requireEqual(dependency.status, 'AVAILABLE', `status.dependencies.${name}.status`, 'ENDPOINT_UNAVAILABLE', true) + requireEqual(dependency.error, null, `status.dependencies.${name}.error`, 'ENDPOINT_UNAVAILABLE', true) + requireFreshInstant( + dependency.checkedAt, + `status.dependencies.${name}.checkedAt`, + nowMs, + reconciliationStaleThresholdMs, + ) + } + for (const [name, value] of Object.entries(dependencies)) { + const dependency = record(value, `status.dependencies.${name}`) + requireEqual(dependency.status, 'AVAILABLE', `status.dependencies.${name}.status`, 'ENDPOINT_UNAVAILABLE', true) + requireEqual(dependency.error, null, `status.dependencies.${name}.error`, 'ENDPOINT_UNAVAILABLE', true) + requireFreshInstant( + dependency.checkedAt, + `status.dependencies.${name}.checkedAt`, + nowMs, + reconciliationStaleThresholdMs, + ) + } + const loop = record(status.autonomousCycleLoop, 'status.autonomousCycleLoop') + requireEqual(loop.configured, true, 'status.autonomousCycleLoop.configured', 'PRODUCTION_CONTRACT_VIOLATION', false) + string(loop.startedAt, 'status.autonomousCycleLoop.startedAt') + if (loop.lastPass === null || loop.lastPass === undefined) { + fail('ENDPOINT_UNAVAILABLE', 'status.autonomousCycleLoop.lastPass is not available yet', true) + } + const lastPass = record(loop.lastPass, 'status.autonomousCycleLoop.lastPass') + requireEqual(lastPass.result, 'SUCCESS', 'status.autonomousCycleLoop.lastPass.result', 'ENDPOINT_UNAVAILABLE', true) + requireFreshInstant( + lastPass.observedAt, + 'status.autonomousCycleLoop.lastPass.observedAt', + nowMs, + reconciliationStaleThresholdMs, + ) + string(lastPass.outcome, 'status.autonomousCycleLoop.lastPass.outcome') + validateAuthority(status.authority) + + const broker = record(status.broker, 'status.broker') + requireEqual(broker.configured, true, 'status.broker.configured', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual(broker.accountBound, true, 'status.broker.accountBound', 'ENDPOINT_UNAVAILABLE', true) + requireEqual(broker.readAvailable, true, 'status.broker.readAvailable', 'ENDPOINT_UNAVAILABLE', true) + requireFreshInstant(broker.checkedAt, 'status.broker.checkedAt', nowMs, reconciliationStaleThresholdMs) + requireEqual( + broker.executionEligible, + false, + 'status.broker.executionEligible', + 'PRODUCTION_CONTRACT_VIOLATION', + false, + ) + requireEqual( + broker.executionDisabledReason, + 'BROKER_ACCESS_READ_ONLY', + 'status.broker.executionDisabledReason', + 'PRODUCTION_CONTRACT_VIOLATION', + false, + ) + requireEqual(broker.reasonCode, null, 'status.broker.reasonCode', 'ENDPOINT_UNAVAILABLE', true) + requireEqual(broker.error, null, 'status.broker.error', 'ENDPOINT_UNAVAILABLE', true) + + const cycle = record(status.cycle, 'status.cycle') + requireEqual(cycle.observationAvailable, true, 'status.cycle.observationAvailable', 'ENDPOINT_UNAVAILABLE', true) + const condition = string(cycle.condition, 'status.cycle.condition') + if (['UNKNOWN', 'FAILED', 'STALLED'].includes(condition)) { + fail('ENDPOINT_UNAVAILABLE', 'status.cycle.condition is not operational', true) + } + string(cycle.reason, 'status.cycle.reason') + requireFreshInstant(cycle.checkedAt, 'status.cycle.checkedAt', nowMs, reconciliationStaleThresholdMs) + requireEqual(cycle.zeroMutation, true, 'status.cycle.zeroMutation', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual(cycle.error, null, 'status.cycle.error', 'ENDPOINT_UNAVAILABLE', true) + const mutations = record(cycle.mutations, 'status.cycle.mutations') + requireEqual(mutations.eventCount, 0, 'status.cycle.mutations.eventCount', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual( + mutations.unresolvedCount, + 0, + 'status.cycle.mutations.unresolvedCount', + 'PRODUCTION_CONTRACT_VIOLATION', + false, + ) + requireEqual( + mutations.oldestUnresolvedAt, + null, + 'status.cycle.mutations.oldestUnresolvedAt', + 'PRODUCTION_CONTRACT_VIOLATION', + false, + ) + requireEqual( + mutations.latestOccurredAt, + null, + 'status.cycle.mutations.latestOccurredAt', + 'PRODUCTION_CONTRACT_VIOLATION', + false, + ) + const alerts = record(cycle.alerts, 'status.cycle.alerts') + for (const name of REQUIRED_ALERTS) { + requireEqual(alerts[name], false, `status.cycle.alerts.${name}`, 'PRODUCTION_CONTRACT_VIOLATION', false) + } + for (const [name, value] of Object.entries(alerts)) { + requireEqual(value, false, `status.cycle.alerts.${name}`, 'PRODUCTION_CONTRACT_VIOLATION', false) + } + if (cycle.reconciliation === null || cycle.reconciliation === undefined) { + fail('ENDPOINT_UNAVAILABLE', 'status.cycle.reconciliation is not available yet', true) + } + const reconciliation = record(cycle.reconciliation, 'status.cycle.reconciliation') + requireEqual(reconciliation.status, 'EXACT', 'status.cycle.reconciliation.status', 'ENDPOINT_UNAVAILABLE', true) + requireEqual( + reconciliation.discrepancyCount, + 0, + 'status.cycle.reconciliation.discrepancyCount', + 'PRODUCTION_CONTRACT_VIOLATION', + false, + ) + requireEqual( + reconciliation.coversLatestMutation, + true, + 'status.cycle.reconciliation.coversLatestMutation', + 'ENDPOINT_UNAVAILABLE', + true, + ) + requireEqual( + cycle.reconciliationCoversLatestMutation, + true, + 'status.cycle.reconciliationCoversLatestMutation', + 'ENDPOINT_UNAVAILABLE', + true, + ) + requireFreshInstant( + reconciliation.reconciledAt, + 'status.cycle.reconciliation.reconciledAt', + nowMs, + reconciliationStaleThresholdMs, + ) + const reconciliationAgeMs = integer(cycle.reconciliationAgeMs, 'status.cycle.reconciliationAgeMs') + if (reconciliationAgeMs < 0) { + fail('ENDPOINT_UNAVAILABLE', 'status.cycle.reconciliationAgeMs cannot be negative', true) + } + if (reconciliationAgeMs >= reconciliationStaleThresholdMs) { + fail('ENDPOINT_UNAVAILABLE', 'status.cycle.reconciliation is stale', true) + } + + const build = record(status.build, 'status.build') + requireEqual( + build.sourceRevision, + expected.sourceRevision, + 'status.build.sourceRevision', + 'PRODUCTION_CONTRACT_VIOLATION', + false, + ) + const image = record(build.image, 'status.build.image') + requireEqual( + image.repository, + expected.repository, + 'status.build.image.repository', + 'PRODUCTION_CONTRACT_VIOLATION', + false, + ) + requireEqual(image.digest, expected.digest, 'status.build.image.digest', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual(build.verification, 'embedded', 'status.build.verification', 'PRODUCTION_CONTRACT_VIOLATION', false) + requireEqual(status.error, null, 'status.error', 'ENDPOINT_UNAVAILABLE', true) +} + +export const validateSnapshot = ( + snapshot: VerificationSnapshot, + reconciledRevision: string, + expected: ExpectedPromotion, + promotionRevision = reconciledRevision, + nowMs = Date.now(), +): void => { + if (!SOURCE_PATTERN.test(reconciledRevision) || !SOURCE_PATTERN.test(promotionRevision)) { + fail('INVALID_MANIFEST', 'Argo revisions must be full commit SHAs', false) + } + validateArgo(snapshot.application, reconciledRevision, promotionRevision, expected) + validateDeployment(snapshot.deployment, expected) + validatePod(snapshot.pods, expected) + const maximumEvidenceAgeMs = reconciliationStaleThresholdMs(snapshot.metrics) + validateReadiness(snapshot.readiness, nowMs, maximumEvidenceAgeMs) + validateStatus(snapshot.status, expected, maximumEvidenceAgeMs, nowMs) +} + +export const redactSensitive = (message: string): string => + message + .replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, 'Bearer [REDACTED]') + .replace( + /((?:account.?id|broker.?identity|authorization|credential|key.?id|password|secret|token)\s*[:=]\s*)[^\s,;}]+/gi, + '$1[REDACTED]', + ) + +type CommandResult = { readonly stdout: string; readonly stderr: string; readonly exitCode: number } +export type RunCommand = (command: readonly string[], signal: AbortSignal) => Promise + +const failureFromAbortSignal = (signal: AbortSignal): VerificationFailure => + signal.reason instanceof VerificationFailure + ? signal.reason + : new VerificationFailure('VERIFICATION_INTERRUPTED', 'verification was interrupted', false) + +export const runCommand: RunCommand = async (command, signal) => { + if (signal.aborted) throw failureFromAbortSignal(signal) + const process = Bun.spawn([...command], { stdout: 'pipe', stderr: 'pipe', signal }) + const terminate = () => { + try { + process.kill() + } catch { + // The process may already have exited. The abort reason still controls the verifier result. + } + } + try { + signal.addEventListener('abort', terminate, { once: true }) + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(process.stdout).text(), + new Response(process.stderr).text(), + process.exited, + ]) + if (signal.aborted) throw failureFromAbortSignal(signal) + return { stdout, stderr, exitCode } + } catch { + if (signal.aborted) throw failureFromAbortSignal(signal) + return fail('ENDPOINT_UNAVAILABLE', 'read command could not be executed', true) + } finally { + signal.removeEventListener('abort', terminate) + } +} + +const parseJsonOutput = (result: CommandResult, label: string, retryable: boolean): unknown => { + if (result.exitCode !== 0) { + const stderr = redactSensitive(result.stderr) + if (/forbidden|unauthorized|cannot get resource/i.test(stderr)) { + fail('RBAC_DENIED', `${label} read was denied`, false) + } + fail('ENDPOINT_UNAVAILABLE', `${label} read failed`, retryable) + } + try { + return JSON.parse(result.stdout) as unknown + } catch { + fail('ENDPOINT_UNAVAILABLE', `${label} returned invalid JSON`, retryable) + } +} + +const parseTextOutput = (result: CommandResult, label: string, retryable: boolean): string => { + if (result.exitCode !== 0) { + const stderr = redactSensitive(result.stderr) + if (/forbidden|unauthorized|cannot get resource/i.test(stderr)) { + fail('RBAC_DENIED', `${label} read was denied`, false) + } + fail('ENDPOINT_UNAVAILABLE', `${label} read failed`, retryable) + } + if (result.stdout.length === 0) fail('ENDPOINT_UNAVAILABLE', `${label} returned an empty response`, retryable) + return result.stdout +} + +const checkReadPermission = async ( + run: RunCommand, + signal: AbortSignal, + verb: string, + resource: string, + namespace: string, + resourceName?: string, +) => { + const command = ['kubectl', 'auth', 'can-i', verb, resource, '-n', namespace] + if (resourceName !== undefined) command.push('--resource-name', resourceName) + const result = await run(command, signal) + const decision = result.stdout.trim() + const permission = `${verb} ${resource}${resourceName === undefined ? '' : `/${resourceName}`} in ${namespace}` + if (result.exitCode !== 0 || result.stderr.trim().length !== 0 || (decision !== 'yes' && decision !== 'no')) { + fail('RBAC_DENIED', `${permission} permission probe was indeterminate`, false) + } + if (decision !== 'yes') { + fail('RBAC_DENIED', `missing ${permission} permission`, false) + } +} + +const requireConclusiveDenial = (result: CommandResult, permission: string, grantedMessage: string): void => { + const decision = result.stdout.trim() + if (result.exitCode !== 0 || result.stderr.trim().length !== 0 || (decision !== 'yes' && decision !== 'no')) { + fail('RBAC_DENIED', `${permission} permission probe was indeterminate`, false) + } + if (decision === 'yes') fail('RBAC_DENIED', grantedMessage, false) +} + +const checkWriteDenied = async ( + run: RunCommand, + signal: AbortSignal, + resource: string, + namespace: string, + resourceNames: readonly string[] = [], +) => { + for (const verb of ['create', 'update', 'patch', 'delete', 'deletecollection']) { + const result = await run(['kubectl', 'auth', 'can-i', verb, resource, '-n', namespace], signal) + requireConclusiveDenial( + result, + `${verb} ${resource} in ${namespace}`, + `workflow identity unexpectedly has ${verb} permission for ${resource} in ${namespace}`, + ) + } + for (const resourceName of resourceNames) { + for (const verb of ['update', 'patch', 'delete']) { + const result = await run( + ['kubectl', 'auth', 'can-i', verb, resource, '-n', namespace, '--resource-name', resourceName], + signal, + ) + requireConclusiveDenial( + result, + `${verb} ${resource}/${resourceName} in ${namespace}`, + `workflow identity unexpectedly has ${verb} permission for ${resource}/${resourceName} in ${namespace}`, + ) + } + } +} + +const READ_ONLY_RULE_VERBS = new Set(['get', 'list', 'watch']) +const SELF_REVIEW_RESOURCES = new Set([ + 'selfsubjectreviews.authentication.k8s.io', + 'selfsubjectaccessreviews.authorization.k8s.io', + 'selfsubjectrulesreviews.authorization.k8s.io', +]) + +const resourceCellCanExposeSecrets = (resourceCell: string): boolean => + resourceCell.split(',').some((resource) => { + const normalized = resource.trim().toLowerCase() + if (normalized === '*' || normalized === '*.*') return true + const groupSeparator = normalized.indexOf('.') + const typeWithSubresource = groupSeparator === -1 ? normalized : normalized.slice(0, groupSeparator) + const apiGroup = groupSeparator === -1 ? '' : normalized.slice(groupSeparator + 1) + const resourceType = typeWithSubresource.split('/')[0] ?? '' + return resourceType === 'secrets' && (apiGroup === '' || apiGroup === '*') + }) + +const validateNoMutationRules = async (run: RunCommand, signal: AbortSignal, namespace: string): Promise => { + const result = await run(['kubectl', 'auth', 'can-i', '--list', '--no-headers', '-n', namespace], signal) + if (result.exitCode !== 0 || result.stderr.trim().length !== 0 || result.stdout.trim().length === 0) { + fail('RBAC_DENIED', `authorization rules audit in ${namespace} was indeterminate`, false) + } + for (const [index, line] of result.stdout.trimEnd().split('\n').entries()) { + const matches = [...line.matchAll(/\[([^\]]*)\]/g)] + const verbsSource = matches.at(-1)?.[1] + if (verbsSource === undefined) { + return fail('RBAC_DENIED', `authorization rules audit in ${namespace} returned an invalid row ${index}`, false) + } + const verbs = verbsSource.split(/\s+/).filter((verb) => verb.length > 0) + const resource = line.trim().split(/\s+/)[0] ?? '' + if (resourceCellCanExposeSecrets(resource)) { + fail('RBAC_DENIED', `workflow identity has a Secret authorization rule in ${namespace}`, false) + } + if (verbs.every((verb) => READ_ONLY_RULE_VERBS.has(verb))) continue + if (SELF_REVIEW_RESOURCES.has(resource) && verbs.every((verb) => verb === 'create')) continue + fail('RBAC_DENIED', `workflow identity has a non-read-only authorization rule in ${namespace}`, false) + } +} + +export const validateReadOnlyPermissions = async ( + run: RunCommand, + signal: AbortSignal, + secretNames: readonly string[] = [], +): Promise => { + await checkReadPermission(run, signal, 'get', 'applications.argoproj.io', 'argocd', 'bayn') + await checkReadPermission(run, signal, 'get', 'deployments.apps', 'bayn', 'bayn') + await checkReadPermission(run, signal, 'list', 'pods', 'bayn') + await checkReadPermission(run, signal, 'get', 'services/proxy', 'bayn', 'bayn') + await checkWriteDenied(run, signal, 'services/proxy', 'bayn', ['bayn']) + await checkWriteDenied(run, signal, 'services', 'bayn', ['bayn']) + await checkWriteDenied(run, signal, 'services/status', 'bayn', ['bayn']) + await checkWriteDenied(run, signal, 'endpoints', 'bayn', ['bayn']) + await checkWriteDenied(run, signal, 'endpointslices.discovery.k8s.io', 'bayn') + await checkWriteDenied(run, signal, 'applications.argoproj.io', 'argocd', ['bayn']) + await checkWriteDenied(run, signal, 'applications.argoproj.io/status', 'argocd', ['bayn']) + await checkWriteDenied(run, signal, 'deployments.apps', 'bayn', ['bayn']) + await checkWriteDenied(run, signal, 'deployments.apps/status', 'bayn', ['bayn']) + await checkWriteDenied(run, signal, 'deployments.apps/scale', 'bayn', ['bayn']) + await checkWriteDenied(run, signal, 'pods', 'bayn') + await checkWriteDenied(run, signal, 'pods/status', 'bayn') + await checkWriteDenied(run, signal, 'pods/eviction', 'bayn') + await checkWriteDenied(run, signal, 'pods/attach', 'bayn') + await checkWriteDenied(run, signal, 'pods/exec', 'bayn') + await checkWriteDenied(run, signal, 'pods/ephemeralcontainers', 'bayn') + await checkWriteDenied(run, signal, 'pods/portforward', 'bayn') + await checkWriteDenied(run, signal, 'pods/proxy', 'bayn') + await checkWriteDenied(run, signal, 'secrets', 'bayn', secretNames) + for (const verb of ['get', 'list', 'watch']) { + const secretRead = await run(['kubectl', 'auth', 'can-i', verb, 'secrets', '-n', 'bayn'], signal) + requireConclusiveDenial( + secretRead, + `${verb} secrets in bayn`, + `workflow identity unexpectedly has ${verb} secret permission in bayn`, + ) + for (const secretName of secretNames) { + const namedSecretRead = await run( + ['kubectl', 'auth', 'can-i', verb, 'secrets', '-n', 'bayn', '--resource-name', secretName], + signal, + ) + requireConclusiveDenial( + namedSecretRead, + `${verb} secrets/${secretName} in bayn`, + `workflow identity unexpectedly has ${verb} permission for secrets/${secretName} in bayn`, + ) + } + } + await validateNoMutationRules(run, signal, 'argocd') + await validateNoMutationRules(run, signal, 'bayn') +} + +export const readArgoSyncRevision = (applicationValue: unknown): string => { + const application = record(applicationValue, 'application') + const status = record(application.status, 'application.status') + const sync = record(status.sync, 'application.status.sync') + const revision = string(sync.revision, 'application.status.sync.revision') + if (!SOURCE_PATTERN.test(revision)) { + fail('ARGO_NOT_CONVERGED', 'Argo sync revision is not a full commit SHA', true) + } + return revision +} + +const runGitCheck = async ( + run: RunCommand, + signal: AbortSignal, + command: readonly string[], + retryableFailure: string, +): Promise => { + const result = await run(command, signal) + if (result.exitCode === 0) return + if (result.exitCode === 1) fail('ARGO_NOT_CONVERGED', retryableFailure, true) + fail('ENDPOINT_UNAVAILABLE', 'Git revision verification failed', true) +} + +export const verifyArgoRevision = async ( + run: RunCommand, + signal: AbortSignal, + root: string, + promotionRevision: string, + reconciledRevision: string, +): Promise => { + if (!SOURCE_PATTERN.test(promotionRevision) || !SOURCE_PATTERN.test(reconciledRevision)) { + fail('ARGO_NOT_CONVERGED', 'Argo revision verification requires full commit SHAs', true) + } + if (promotionRevision === reconciledRevision) return + + const fetch = await run(['git', '-C', root, 'fetch', '--no-tags', '--quiet', 'origin', 'main'], signal) + if (fetch.exitCode !== 0) fail('ENDPOINT_UNAVAILABLE', 'Current main revision could not be refreshed', true) + await runGitCheck( + run, + signal, + ['git', '-C', root, 'merge-base', '--is-ancestor', reconciledRevision, 'origin/main'], + 'Argo revision is not on current main', + ) + await runGitCheck( + run, + signal, + ['git', '-C', root, 'merge-base', '--is-ancestor', promotionRevision, reconciledRevision], + 'Argo has not reconciled the promotion revision or a verified descendant', + ) + const manifestDiff = await run( + [ + 'git', + '-C', + root, + 'diff', + '--quiet', + `${promotionRevision}..${reconciledRevision}`, + '--', + 'argocd/applications/bayn/deployment.yaml', + 'argocd/applications/bayn/kustomization.yaml', + ], + signal, + ) + if (manifestDiff.exitCode === 1) { + fail('PRODUCTION_CONTRACT_VIOLATION', 'A later main revision superseded the promoted Bayn manifests', false) + } + if (manifestDiff.exitCode !== 0) fail('ENDPOINT_UNAVAILABLE', 'Bayn manifest lineage could not be verified', true) +} + +const fetchSnapshot = async (run: RunCommand, signal: AbortSignal): Promise => { + const [application, deployment, pods, readiness, status, metrics] = await Promise.all([ + run(['kubectl', 'get', 'application', 'bayn', '-n', 'argocd', '-o', 'json'], signal), + run(['kubectl', 'get', 'deployment', 'bayn', '-n', 'bayn', '-o', 'json'], signal), + run(['kubectl', 'get', 'pods', '-n', 'bayn', '-l', 'app.kubernetes.io/name=bayn', '-o', 'json'], signal), + run(['kubectl', 'get', '--raw', '/api/v1/namespaces/bayn/services/http:bayn:80/proxy/readyz'], signal), + run(['kubectl', 'get', '--raw', '/api/v1/namespaces/bayn/services/http:bayn:80/proxy/v1/status'], signal), + run(['kubectl', 'get', '--raw', '/api/v1/namespaces/bayn/services/http:bayn:80/proxy/metrics'], signal), + ]) + return { + application: parseJsonOutput(application, 'Argo application', true), + deployment: parseJsonOutput(deployment, 'Bayn deployment', true), + pods: parseJsonOutput(pods, 'Bayn pods', true), + readiness: parseJsonOutput(readiness, 'Bayn readiness endpoint', true), + status: parseJsonOutput(status, 'Bayn status endpoint', true), + metrics: parseTextOutput(metrics, 'Bayn metrics endpoint', true), + } +} + +export type RetryOptions = { + readonly deadlineMs: number + readonly deadlineAt?: number + readonly intervalMs: number + readonly now?: () => number + readonly sleep?: (milliseconds: number, signal: AbortSignal) => Promise +} + +const sleep = (milliseconds: number, signal: AbortSignal): Promise => + new Promise((resolve, reject) => { + let timeout: ReturnType + const abort = () => { + clearTimeout(timeout) + reject(new VerificationFailure('VERIFICATION_INTERRUPTED', 'verification was interrupted', false)) + } + const complete = () => { + signal.removeEventListener('abort', abort) + resolve() + } + timeout = setTimeout(complete, milliseconds) + signal.addEventListener('abort', abort, { once: true }) + }) + +export const runWithinDeadline = async ( + operation: (signal: AbortSignal) => Promise, + parentSignal: AbortSignal, + deadlineAt: number, + now: () => number = Date.now, +): Promise => { + if (parentSignal.aborted) throw failureFromAbortSignal(parentSignal) + const remainingMs = deadlineAt - now() + if (remainingMs <= 0) { + fail('VERIFICATION_TIMEOUT', 'configured verification deadline expired before the next operation', false) + } + + const controller = new AbortController() + const interrupted = () => controller.abort(failureFromAbortSignal(parentSignal)) + parentSignal.addEventListener('abort', interrupted, { once: true }) + const timeoutFailure = new VerificationFailure( + 'VERIFICATION_TIMEOUT', + 'configured verification deadline expired during an operation', + false, + ) + let timeout: ReturnType | undefined + const timeoutPromise = new Promise((_, reject) => { + timeout = setTimeout(() => { + controller.abort(timeoutFailure) + reject(timeoutFailure) + }, remainingMs) + }) + + try { + const value = await Promise.race([operation(controller.signal), timeoutPromise]) + if (parentSignal.aborted) throw failureFromAbortSignal(parentSignal) + if (controller.signal.aborted) throw failureFromAbortSignal(controller.signal) + if (now() > deadlineAt) { + controller.abort(timeoutFailure) + throw timeoutFailure + } + return value + } catch (error) { + if (parentSignal.aborted) throw failureFromAbortSignal(parentSignal) + if (controller.signal.aborted) throw failureFromAbortSignal(controller.signal) + throw error + } finally { + if (timeout !== undefined) clearTimeout(timeout) + parentSignal.removeEventListener('abort', interrupted) + } +} + +export const retryVerification = async ( + operation: (signal: AbortSignal) => Promise, + signal: AbortSignal, + options: RetryOptions, +): Promise => { + const now = options.now ?? Date.now + const wait = options.sleep ?? sleep + const deadline = options.deadlineAt ?? now() + options.deadlineMs + let lastFailure: VerificationFailure | undefined + while (now() <= deadline) { + if (signal.aborted) fail('VERIFICATION_INTERRUPTED', 'verification was interrupted', false) + try { + await runWithinDeadline(operation, signal, deadline, now) + return + } catch (error) { + const failure = + error instanceof VerificationFailure + ? error + : new VerificationFailure('ENDPOINT_UNAVAILABLE', 'unexpected verifier failure', false) + if (!failure.retryable) throw failure + lastFailure = failure + } + if (now() + options.intervalMs > deadline) break + await wait(options.intervalMs, signal) + } + throw new VerificationFailure( + 'VERIFICATION_TIMEOUT', + `deadline expired; last blocker was ${lastFailure?.code ?? 'unknown'}`, + false, + ) +} + +type CliOptions = { + readonly expectedRevision: string + readonly deadlineSeconds: number + readonly intervalSeconds: number + readonly root: string +} + +const parsePositive = (value: string, name: string): number => { + const parsed = Number(value) + if (!Number.isFinite(parsed) || parsed <= 0) fail('INVALID_MANIFEST', `${name} must be positive`, false) + return parsed +} + +const parseCli = (args: readonly string[]): CliOptions => { + const values = new Map() + for (let index = 0; index < args.length; index += 1) { + const argument = args[index] + const value = args[index + 1] + if (!argument?.startsWith('--') || value === undefined || value.startsWith('--')) { + fail('INVALID_MANIFEST', `invalid argument ${argument ?? ''}`, false) + } + values.set(argument, value) + index += 1 + } + const expectedRevision = values.get('--expected-revision') ?? '' + if (!SOURCE_PATTERN.test(expectedRevision)) { + fail('INVALID_MANIFEST', '--expected-revision must be a full commit SHA', false) + } + return { + expectedRevision, + deadlineSeconds: parsePositive(values.get('--deadline-seconds') ?? '900', '--deadline-seconds'), + intervalSeconds: parsePositive(values.get('--interval-seconds') ?? '10', '--interval-seconds'), + root: values.get('--root') ?? process.cwd(), + } +} + +const main = async (): Promise => { + const options = parseCli(process.argv.slice(2)) + const controller = new AbortController() + const deadlineAt = Date.now() + options.deadlineSeconds * 1_000 + const interrupt = () => controller.abort() + process.once('SIGINT', interrupt) + process.once('SIGTERM', interrupt) + try { + const kustomizationPath = join(options.root, 'argocd/applications/bayn/kustomization.yaml') + const deploymentPath = join(options.root, 'argocd/applications/bayn/deployment.yaml') + const [kustomizationSource, deploymentSource] = await runWithinDeadline( + (signal) => + Promise.all([ + readFile(kustomizationPath, { encoding: 'utf8', signal }), + readFile(deploymentPath, { encoding: 'utf8', signal }), + ]), + controller.signal, + deadlineAt, + ) + const expected = parseExpectedPromotion(kustomizationSource, deploymentSource) + await runWithinDeadline( + (signal) => validateReadOnlyPermissions(runCommand, signal, expected.secretNames), + controller.signal, + deadlineAt, + ) + await retryVerification( + async (signal) => { + const snapshot = await fetchSnapshot(runCommand, signal) + const reconciledRevision = readArgoSyncRevision(snapshot.application) + await verifyArgoRevision(runCommand, signal, options.root, options.expectedRevision, reconciledRevision) + validateSnapshot(snapshot, reconciledRevision, expected, options.expectedRevision) + }, + controller.signal, + { + deadlineMs: options.deadlineSeconds * 1_000, + deadlineAt, + intervalMs: options.intervalSeconds * 1_000, + }, + ) + console.log( + `Bayn post-deploy verification passed for Argo revision ${options.expectedRevision}, source ${expected.sourceRevision}, digest ${expected.digest}`, + ) + } catch (error) { + const message = redactSensitive(error instanceof Error ? error.message : 'unknown verifier failure') + console.error(message) + process.exitCode = 1 + } finally { + process.off('SIGINT', interrupt) + process.off('SIGTERM', interrupt) + } +} + +if (import.meta.main) await main()