diff --git a/.github/scripts/materialize-homebrew-candidate-package-input.sh b/.github/scripts/materialize-homebrew-candidate-package-input.sh new file mode 100755 index 0000000000..4b27402e34 --- /dev/null +++ b/.github/scripts/materialize-homebrew-candidate-package-input.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# Materialize the complete immutable PR package ledger used by one candidate +# bottle build. This is a read-only operation; it never writes package state. +set -euo pipefail + +TAG="" +PR_NUMBER="" +RUN_ID="" +RUN_ATTEMPT="" +PRODUCER_SHA="" +EXPECTED_ABI="" +EXCLUSIONS="" +CONSUMER_ROOT="" +CONSUMER_SHA="" +XTASK="" +OUTPUT_DIR="" + +while [ "$#" -gt 0 ]; do + case "$1" in + --tag) TAG="$2"; shift 2 ;; + --pr-number) PR_NUMBER="$2"; shift 2 ;; + --run-id) RUN_ID="$2"; shift 2 ;; + --run-attempt) RUN_ATTEMPT="$2"; shift 2 ;; + --producer-sha) PRODUCER_SHA="$2"; shift 2 ;; + --expected-abi) EXPECTED_ABI="$2"; shift 2 ;; + --exclude) EXCLUSIONS="$2"; shift 2 ;; + --consumer-root) CONSUMER_ROOT="$2"; shift 2 ;; + --consumer-sha) CONSUMER_SHA="$2"; shift 2 ;; + --xtask) XTASK="$2"; shift 2 ;; + --output-dir) OUTPUT_DIR="$2"; shift 2 ;; + *) + echo "materialize-homebrew-candidate-package-input: unknown flag $1" >&2 + exit 2 + ;; + esac +done + +expected_tag="pr-${PR_NUMBER}-staging-run-${RUN_ID}-attempt-${RUN_ATTEMPT}" +if ! [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || + ! [[ "$RUN_ID" =~ ^[1-9][0-9]*$ ]] || + ! [[ "$RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]] || + ! [[ "$PRODUCER_SHA" =~ ^[0-9a-f]{40}$ ]] || + ! [[ "$EXPECTED_ABI" =~ ^[1-9][0-9]*$ ]] || + [ "$TAG" != "$expected_tag" ] || + ! [[ "$CONSUMER_SHA" =~ ^[0-9a-f]{40}$ ]] || + [ ! -d "$CONSUMER_ROOT" ] || [ -L "$CONSUMER_ROOT" ] || + [ ! -x "$XTASK" ] || [ -L "$XTASK" ] || + [ -z "$EXCLUSIONS" ] || + [ -z "$OUTPUT_DIR" ] || [ "$OUTPUT_DIR" = / ]; then + echo "materialize-homebrew-candidate-package-input: exact run, source, ABI, tools, exclusions, and output are required" >&2 + exit 2 +fi +if [ -e "$OUTPUT_DIR" ] || [ -L "$OUTPUT_DIR" ]; then + echo "materialize-homebrew-candidate-package-input: output already exists" >&2 + exit 2 +fi +if [ "$(git -C "$CONSUMER_ROOT" rev-parse HEAD)" != "$CONSUMER_SHA" ] || + [ -n "$(git -C "$CONSUMER_ROOT" status --porcelain=v1 --untracked-files=all)" ]; then + echo "materialize-homebrew-candidate-package-input: Kandelo consumer is not the exact clean source" >&2 + exit 2 +fi + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +AUTHORITY_ROOT="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)" +PARENT="$(dirname "$OUTPUT_DIR")" +mkdir -p "$PARENT" +TMP_ROOT="$(mktemp -d "$PARENT/.homebrew-candidate-packages.XXXXXX")" +trap 'rm -rf "$TMP_ROOT"' EXIT +mkdir "$TMP_ROOT/output" + +EXPECTED="$TMP_ROOT/output/expected-ledger.json" +RELEASE="$TMP_ROOT/output/release-evidence.json" +SNAPSHOT="$TMP_ROOT/output/resolver" +BODY="$TMP_ROOT/release-body.txt" + +env -u GH_TOKEN -u GITHUB_TOKEN \ + -u HOMEBREW_GITHUB_API_TOKEN \ + -u HOMEBREW_GITHUB_PACKAGES_TOKEN \ + -u HOMEBREW_DOCKER_REGISTRY_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_URL \ + -u ACTIONS_RUNTIME_TOKEN \ + "$XTASK" staging-reuse expected \ + --registry "$CONSUMER_ROOT/packages/registry" \ + --expected-abi "$EXPECTED_ABI" \ + --exclude "$EXCLUSIONS" \ + --output "$EXPECTED" + +printf 'PR #%s staging build run %s attempt %s' \ + "$PR_NUMBER" "$RUN_ID" "$RUN_ATTEMPT" >"$BODY" + +# WHY: package build success is only an ordering signal. Revalidate the +# public release, direct tag, complete ledger, archive manifests, and archive +# bytes before any of them can execute as a bottle-build dependency. +GITHUB_REPOSITORY=Automattic/kandelo \ + bash "$SCRIPT_DIR/package-release-lifecycle.sh" verify-immutable \ + --tag "$TAG" \ + --target-commit "$PRODUCER_SHA" \ + --title "$TAG" \ + --body-file "$BODY" \ + --prerelease true + +GITHUB_REPOSITORY=Automattic/kandelo \ + bash "$SCRIPT_DIR/validate-staging-release.sh" \ + --tag "$TAG" \ + --expected-ledger "$EXPECTED" \ + --mode current \ + --materialize \ + --output-dir "$SNAPSHOT" \ + --xtask "$XTASK" + +gh api "/repos/Automattic/kandelo/releases/tags/$TAG" | + jq -eS \ + --arg repository Automattic/kandelo \ + --arg producer "$PRODUCER_SHA" \ + --arg tag "$TAG" \ + --argjson pr "$PR_NUMBER" \ + --argjson run "$RUN_ID" \ + --argjson attempt "$RUN_ATTEMPT" ' + select( + .tag_name == $tag and + .target_commitish == $producer and + .draft == false and .prerelease == true and .immutable == true and + (.id | type == "number" and . > 0) + ) | + { + schema:1, + repository:$repository, + tag:.tag_name, + release_id:.id, + target_commit:$producer, + immutable:true, + pr_number:$pr, + run_id:$run, + attempt:$attempt + } + ' >"$RELEASE" + +env -u GH_TOKEN -u GITHUB_TOKEN \ + -u HOMEBREW_GITHUB_API_TOKEN \ + -u HOMEBREW_GITHUB_PACKAGES_TOKEN \ + -u HOMEBREW_DOCKER_REGISTRY_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_URL \ + -u ACTIONS_RUNTIME_TOKEN \ + python3 "$AUTHORITY_ROOT/scripts/homebrew-bottle-candidate.py" package-input \ + --expected-ledger "$EXPECTED" \ + --snapshot "$SNAPSHOT/snapshot.json" \ + --release-evidence "$RELEASE" \ + --index "$SNAPSHOT/source-index.toml" \ + --producer-commit "$PRODUCER_SHA" \ + --abi "$EXPECTED_ABI" \ + --out "$TMP_ROOT/output/package-input.json" + +# validate-staging-release writes its temporary path into index-url.txt. The +# resolver is moved atomically below, so write the final local URL explicitly. +printf 'file://%s/resolver/archives/index.toml\n' "$OUTPUT_DIR" \ + >"$SNAPSHOT/index-url.txt" +mv "$TMP_ROOT/output" "$OUTPUT_DIR" +rm -rf "$TMP_ROOT" +trap - EXIT +echo "materialize-homebrew-candidate-package-input: froze $TAG at $OUTPUT_DIR" diff --git a/.github/workflows/homebrew-native-publisher-compatibility.yml b/.github/workflows/homebrew-native-publisher-compatibility.yml index e53bc2e694..49b717fbdc 100644 --- a/.github/workflows/homebrew-native-publisher-compatibility.yml +++ b/.github/workflows/homebrew-native-publisher-compatibility.yml @@ -4,7 +4,9 @@ on: pull_request: paths: - .github/workflows/homebrew-native-publisher-compatibility.yml + - .github/workflows/reusable-homebrew-bottle-candidate-materialize.yml - .github/workflows/reusable-homebrew-bottle-publish.yml + - .github/workflows/reusable-homebrew-candidate-campaign.yml - .github/workflows/reusable-homebrew-prefix-first-child-publish.yml - flake.lock - flake.nix diff --git a/.github/workflows/reusable-homebrew-bottle-candidate-materialize.yml b/.github/workflows/reusable-homebrew-bottle-candidate-materialize.yml new file mode 100644 index 0000000000..c96202ef5f --- /dev/null +++ b/.github/workflows/reusable-homebrew-bottle-candidate-materialize.yml @@ -0,0 +1,682 @@ +name: Re-materialize an exact merged Homebrew bottle candidate + +on: + workflow_call: + inputs: + candidate-tag: + type: string + required: true + formula: + type: string + required: true + producer-sha: + type: string + required: true + merge-commit: + type: string + required: true + outputs: + abi: + value: ${{ jobs.materialize.outputs.abi }} + campaign-tag: + value: ${{ jobs.materialize.outputs.campaign-tag }} + formula: + value: ${{ jobs.materialize.outputs.formula }} + release-tag: + value: ${{ jobs.materialize.outputs.release-tag }} + source-tap-commit: + value: ${{ jobs.materialize.outputs.source-tap-commit }} + +jobs: + materialize: + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + actions: read + contents: read + outputs: + abi: ${{ steps.candidate.outputs.abi }} + campaign-tag: ${{ steps.candidate.outputs.campaign-tag }} + formula: ${{ steps.candidate.outputs.formula }} + release-tag: ${{ steps.candidate.outputs.release-tag }} + source-tap-commit: ${{ steps.candidate.outputs.source-tap-commit }} + steps: + - name: Admit the protected tap promotion caller + shell: bash + env: + CANDIDATE_TAG: ${{ inputs.candidate-tag }} + CALLER_REF: ${{ github.ref }} + CALLER_REPOSITORY: ${{ github.repository }} + CALLER_WORKFLOW_REF: ${{ github.workflow_ref }} + FORMULA: ${{ inputs.formula }} + MERGE_COMMIT: ${{ inputs.merge-commit }} + PRODUCER_SHA: ${{ inputs.producer-sha }} + run: | + set -euo pipefail + expected_caller="$CALLER_REPOSITORY/.github/workflows/" + expected_caller+="promote-candidate-bottle.yml@refs/heads/main" + [ "$CALLER_REPOSITORY" = \ + kandelo-dev/homebrew-tap-core ] && + [ "$CALLER_REF" = refs/heads/main ] && + [ "$CALLER_WORKFLOW_REF" = "$expected_caller" ] && + [[ "$FORMULA" =~ ^[a-z0-9][a-z0-9._-]{0,254}$ ]] && + [[ "$PRODUCER_SHA" =~ ^[0-9a-f]{40}$ ]] && + [[ "$MERGE_COMMIT" =~ ^[0-9a-f]{40}$ ]] && + [[ "$CANDIDATE_TAG" =~ \ + ^homebrew-bottle-candidate-pr-[1-9][0-9]*-run-[1-9][0-9]*-attempt-[1-9][0-9]*-sha256-[0-9a-f]{64}$ ]] || { + echo "::error::candidate promotion caller is not exact" + exit 2 + } + + - name: Checkout exact merged Kandelo authority + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: Automattic/kandelo + ref: ${{ inputs.merge-commit }} + path: kandelo-main + fetch-depth: 0 + submodules: false + + - name: Checkout exact bottle producer as inert data + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: Automattic/kandelo + ref: ${{ inputs.producer-sha }} + path: producer + fetch-depth: 0 + submodules: false + + - name: Checkout protected tap authority + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + ref: ${{ github.sha }} + fetch-depth: 0 + path: tap-authority + + - name: Require the promotion caller to pin the exact merge + shell: bash + env: + MERGE_COMMIT: ${{ inputs.merge-commit }} + run: | + set -euo pipefail + # WHY: the promotion token belongs only to code at merge M. Checking + # caller commit C prevents a mutable @main ref from selecting later + # Kandelo code before this protected workflow can inspect it. + python3 kandelo-main/scripts/homebrew-candidate-caller-pins.py \ + validate --tap-root "$GITHUB_WORKSPACE/tap-authority" \ + --mode promotion --kandelo-sha "$MERGE_COMMIT" + + - name: Authenticate candidate runs and locate sealer receipts + id: candidate + shell: bash + env: + CANDIDATE_TAG: ${{ inputs.candidate-tag }} + EXPECTED_FORMULA: ${{ inputs.formula }} + EXPECTED_PRODUCER: ${{ inputs.producer-sha }} + GH_TOKEN: ${{ github.token }} + TAP_REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + release_root="$RUNNER_TEMP/homebrew-candidate-release" + mkdir "$release_root" + gh api "/repos/$TAP_REPOSITORY/releases/tags/$CANDIDATE_TAG" \ + >"$RUNNER_TEMP/candidate-release.json" + release_id="$(jq -er .id "$RUNNER_TEMP/candidate-release.json")" + gh api --paginate --slurp \ + "/repos/$TAP_REPOSITORY/releases/$release_id/assets?per_page=100" \ + >"$RUNNER_TEMP/candidate-release-asset-pages.json" + jq -e '[.[].[]]' \ + "$RUNNER_TEMP/candidate-release-asset-pages.json" \ + >"$RUNNER_TEMP/candidate-release-assets.json" + jq -e --arg tag "$CANDIDATE_TAG" ' + .tag_name == $tag and .draft == false and + .prerelease == false and .immutable == true and + (.id | type == "number" and . > 0) + ' "$RUNNER_TEMP/candidate-release.json" >/dev/null || { + echo "::error::candidate release is not public and immutable" + exit 1 + } + jq -er ' + map(select( + .name == "candidate.json" and .state == "uploaded" and + (.id | type == "number" and . > 0) and + (.size | type == "number" and . > 0 and . <= 16777216) and + (.digest | type == "string" and + test("^sha256:[0-9a-f]{64}$")) and + (.browser_download_url | type == "string") + )) | select(length == 1) | .[0].browser_download_url + ' "$RUNNER_TEMP/candidate-release-assets.json" \ + >"$RUNNER_TEMP/candidate-manifest-url.txt" + # WHY: candidate.json is the bounded allowlist for every larger + # release asset. Validate it before downloading those bytes. + env -u GH_TOKEN -u GITHUB_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_URL \ + -u ACTIONS_RUNTIME_TOKEN \ + curl --disable --fail --location --silent --show-error \ + --output "$release_root/candidate.json" \ + "$(cat "$RUNNER_TEMP/candidate-manifest-url.txt")" + env -u GH_TOKEN -u GITHUB_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_URL \ + -u ACTIONS_RUNTIME_TOKEN \ + python3 kandelo-main/scripts/homebrew-bottle-candidate.py \ + describe-release \ + --candidate "$release_root/candidate.json" \ + --candidate-tag "$CANDIDATE_TAG" \ + --out "$RUNNER_TEMP/candidate-description.json" + jq -e \ + --arg formula "$EXPECTED_FORMULA" \ + --arg producer "$EXPECTED_PRODUCER" \ + --arg repository "$TAP_REPOSITORY" \ + --slurpfile release "$RUNNER_TEMP/candidate-release.json" ' + .manifest.formula.name == $formula and + .manifest.formula.arch == "wasm32" and + .manifest.dependencies == [] and + .manifest.source.producer_commit == $producer and + (.manifest.source.tap_repository | ascii_downcase) == + ($repository | ascii_downcase) and + .manifest.run.caller_commit == + $release[0].target_commitish + ' "$RUNNER_TEMP/candidate-description.json" >/dev/null || { + echo "::error::candidate release differs from the request" + exit 1 + } + campaign_tag="$(jq -er \ + '.manifest.source.prefix_campaign_tag' \ + "$RUNNER_TEMP/candidate-description.json")" + campaign_root="$RUNNER_TEMP/homebrew-candidate-campaign-release" + mkdir "$campaign_root" + env -u GH_TOKEN -u GITHUB_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_URL \ + -u ACTIONS_RUNTIME_TOKEN \ + python3 \ + kandelo-main/scripts/homebrew-candidate-campaign.py \ + fetch-release \ + --repository "$TAP_REPOSITORY" \ + --tag "$campaign_tag" \ + --out "$campaign_root/campaign.json" \ + --candidate-out \ + "$campaign_root/candidate-campaign.json" \ + --receipt-out \ + "$RUNNER_TEMP/candidate-campaign-readback.json" + cp "$campaign_root/campaign.json" \ + "$RUNNER_TEMP/candidate-prefix-campaign.json" + cp "$campaign_root/candidate-campaign.json" \ + "$RUNNER_TEMP/candidate-campaign-manifest.json" + jq -e \ + --slurpfile bottle \ + "$RUNNER_TEMP/candidate-description.json" ' + .source.producer_commit == + $bottle[0].manifest.source.producer_commit and + .source.pr_number == + $bottle[0].manifest.source.pr_number and + .source.source_tap_commit == + $bottle[0].manifest.source.tap_commit and + .source.abi == $bottle[0].manifest.source.abi and + .source.guest_layout.sha256 == + $bottle[0].manifest.source.guest_layout.sha256 + ' "$RUNNER_TEMP/candidate-campaign-manifest.json" >/dev/null || { + echo "::error::bottle candidate names another campaign source" + exit 1 + } + + campaign_release_id="$(gh api \ + "/repos/$TAP_REPOSITORY/releases/tags/$campaign_tag" \ + --jq .id)" + gh api "/repos/$TAP_REPOSITORY/releases/$campaign_release_id" \ + >"$RUNNER_TEMP/candidate-campaign-release.json" + gh api --paginate --slurp \ + "/repos/$TAP_REPOSITORY/releases/$campaign_release_id/assets?per_page=100" \ + >"$RUNNER_TEMP/candidate-campaign-release-asset-pages.json" + jq -e '[.[].[]]' \ + "$RUNNER_TEMP/candidate-campaign-release-asset-pages.json" \ + >"$RUNNER_TEMP/candidate-campaign-release-assets.json" + + locate_receipt() { + local prefix="$1" run_id="$2" attempt="$3" caller="$4" + local workflow_path="$5" receipt_name="$6" + local run_json="$RUNNER_TEMP/$prefix-run.json" + local pages="$RUNNER_TEMP/$prefix-artifact-pages.json" + local artifacts="$RUNNER_TEMP/$prefix-artifacts.json" + local jobs_pages="$RUNNER_TEMP/$prefix-job-pages.json" + local jobs="$RUNNER_TEMP/$prefix-jobs.json" + # WHY: a later rerun must not rewrite the identity of the exact + # successful attempt that sealed this immutable candidate. + gh api \ + "/repos/$TAP_REPOSITORY/actions/runs/$run_id/attempts/$attempt" \ + >"$run_json" + gh api --paginate --slurp \ + "/repos/$TAP_REPOSITORY/actions/runs/$run_id/artifacts?per_page=100" \ + >"$pages" + jq -e '[.[].artifacts[]]' "$pages" >"$artifacts" + gh api --paginate --slurp \ + "/repos/$TAP_REPOSITORY/actions/runs/$run_id/attempts/$attempt/jobs?per_page=100" \ + >"$jobs_pages" + jq -e '[.[].jobs[]]' "$jobs_pages" >"$jobs" + jq -e \ + --arg caller "$caller" \ + --arg repository "$TAP_REPOSITORY" \ + --arg workflow "$workflow_path" \ + --argjson run_id "$run_id" \ + --argjson attempt "$attempt" ' + .id == $run_id and .run_attempt == $attempt and + .head_sha == $caller and .path == $workflow and + (.repository.full_name | ascii_downcase) == + ($repository | ascii_downcase) and + .event == "repository_dispatch" and + .status == "completed" and .conclusion == "success" + ' "$run_json" >/dev/null || { + echo "::error::$prefix workflow run is not exact and successful" + exit 1 + } + jq -e --argjson attempt "$attempt" ' + length > 0 and all(.[]; + .run_attempt == $attempt and .status == "completed" and + (.conclusion == "success" or .conclusion == "skipped")) + ' "$jobs" >/dev/null || { + echo "::error::$prefix did not complete one coherent attempt" + exit 1 + } + jq -e \ + --arg caller "$caller" \ + --arg name "$receipt_name" \ + --argjson run_id "$run_id" ' + map(select(.name == $name)) as $selected | + ($selected | length) == 1 and + $selected[0].expired == false and + ($selected[0].id | type == "number" and . > 0) and + ($selected[0].size_in_bytes | type == "number" and + . > 0 and . <= 16777216) and + ($selected[0].digest | type == "string" and + test("^sha256:[0-9a-f]{64}$")) and + $selected[0].workflow_run.id == $run_id and + $selected[0].workflow_run.head_sha == $caller + ' "$artifacts" >/dev/null || { + echo "::error::$prefix lacks one exact live sealer receipt" + exit 1 + } + jq -er --arg name "$receipt_name" \ + '.[] | select(.name == $name) | .id' "$artifacts" + } + + bottle_run="$(jq -er '.manifest.run.run_id' \ + "$RUNNER_TEMP/candidate-description.json")" + bottle_attempt="$(jq -er '.manifest.run.run_attempt' \ + "$RUNNER_TEMP/candidate-description.json")" + bottle_caller="$(jq -er '.manifest.run.caller_commit' \ + "$RUNNER_TEMP/candidate-description.json")" + bottle_receipt="homebrew-candidate-release-receipt-" + bottle_receipt+="$EXPECTED_FORMULA-wasm32-attempt-$bottle_attempt" + bottle_receipt_id="$(locate_receipt bottle "$bottle_run" \ + "$bottle_attempt" "$bottle_caller" \ + .github/workflows/candidate-bottles.yml "$bottle_receipt")" + + campaign_run="$(jq -er '.run.run_id' \ + "$RUNNER_TEMP/candidate-campaign-manifest.json")" + campaign_attempt="$(jq -er '.run.run_attempt' \ + "$RUNNER_TEMP/candidate-campaign-manifest.json")" + campaign_caller="$(jq -er '.run.caller_commit' \ + "$RUNNER_TEMP/candidate-campaign-manifest.json")" + campaign_receipt="homebrew-candidate-campaign-release-" + campaign_receipt+="attempt-$campaign_attempt" + campaign_receipt_id="$(locate_receipt campaign "$campaign_run" \ + "$campaign_attempt" "$campaign_caller" \ + .github/workflows/candidate-campaign.yml "$campaign_receipt")" + + jq -S '.manifest.run + { + status:"completed", conclusion:"success" + }' "$RUNNER_TEMP/candidate-description.json" \ + >"$RUNNER_TEMP/completed-candidate-run.json" + jq -S '.run + { + status:"completed", conclusion:"success" + }' "$RUNNER_TEMP/candidate-campaign-manifest.json" \ + >"$RUNNER_TEMP/completed-candidate-campaign-run.json" + jq -r '.manifest.files[] | + select(.path == "package-input.json") | .asset_name' \ + "$RUNNER_TEMP/candidate-description.json" \ + >"$RUNNER_TEMP/candidate-package-asset.txt" + [ "$(wc -l <"$RUNNER_TEMP/candidate-package-asset.txt")" -eq 1 ] || { + echo "::error::candidate has no unique package input" + exit 1 + } + { + echo "abi=$(jq -r '.manifest.source.abi' \ + "$RUNNER_TEMP/candidate-description.json")" + echo "campaign-tag=$(jq -r \ + '.manifest.source.prefix_campaign_tag' \ + "$RUNNER_TEMP/candidate-description.json")" + echo "formula=$EXPECTED_FORMULA" + echo "release-tag=$(jq -r '.manifest.source.release_tag' \ + "$RUNNER_TEMP/candidate-description.json")" + echo "source-tap-commit=$(jq -r \ + '.manifest.source.tap_commit' \ + "$RUNNER_TEMP/candidate-description.json")" + echo "native-homebrew-commit=$(jq -r \ + '.source.native_homebrew_commit' \ + "$RUNNER_TEMP/candidate-campaign-manifest.json")" + echo "bottle-run-id=$bottle_run" + echo "bottle-receipt-artifact-id=$bottle_receipt_id" + echo "campaign-run-id=$campaign_run" + echo "campaign-receipt-artifact-id=$campaign_receipt_id" + } >>"$GITHUB_OUTPUT" + + - name: Download exact bottle sealer receipt + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ steps.candidate.outputs.bottle-receipt-artifact-id }} + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ steps.candidate.outputs.bottle-run-id }} + path: ${{ runner.temp }}/bottle-sealer-receipt + + - name: Download exact campaign sealer receipt + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ steps.candidate.outputs.campaign-receipt-artifact-id }} + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ steps.candidate.outputs.campaign-run-id }} + path: ${{ runner.temp }}/campaign-sealer-receipt + + - name: Bind public candidate bytes to protected sealer receipts + shell: bash + env: + CANDIDATE_TAG: ${{ inputs.candidate-tag }} + TAP_REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + bottle_target="$(jq -er '.manifest.run.caller_commit' \ + "$RUNNER_TEMP/candidate-description.json")" + campaign_tag="$(jq -er \ + '.manifest.source.prefix_campaign_tag' \ + "$RUNNER_TEMP/candidate-description.json")" + campaign_target="$(jq -er '.run.caller_commit' \ + "$RUNNER_TEMP/candidate-campaign-manifest.json")" + python3 kandelo-main/scripts/homebrew-candidate-release-receipt.py \ + plan \ + --receipt \ + "$RUNNER_TEMP/bottle-sealer-receipt/candidate-release-receipt.json" \ + --release "$RUNNER_TEMP/candidate-release.json" \ + --release-assets "$RUNNER_TEMP/candidate-release-assets.json" \ + --repository "$TAP_REPOSITORY" --tag "$CANDIDATE_TAG" \ + --target-commit "$bottle_target" \ + --out "$RUNNER_TEMP/candidate-release-readback-plan.json" + python3 kandelo-main/scripts/homebrew-candidate-release-receipt.py \ + plan \ + --receipt \ + "$RUNNER_TEMP/campaign-sealer-receipt/candidate-campaign-release.json" \ + --release "$RUNNER_TEMP/candidate-campaign-release.json" \ + --release-assets \ + "$RUNNER_TEMP/candidate-campaign-release-assets.json" \ + --repository "$TAP_REPOSITORY" --tag "$campaign_tag" \ + --target-commit "$campaign_target" \ + --out "$RUNNER_TEMP/campaign-release-readback-plan.json" + + while IFS=$'\t' read -r name url; do + [ "$name" != candidate.json ] || continue + env -u GH_TOKEN -u GITHUB_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_URL \ + -u ACTIONS_RUNTIME_TOKEN \ + curl --disable --fail --location --silent --show-error \ + --output "$RUNNER_TEMP/homebrew-candidate-release/$name" \ + "$url" + done < <(jq -er '.assets[] | [.name, .url] | @tsv' \ + "$RUNNER_TEMP/candidate-release-readback-plan.json") + python3 kandelo-main/scripts/homebrew-candidate-release-receipt.py \ + verify-readback \ + --plan "$RUNNER_TEMP/candidate-release-readback-plan.json" \ + --asset-root "$RUNNER_TEMP/homebrew-candidate-release" + python3 kandelo-main/scripts/homebrew-candidate-release-receipt.py \ + verify-readback \ + --plan "$RUNNER_TEMP/campaign-release-readback-plan.json" \ + --asset-root "$RUNNER_TEMP/homebrew-candidate-campaign-release" + + - name: Checkout exact candidate tap source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: ${{ github.repository }} + ref: ${{ steps.candidate.outputs.source-tap-commit }} + path: tap-source + fetch-depth: 0 + + - name: Checkout exact native Homebrew campaign input + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: Homebrew/brew + ref: ${{ steps.candidate.outputs.native-homebrew-commit }} + path: native-homebrew + + - name: Install Nix for protected-main validation + uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 + with: + github-token: "" + + - name: Cache Nix store and flake evaluation + uses: DeterminateSystems/magic-nix-cache-action@908b263ff629f4cc17666315b7fd3ec127c6244d # v14 + with: + use-gha-cache: false + use-flakehub: false + + - name: Regenerate and admit exact package input on the merge tree + shell: bash + env: + EXPECTED_ABI: ${{ steps.candidate.outputs.abi }} + GH_TOKEN: ${{ github.token }} + MERGE_COMMIT: ${{ inputs.merge-commit }} + PRODUCER_SHA: ${{ inputs.producer-sha }} + run: | + set -euo pipefail + package_asset="$(cat \ + "$RUNNER_TEMP/candidate-package-asset.txt")" + candidate_package="$RUNNER_TEMP/homebrew-candidate-release/" + candidate_package+="$package_asset" + staging_tag="$(jq -er \ + '.staging_release.tag' "$candidate_package")" + pr_number="$(jq -er \ + '.staging_release.pr_number' "$candidate_package")" + staging_run="$(jq -er \ + '.staging_release.run_id' "$candidate_package")" + staging_attempt="$(jq -er \ + '.staging_release.attempt' "$candidate_package")" + cd kandelo-main + host="$(env -u GH_TOKEN -u GITHUB_TOKEN \ + bash scripts/dev-shell.sh rustc -vV | + sed -n 's/^host: //p')" + [ -n "$host" ] || { + echo "::error::cannot resolve the protected Rust host" + exit 2 + } + env -u GH_TOKEN -u GITHUB_TOKEN \ + bash scripts/dev-shell.sh bash \ + .github/scripts/prepare-homebrew-package-materializer.sh \ + --host-target "$host" + regenerated="$RUNNER_TEMP/regenerated-candidate-packages" + bash .github/scripts/materialize-homebrew-candidate-package-input.sh \ + --tag "$staging_tag" \ + --pr-number "$pr_number" \ + --run-id "$staging_run" \ + --run-attempt "$staging_attempt" \ + --producer-sha "$PRODUCER_SHA" \ + --expected-abi "$EXPECTED_ABI" \ + --exclude erlang-vfs,perl,perl-vfs,python-vfs,redis,texlive \ + --consumer-root "$PWD" \ + --consumer-sha "$MERGE_COMMIT" \ + --xtask "$PWD/target/$host/release/xtask" \ + --output-dir "$regenerated" + env -u GH_TOKEN -u GITHUB_TOKEN \ + python3 scripts/homebrew-bottle-candidate.py \ + admit-package-input \ + --candidate-package-input "$candidate_package" \ + --regenerated-package-input \ + "$regenerated/package-input.json" \ + --producer-commit "$PRODUCER_SHA" \ + --validated-main "$MERGE_COMMIT" \ + --validated-main-root "$PWD" \ + --out "$RUNNER_TEMP/admitted-package-input.json" + + - name: Recreate the exact prepared candidate tap + shell: bash + env: + CAMPAIGN_TAG: ${{ steps.candidate.outputs.campaign-tag }} + FORMULA: ${{ steps.candidate.outputs.formula }} + PRODUCER_SHA: ${{ inputs.producer-sha }} + TAP_NAME: kandelo-dev/tap-core + TAP_REPOSITORY: ${{ github.repository }} + TAP_SHA: ${{ steps.candidate.outputs.source-tap-commit }} + run: | + set -euo pipefail + bash kandelo-main/scripts/dev-shell.sh \ + python3 \ + kandelo-main/scripts/homebrew-prefix-campaign-publisher.py \ + prepare \ + --tap-root "$GITHUB_WORKSPACE/tap-source" \ + --kandelo-root "$GITHUB_WORKSPACE/producer" \ + --kandelo-commit "$PRODUCER_SHA" \ + --tap-repository "$TAP_REPOSITORY" \ + --tap-name "$TAP_NAME" \ + --source-tap-commit "$TAP_SHA" \ + --campaign-tag "$CAMPAIGN_TAG" \ + --dependencies '{"dependencies":[],"schema":1}' \ + --formula "$FORMULA" \ + --arch wasm32 \ + --work-root "$RUNNER_TEMP/candidate-campaign" \ + --receipt-out "$RUNNER_TEMP/candidate-campaign.json" \ + --github-env "$GITHUB_ENV" + + - name: Admit the exact merge with protected-main code + shell: bash + env: + CANDIDATE_TAG: ${{ inputs.candidate-tag }} + GH_TOKEN: ${{ github.token }} + MERGE_COMMIT: ${{ inputs.merge-commit }} + TAP_REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + description="$RUNNER_TEMP/candidate-description.json" + campaign_manifest="$RUNNER_TEMP/candidate-campaign-manifest.json" + current_tap_main="$(gh api \ + "/repos/$TAP_REPOSITORY/git/ref/heads/main" \ + --jq .object.sha)" + current_kandelo_main="$(gh api \ + /repos/Automattic/kandelo/git/ref/heads/main \ + --jq .object.sha)" + main_status="$(gh api \ + "/repos/Automattic/kandelo/compare/$MERGE_COMMIT...$current_kandelo_main" \ + --jq .status)" + case "$main_status" in + ahead|identical) ;; + *) + echo "::error::candidate merge is not on protected main" + exit 1 + ;; + esac + campaign_tag="$(jq -er \ + '.manifest.source.prefix_campaign_tag' "$description")" + env -u GH_TOKEN -u GITHUB_TOKEN \ + python3 \ + kandelo-main/scripts/homebrew-candidate-campaign.py \ + admit \ + --candidate "$campaign_manifest" \ + --campaign "$RUNNER_TEMP/candidate-prefix-campaign.json" \ + --candidate-tag "$campaign_tag" \ + --completed-run-evidence \ + "$RUNNER_TEMP/completed-candidate-campaign-run.json" \ + --kandelo-main-root "$GITHUB_WORKSPACE/kandelo-main" \ + --producer-root "$GITHUB_WORKSPACE/producer" \ + --tap-root "$GITHUB_WORKSPACE/tap-source" \ + --native-brew-root "$GITHUB_WORKSPACE/native-homebrew" \ + --merge-commit "$MERGE_COMMIT" \ + --current-kandelo-main "$current_kandelo_main" \ + --current-tap-main "$current_tap_main" \ + --out "$RUNNER_TEMP/candidate-campaign-admission.json" + env -u GH_TOKEN -u GITHUB_TOKEN \ + python3 kandelo-main/scripts/homebrew-bottle-candidate.py \ + materialize \ + --candidate-root \ + "$RUNNER_TEMP/homebrew-candidate-release" \ + --candidate-tag "$CANDIDATE_TAG" \ + --completed-run-evidence \ + "$RUNNER_TEMP/completed-candidate-run.json" \ + --kandelo-root "$GITHUB_WORKSPACE/kandelo-main" \ + --tap-root "$GITHUB_WORKSPACE/tap-source" \ + --merge-commit "$MERGE_COMMIT" \ + --current-kandelo-main "$current_kandelo_main" \ + --current-tap-main "$current_tap_main" \ + --admitted-package-input \ + "$RUNNER_TEMP/admitted-package-input.json" \ + --out-build-handoff \ + "$RUNNER_TEMP/homebrew-build-handoff" \ + --out-oci-child "$RUNNER_TEMP/homebrew-oci-child" \ + --out-package-input \ + "$RUNNER_TEMP/candidate-package-input.json" \ + --out-receipt "$RUNNER_TEMP/candidate-promotion.json" + + - name: Re-probe the collision-sensitive child reference + shell: bash + env: + FORMULA: ${{ steps.candidate.outputs.formula }} + TAP_REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + receipt="$RUNNER_TEMP/homebrew-oci-child/receipt.json" + remote="ghcr.io/${TAP_REPOSITORY,,}/$FORMULA" + child_ref="$(jq -er '.oci.transport_tag' "$receipt")" + expected_digest="$(jq -er '.oci.manifest.digest' "$receipt")" + printf '{"auths":{}}\n' \ + >"$RUNNER_TEMP/anonymous-oras.json" + env -u GH_TOKEN -u GITHUB_TOKEN \ + bash kandelo-main/scripts/dev-shell.sh \ + python3 kandelo-main/scripts/homebrew-oci-layout.py \ + probe-registry --kind manifest --remote "$remote" \ + --reference "$child_ref" \ + --registry-config "$RUNNER_TEMP/anonymous-oras.json" \ + --out-result "$RUNNER_TEMP/live-child-probe.json" + jq -e --arg digest "$expected_digest" ' + (.status == "missing" and .digest == null) or + (.status == "present" and .digest == $digest) + ' "$RUNNER_TEMP/live-child-probe.json" >/dev/null || { + echo "::error::candidate child ref now contains different bytes" + exit 1 + } + + - name: Upload exact build handoff for the publisher + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: >- + homebrew-build-handoff-${{ steps.candidate.outputs.formula }}-wasm32-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/homebrew-build-handoff + compression-level: 0 + if-no-files-found: error + retention-days: 2 + + - name: Upload exact OCI child for the publisher + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: >- + homebrew-oci-child-${{ steps.candidate.outputs.formula }}-wasm32-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/homebrew-oci-child + compression-level: 0 + if-no-files-found: error + retention-days: 2 + + - name: Retain bounded promotion evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: >- + homebrew-candidate-promotion-${{ steps.candidate.outputs.formula }}-wasm32-attempt-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/candidate-promotion.json + ${{ runner.temp }}/candidate-campaign-admission.json + ${{ runner.temp }}/candidate-package-input.json + ${{ runner.temp }}/live-child-probe.json + compression-level: 0 + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/reusable-homebrew-bottle-publish.yml b/.github/workflows/reusable-homebrew-bottle-publish.yml index d4e9645335..9646fefcbe 100644 --- a/.github/workflows/reusable-homebrew-bottle-publish.yml +++ b/.github/workflows/reusable-homebrew-bottle-publish.yml @@ -57,6 +57,18 @@ on: prefix-campaign-dependencies: type: string default: "" + candidate-pr-number: + type: string + default: "" + candidate-package-staging-tag: + type: string + default: "" + candidate-promotion-tag: + type: string + default: "" + candidate-producer-sha: + type: string + default: "" jobs: plan: @@ -81,7 +93,16 @@ jobs: prefix-campaign-tag: ${{ steps.trust.outputs.prefix-campaign-tag }} prefix-campaign-dependencies: ${{ steps.trust.outputs.prefix-campaign-dependencies }} prefix-campaign-layout-sha256: ${{ steps.campaign-source.outputs.prefix-campaign-layout-sha256 }} + prefix-campaign-prepared-tap-commit: ${{ steps.campaign-source.outputs.prefix-campaign-prepared-tap-commit }} + prefix-campaign-prepared-tap-tree: ${{ steps.campaign-source.outputs.prefix-campaign-prepared-tap-tree }} artifact-name-prefix: ${{ steps.artifact-scope.outputs.prefix }} + candidate-mode: ${{ steps.trust.outputs.candidate-mode }} + candidate-pr-number: ${{ steps.trust.outputs.candidate-pr-number }} + candidate-package-staging-tag: ${{ steps.trust.outputs.candidate-package-staging-tag }} + candidate-workflow-authority-sha: ${{ steps.candidate-authority.outputs.kandelo-sha }} + candidate-tap-workflow-authority-sha: ${{ steps.trust.outputs.candidate-tap-workflow-authority-sha }} + candidate-promotion-mode: ${{ steps.trust.outputs.candidate-promotion-mode }} + bottle-producer-sha: ${{ steps.bottle-producer.outputs.sha }} steps: - name: Validate caller trust boundary id: trust @@ -90,6 +111,7 @@ jobs: CALLER_EVENT_NAME: ${{ github.event_name }} CALLER_REF: ${{ github.ref }} CALLER_REPOSITORY: ${{ github.repository }} + CALLER_SHA: ${{ github.sha }} CALLER_WORKFLOW_REF: ${{ github.workflow_ref }} DEFER_TAP_FINALIZATION: ${{ inputs.defer-tap-finalization }} DRY_RUN: ${{ inputs.dry-run }} @@ -107,6 +129,10 @@ jobs: TAP_REPOSITORY: ${{ inputs.tap-repository }} TAP_REF: ${{ inputs.tap-ref }} BOTTLE_ROOT_URL: ${{ inputs.bottle-root-url }} + CANDIDATE_PACKAGE_STAGING_TAG: ${{ inputs.candidate-package-staging-tag }} + CANDIDATE_PR_NUMBER: ${{ inputs.candidate-pr-number }} + CANDIDATE_PROMOTION_TAG: ${{ inputs.candidate-promotion-tag }} + CANDIDATE_PRODUCER_SHA: ${{ inputs.candidate-producer-sha }} run: | set -euo pipefail normalize_dry_run_source_ref() { @@ -246,6 +272,8 @@ jobs: validated_campaign_tag="" validated_campaign_dependencies="" campaign_caller="$CALLER_REPOSITORY/.github/workflows/prefix-campaign-bottles.yml@refs/heads/main" + candidate_caller="$CALLER_REPOSITORY/.github/workflows/candidate-bottles.yml@refs/heads/main" + promotion_caller="$CALLER_REPOSITORY/.github/workflows/promote-candidate-bottle.yml@refs/heads/main" if [ "$CALLER_WORKFLOW_REF" = "$campaign_caller" ]; then [ "$DEFER_TAP_FINALIZATION" = "true" ] && [ "$FORCE_REBUILD" = "true" ] && @@ -265,17 +293,133 @@ jobs: validated_campaign_dependencies="$( normalize_campaign_dependencies "$PREFIX_CAMPAIGN_DEPENDENCIES" )" + [ -z "$CANDIDATE_PR_NUMBER" ] && + [ -z "$CANDIDATE_PACKAGE_STAGING_TAG" ] && + [ -z "$CANDIDATE_PROMOTION_TAG" ] && + [ -z "$CANDIDATE_PRODUCER_SHA" ] || { + echo "::error::prefix campaigns cannot carry candidate authority" + exit 2 + } validated_campaign_mode=true validated_campaign_tag="$PREFIX_CAMPAIGN_TAG" - else + elif [ "$CALLER_WORKFLOW_REF" != "$candidate_caller" ] && + [ "$CALLER_WORKFLOW_REF" != "$promotion_caller" ]; then [ "$DEFER_TAP_FINALIZATION" = "false" ] && [ -z "$PREFIX_CAMPAIGN_TAG" ] && [ -z "$PREFIX_CAMPAIGN_DEPENDENCIES" ] || { echo "::error::only the reviewed prefix campaign caller may defer tap finalization or pass campaign authority"; exit 2; } fi - if [ "$DRY_RUN" = "true" ] && + validated_candidate_mode=false + validated_candidate_pr_number="" + validated_candidate_staging_tag="" + validated_candidate_tap_authority="" + validated_candidate_promotion_mode=false + if [ "$CALLER_WORKFLOW_REF" = "$candidate_caller" ]; then + [ "$DRY_RUN" = "true" ] && + [ "$FORCE_REBUILD" = "true" ] && + [ "$DEFER_TAP_FINALIZATION" = "false" ] && + [ "$REQUIRE_VFS_ACCEPTANCE" = "false" ] && + [ -z "$PACKAGE_GENERATION_WASM32" ] && + [ -z "$PACKAGE_GENERATION_WASM64" ] || { + echo "::error::candidate bottles require an isolated forced no-write invocation"; exit 2; + } + [[ "$FORMULAE" =~ ^[a-z0-9][a-z0-9._-]{0,254}$ ]] && + [ "$ARCHES" = wasm32 ] || { + echo "::error::candidate bottle v1 requires one wasm32 Formula" + exit 2 + } + [[ "$CANDIDATE_PR_NUMBER" =~ ^[1-9][0-9]*$ ]] && + [[ "$CALLER_SHA" =~ ^[0-9a-f]{40}$ ]] && + [[ "$KANDELO_REF" =~ ^[0-9a-f]{40}$ ]] && + [[ "$TAP_REF" =~ ^[0-9a-f]{40}$ ]] || { + echo "::error::candidate bottles require exact PR and source authority"; exit 2; + } + staging_pattern="^pr-${CANDIDATE_PR_NUMBER}-staging-run-[1-9][0-9]*-attempt-[1-9][0-9]*$" + [[ "$CANDIDATE_PACKAGE_STAGING_TAG" =~ $staging_pattern ]] || { + echo "::error::candidate package staging tag differs from its PR"; exit 2; + } + [ -z "$CANDIDATE_PROMOTION_TAG" ] && + [ -z "$CANDIDATE_PRODUCER_SHA" ] || { + echo "::error::candidate builds cannot carry promotion authority" + exit 2 + } + validated_candidate_mode=true + candidate_campaign_pattern="^homebrew-prefix-campaign-candidate-pr-${CANDIDATE_PR_NUMBER}-run-[1-9][0-9]*-attempt-[1-9][0-9]*-sha256-[0-9a-f]{64}$" + [[ "$PREFIX_CAMPAIGN_TAG" =~ $candidate_campaign_pattern ]] || { + echo "::error::candidate bottle needs an immutable candidate campaign"; exit 2; + } + validated_campaign_dependencies="$( + normalize_campaign_dependencies "$PREFIX_CAMPAIGN_DEPENDENCIES" + )" + [ "$validated_campaign_dependencies" = \ + '{"dependencies":[],"schema":1}' ] || { + echo "::error::candidate bottle v1 supports only leaf Formulae" + exit 2 + } + validated_campaign_mode=true + validated_campaign_tag="$PREFIX_CAMPAIGN_TAG" + validated_candidate_pr_number="$CANDIDATE_PR_NUMBER" + validated_candidate_staging_tag="$CANDIDATE_PACKAGE_STAGING_TAG" + validated_candidate_tap_authority="$CALLER_SHA" + validated_kandelo_ref="$KANDELO_REF" + validated_tap_ref="$TAP_REF" + validated_generation_wasm32="" + validated_generation_wasm64="" + validated_generation_kind="none" + elif [ "$CALLER_WORKFLOW_REF" = "$promotion_caller" ]; then + [ "$DRY_RUN" = "false" ] && + [ "$FORCE_REBUILD" = "true" ] && + [ "$DEFER_TAP_FINALIZATION" = "true" ] && + [ "$REQUIRE_VFS_ACCEPTANCE" = "false" ] || { + echo "::error::candidate promotion requires the deferred write lane"; exit 2; + } + [[ "$FORMULAE" =~ ^[a-z0-9][a-z0-9._-]{0,254}$ ]] && + [ "$ARCHES" = wasm32 ] && + [[ "$KANDELO_REF" =~ ^[0-9a-f]{40}$ ]] && + [[ "$TAP_REF" =~ ^[0-9a-f]{40}$ ]] && + [[ "$CANDIDATE_PRODUCER_SHA" =~ ^[0-9a-f]{40}$ ]] && + [[ "$CANDIDATE_PROMOTION_TAG" =~ ^homebrew-bottle-candidate-pr-[1-9][0-9]*-run-[1-9][0-9]*-attempt-[1-9][0-9]*-sha256-[0-9a-f]{64}$ ]] || { + echo "::error::candidate promotion v1 requires one wasm32 leaf and exact identities"; exit 2; + } + [ -z "$CANDIDATE_PR_NUMBER" ] && + [ -z "$CANDIDATE_PACKAGE_STAGING_TAG" ] || { + echo "::error::candidate promotion derives PR staging identity from its immutable candidate"; exit 2; + } + validated_campaign_dependencies="$( + normalize_campaign_dependencies "$PREFIX_CAMPAIGN_DEPENDENCIES" + )" + [ "$validated_campaign_dependencies" = \ + '{"dependencies":[],"schema":1}' ] && + [[ "$PREFIX_CAMPAIGN_TAG" =~ ^homebrew-prefix-campaign-candidate-pr-[1-9][0-9]*-run-[1-9][0-9]*-attempt-[1-9][0-9]*-sha256-[0-9a-f]{64}$ ]] || { + echo "::error::candidate promotion requires its immutable candidate campaign"; exit 2; + } + validated_campaign_mode=true + validated_campaign_tag="$PREFIX_CAMPAIGN_TAG" + validated_candidate_promotion_mode=true + validated_kandelo_ref="$(normalize_write_kandelo_ref "$KANDELO_REF")" + validated_tap_ref="$(normalize_write_tap_ref "$TAP_REF")" + validated_generation_wasm32="$( + normalize_package_generation wasm32 "$PACKAGE_GENERATION_WASM32" + )" + case "$validated_generation_wasm32" in + package-generation-rootfs-wasm32-*) + [ -z "$PACKAGE_GENERATION_WASM64" ] || { + echo "::error::candidate promotion rootfs input forbids wasm64"; exit 2; + } + validated_generation_wasm64="" + validated_generation_kind="rootfs-wasm32" + ;; + *) echo "::error::candidate promotion v1 requires exact rootfs-wasm32 input"; exit 2 ;; + esac + elif [ "$DRY_RUN" = "true" ] && [ "$validated_campaign_mode" != "true" ]; then + [ -z "$CANDIDATE_PR_NUMBER" ] && + [ -z "$CANDIDATE_PACKAGE_STAGING_TAG" ] && + [ -z "$CANDIDATE_PROMOTION_TAG" ] && + [ -z "$CANDIDATE_PRODUCER_SHA" ] || { + echo "::error::only the reviewed candidate caller may pass candidate authority"; exit 2; + } [ "$CALLER_WORKFLOW_REF" = "$CALLER_REPOSITORY/.github/workflows/dry-run-bottles.yml@refs/heads/main" ] || { echo "::error::dry-run publication requires the reviewed tap dry-run workflow"; exit 2; } @@ -308,6 +452,12 @@ jobs: validated_generation_kind="browser-inputs" fi else + [ -z "$CANDIDATE_PR_NUMBER" ] && + [ -z "$CANDIDATE_PACKAGE_STAGING_TAG" ] && + [ -z "$CANDIDATE_PROMOTION_TAG" ] && + [ -z "$CANDIDATE_PRODUCER_SHA" ] || { + echo "::error::only the reviewed candidate caller may pass candidate authority"; exit 2; + } if [ "$validated_campaign_mode" != "true" ]; then case "$CALLER_WORKFLOW_REF" in "$CALLER_REPOSITORY/.github/workflows/publish-bottles.yml@refs/heads/main"|\ @@ -355,6 +505,11 @@ jobs: echo "prefix-campaign-mode=$validated_campaign_mode" echo "prefix-campaign-tag=$validated_campaign_tag" echo "prefix-campaign-dependencies=$validated_campaign_dependencies" + echo "candidate-mode=$validated_candidate_mode" + echo "candidate-pr-number=$validated_candidate_pr_number" + echo "candidate-package-staging-tag=$validated_candidate_staging_tag" + echo "candidate-tap-workflow-authority-sha=$validated_candidate_tap_authority" + echo "candidate-promotion-mode=$validated_candidate_promotion_mode" } >> "$GITHUB_OUTPUT" - name: Admit exact Kandelo main source @@ -386,7 +541,7 @@ jobs: } - name: Admit prefix-campaign Kandelo main history - if: ${{ steps.trust.outputs.prefix-campaign-mode == 'true' }} + if: ${{ steps.trust.outputs.prefix-campaign-mode == 'true' && steps.trust.outputs.candidate-mode != 'true' }} shell: bash env: GH_TOKEN: ${{ github.token }} @@ -416,6 +571,115 @@ jobs: path: kandelo submodules: false + - name: Bind candidate validator to the exact merge base + id: candidate-authority + if: ${{ steps.trust.outputs.candidate-mode == 'true' }} + shell: bash + env: + GH_TOKEN: ${{ github.token }} + KANDELO_REPOSITORY: ${{ inputs.kandelo-repository }} + PR_NUMBER: ${{ steps.trust.outputs.candidate-pr-number }} + run: | + set -euo pipefail + main_sha="$(gh api "/repos/$KANDELO_REPOSITORY/git/ref/heads/main" --jq .object.sha)" + base_sha="$(gh api "/repos/$KANDELO_REPOSITORY/pulls/$PR_NUMBER" --jq .base.sha)" + [[ "$main_sha" =~ ^[0-9a-f]{40}$ ]] && + [ "$base_sha" = "$main_sha" ] || { + echo "::error::candidate PR base must equal current protected main" + exit 1 + } + # WHY: candidate code must never choose an older trusted validator. + # The exact PR base both produced this workflow and will become the + # first parent of the only admitted merge commit. + echo "kandelo-sha=$main_sha" >>"$GITHUB_OUTPUT" + + - name: Checkout protected candidate validator authority + if: ${{ steps.trust.outputs.candidate-mode == 'true' }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ steps.candidate-authority.outputs.kandelo-sha }} + path: candidate-authority + submodules: false + + - name: Checkout exact candidate caller as inert data + if: ${{ steps.trust.outputs.candidate-mode == 'true' || steps.trust.outputs.candidate-promotion-mode == 'true' }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ${{ inputs.tap-repository }} + ref: ${{ github.sha }} + path: candidate-caller-data + submodules: false + + - name: Require the candidate caller to pin exact Kandelo authority + if: ${{ steps.trust.outputs.candidate-mode == 'true' || steps.trust.outputs.candidate-promotion-mode == 'true' }} + shell: bash + env: + CANDIDATE_MODE: ${{ steps.trust.outputs.candidate-mode }} + CANDIDATE_SHA: ${{ steps.candidate-authority.outputs.kandelo-sha }} + MERGE_SHA: ${{ steps.trust.outputs.kandelo-ref }} + run: | + set -euo pipefail + if [ "$CANDIDATE_MODE" = true ]; then + authority_root=candidate-authority + expected_sha="$CANDIDATE_SHA" + mode=bottle + else + authority_root=kandelo + expected_sha="$MERGE_SHA" + mode=promotion + fi + # WHY: repository_dispatch identifies caller commit C, but only C's + # literal reusable-workflow SHA proves which Kandelo code GitHub ran. + python3 "$authority_root/scripts/homebrew-candidate-caller-pins.py" \ + validate --tap-root "$GITHUB_WORKSPACE/candidate-caller-data" \ + --mode "$mode" --kandelo-sha "$expected_sha" + + - name: Checkout promoted bottle producer + if: ${{ steps.trust.outputs.candidate-promotion-mode == 'true' }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ inputs.candidate-producer-sha }} + path: bottle-producer + fetch-depth: 0 + submodules: false + + - name: Admit promoted producer on exact-main tree + if: ${{ steps.trust.outputs.candidate-promotion-mode == 'true' }} + shell: bash + env: + GH_TOKEN: ${{ github.token }} + KANDELO_REPOSITORY: ${{ inputs.kandelo-repository }} + MAIN_SHA: ${{ steps.trust.outputs.kandelo-ref }} + PRODUCER_SHA: ${{ inputs.candidate-producer-sha }} + run: | + set -euo pipefail + [ "$(git -C kandelo rev-parse 'HEAD^{tree}')" = \ + "$(git -C bottle-producer rev-parse 'HEAD^{tree}')" ] || { + echo "::error::promoted producer tree differs from main" + exit 1 + } + status="$(gh api \ + "/repos/$KANDELO_REPOSITORY/compare/$PRODUCER_SHA...main" \ + --jq .status)" + case "$status" in + ahead|identical) ;; + *) + echo "::error::promoted producer is not on protected main" + exit 1 + ;; + esac + [ "$(git -C kandelo rev-parse HEAD)" = "$MAIN_SHA" ] && + [ "$(git -C bottle-producer rev-parse HEAD)" = \ + "$PRODUCER_SHA" ] || { + echo "::error::promotion source checkout changed" + exit 1 + } + - name: Checkout tap uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -430,6 +694,7 @@ jobs: shell: bash env: PREFIX_CAMPAIGN_MODE: ${{ steps.trust.outputs.prefix-campaign-mode }} + CANDIDATE_MODE: ${{ steps.trust.outputs.candidate-mode }} DRY_RUN: ${{ inputs.dry-run }} REQUESTED_KANDELO_SHA: ${{ inputs.kandelo-ref }} run: | @@ -443,9 +708,10 @@ jobs: echo "::error::tap checkout did not resolve to a commit SHA"; exit 2; } if { [ "$DRY_RUN" = "false" ] || + [ "$CANDIDATE_MODE" = "true" ] || [ "$PREFIX_CAMPAIGN_MODE" = "true" ]; } && [ "$kandelo_sha" != "$REQUESTED_KANDELO_SHA" ]; then - echo "::error::Kandelo checkout differs from the exact admitted main commit" + echo "::error::Kandelo checkout differs from the exact requested source" exit 2 fi { @@ -453,8 +719,28 @@ jobs: echo "tap-sha=$tap_sha" } >> "$GITHUB_OUTPUT" + - name: Select immutable bottle producer + id: bottle-producer + shell: bash + env: + CANDIDATE_PROMOTION_MODE: >- + ${{ steps.trust.outputs.candidate-promotion-mode }} + PROMOTION_PRODUCER: ${{ inputs.candidate-producer-sha }} + SOURCE_COMMIT: ${{ steps.source-commits.outputs.kandelo-sha }} + run: | + set -euo pipefail + producer="$SOURCE_COMMIT" + if [ "$CANDIDATE_PROMOTION_MODE" = true ]; then + producer="$PROMOTION_PRODUCER" + fi + [[ "$producer" =~ ^[0-9a-f]{40}$ ]] || { + echo "::error::bottle producer is not an exact commit" + exit 2 + } + echo "sha=$producer" >>"$GITHUB_OUTPUT" + - name: Bind write tap source to protected main history - if: ${{ !inputs.dry-run || steps.trust.outputs.prefix-campaign-mode == 'true' }} + if: ${{ !inputs.dry-run || steps.trust.outputs.prefix-campaign-mode == 'true' || steps.trust.outputs.candidate-mode == 'true' }} shell: bash env: GH_TOKEN: ${{ github.token }} @@ -492,12 +778,28 @@ jobs: TAP_NAME: ${{ inputs.tap-name }} TAP_REPOSITORY: ${{ inputs.tap-repository }} TAP_SHA: ${{ steps.source-commits.outputs.tap-sha }} + CANDIDATE_MODE: ${{ steps.trust.outputs.candidate-mode }} + CANDIDATE_PROMOTION_MODE: >- + ${{ steps.trust.outputs.candidate-promotion-mode }} + BOTTLE_PRODUCER_SHA: ${{ steps.bottle-producer.outputs.sha }} run: | set -euo pipefail - python3 kandelo/scripts/homebrew-prefix-campaign-publisher.py \ + authority_root=kandelo + producer_args=() + campaign_commit="$KANDELO_SHA" + if [ "$CANDIDATE_MODE" = true ]; then + authority_root=candidate-authority + producer_args+=(--kandelo-root "$GITHUB_WORKSPACE/kandelo") + elif [ "$CANDIDATE_PROMOTION_MODE" = true ]; then + producer_args+=(--kandelo-root \ + "$GITHUB_WORKSPACE/bottle-producer") + campaign_commit="$BOTTLE_PRODUCER_SHA" + fi + python3 "$authority_root/scripts/homebrew-prefix-campaign-publisher.py" \ prepare \ --tap-root "$GITHUB_WORKSPACE/tap" \ - --kandelo-commit "$KANDELO_SHA" \ + "${producer_args[@]}" \ + --kandelo-commit "$campaign_commit" \ --tap-repository "$TAP_REPOSITORY" \ --tap-name "$TAP_NAME" \ --source-tap-commit "$TAP_SHA" \ @@ -515,6 +817,8 @@ jobs: env: ADMISSION_KIND: >- ${{ steps.campaign-source.outputs.prefix-campaign-destination-admission-kind }} + CANDIDATE_MODE: >- + ${{ steps.trust.outputs.candidate-mode }} DRY_RUN: ${{ inputs.dry-run }} PREFIX_CAMPAIGN_MODE: >- ${{ steps.trust.outputs.prefix-campaign-mode }} @@ -522,7 +826,8 @@ jobs: set -euo pipefail prefix="" if [ "$PREFIX_CAMPAIGN_MODE" = "true" ] && - [ "$DRY_RUN" = "true" ]; then + [ "$DRY_RUN" = "true" ] && + [ "$CANDIDATE_MODE" != "true" ]; then # WHY: the ordinary publisher runs later in this same workflow # run. A fixed bootstrap-only prefix prevents Actions artifact # names from colliding without letting the caller choose a name. @@ -904,7 +1209,7 @@ jobs: build-and-test: needs: [plan] - if: ${{ needs.plan.outputs.matrix != '[]' }} + if: ${{ needs.plan.outputs.matrix != '[]' && needs.plan.outputs.candidate-promotion-mode != 'true' }} runs-on: ubuntu-latest timeout-minutes: 1440 # WHY: upload-artifact uses the current run's scoped artifact service. @@ -926,6 +1231,16 @@ jobs: path: kandelo submodules: false + - name: Checkout protected candidate package-reader authority + if: ${{ needs.plan.outputs.candidate-mode == 'true' }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ needs.plan.outputs.candidate-workflow-authority-sha }} + path: candidate-authority + submodules: false + - name: Checkout tap uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -997,7 +1312,8 @@ jobs: run: | set -euo pipefail cd kandelo - host="$(bash scripts/dev-shell.sh rustc -vV | sed -n 's/^host: //p')" + host="$(env -u GH_TOKEN -u GITHUB_TOKEN \ + bash scripts/dev-shell.sh rustc -vV | sed -n 's/^host: //p')" [ -n "$host" ] || { echo "::error::unable to resolve the Rust host target"; exit 2; } @@ -1027,6 +1343,66 @@ jobs: } echo "WASM_POSIX_BINARY_INDEX_URL=$index_url" >> "$GITHUB_ENV" + - name: Materialize immutable candidate Formula runtime packages + if: ${{ needs.plan.outputs.candidate-mode == 'true' }} + shell: bash + env: + CANDIDATE_PR_NUMBER: ${{ needs.plan.outputs.candidate-pr-number }} + CANDIDATE_STAGING_TAG: ${{ needs.plan.outputs.candidate-package-staging-tag }} + KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + EXPECTED_ABI: ${{ needs.plan.outputs.abi }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + cd candidate-authority + [[ "$CANDIDATE_STAGING_TAG" =~ ^pr-[1-9][0-9]*-staging-run-([1-9][0-9]*)-attempt-([1-9][0-9]*)$ ]] + staging_run="${BASH_REMATCH[1]}" + staging_attempt="${BASH_REMATCH[2]}" + host="$(env -u GH_TOKEN -u GITHUB_TOKEN \ + bash scripts/dev-shell.sh rustc -vV | sed -n 's/^host: //p')" + [ -n "$host" ] || { + echo "::error::unable to resolve the Rust host target"; exit 2; + } + env -u GH_TOKEN -u GITHUB_TOKEN \ + bash scripts/dev-shell.sh bash \ + .github/scripts/prepare-homebrew-package-materializer.sh \ + --host-target "$host" + authority_xtask="$PWD/target/$host/release/xtask" + cd ../kandelo + output="$RUNNER_TEMP/homebrew-candidate-packages" + # The wrapper and xtask both come from protected main. They read the + # candidate registry as inert data while the read token is present; + # no executable from the candidate receives that token. + bash ../candidate-authority/.github/scripts/materialize-homebrew-candidate-package-input.sh \ + --tag "$CANDIDATE_STAGING_TAG" \ + --pr-number "$CANDIDATE_PR_NUMBER" \ + --run-id "$staging_run" \ + --run-attempt "$staging_attempt" \ + --producer-sha "$KANDELO_SHA" \ + --expected-abi "$EXPECTED_ABI" \ + --exclude erlang-vfs,perl,perl-vfs,python-vfs,redis,texlive \ + --consumer-root "$PWD" \ + --consumer-sha "$KANDELO_SHA" \ + --xtask "$authority_xtask" \ + --output-dir "$output" + index_url="$(cat "$output/resolver/index-url.txt")" + [ "$index_url" = \ + "file://$output/resolver/archives/index.toml" ] || { + echo "::error::candidate package resolver is not the frozen local index" + exit 2 + } + echo "WASM_POSIX_BINARY_INDEX_URL=$index_url" >>"$GITHUB_ENV" + + - name: Upload candidate package-input identity + if: ${{ needs.plan.outputs.candidate-mode == 'true' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: homebrew-candidate-package-input-${{ matrix.formula }}-${{ matrix.arch }}-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/homebrew-candidate-packages/package-input.json + compression-level: 0 + if-no-files-found: error + retention-days: 2 + - name: Prepare sealed campaign Formula dependencies if: ${{ needs.plan.outputs.prefix-campaign-mode == 'true' }} shell: bash @@ -1036,6 +1412,9 @@ jobs: CAMPAIGN_TAG: ${{ needs.plan.outputs.prefix-campaign-tag }} FORMULA: ${{ matrix.formula }} KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + BOTTLE_PRODUCER_SHA: ${{ needs.plan.outputs.bottle-producer-sha }} + CANDIDATE_PROMOTION_MODE: >- + ${{ needs.plan.outputs.candidate-promotion-mode }} TAP_NAME: ${{ inputs.tap-name }} TAP_REPOSITORY: ${{ inputs.tap-repository }} TAP_SHA: ${{ needs.plan.outputs.tap-sha }} @@ -1762,6 +2141,9 @@ jobs: CAMPAIGN_TAG: ${{ needs.plan.outputs.prefix-campaign-tag }} FORMULA: ${{ matrix.formula }} KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + BOTTLE_PRODUCER_SHA: ${{ needs.plan.outputs.bottle-producer-sha }} + CANDIDATE_PROMOTION_MODE: >- + ${{ needs.plan.outputs.candidate-promotion-mode }} TAP_NAME: ${{ inputs.tap-name }} TAP_REPOSITORY: ${{ inputs.tap-repository }} TAP_SHA: ${{ needs.plan.outputs.tap-sha }} @@ -2035,6 +2417,16 @@ jobs: path: kandelo submodules: false + - name: Checkout promoted bottle producer for upload + if: ${{ needs.plan.outputs.candidate-promotion-mode == 'true' }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ needs.plan.outputs.bottle-producer-sha }} + path: bottle-producer + submodules: false + - name: Checkout exact tap source for upload validation uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -2058,6 +2450,9 @@ jobs: shell: bash env: KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + BOTTLE_PRODUCER_SHA: ${{ needs.plan.outputs.bottle-producer-sha }} + CANDIDATE_PROMOTION_MODE: >- + ${{ needs.plan.outputs.candidate-promotion-mode }} run: | set -euo pipefail [[ "$KANDELO_SHA" =~ ^[0-9a-f]{40}$ ]] || { @@ -2076,15 +2471,26 @@ jobs: CAMPAIGN_TAG: ${{ needs.plan.outputs.prefix-campaign-tag }} FORMULA: ${{ matrix.formula }} KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + BOTTLE_PRODUCER_SHA: ${{ needs.plan.outputs.bottle-producer-sha }} + CANDIDATE_PROMOTION_MODE: >- + ${{ needs.plan.outputs.candidate-promotion-mode }} TAP_NAME: ${{ inputs.tap-name }} TAP_REPOSITORY: ${{ inputs.tap-repository }} TAP_SHA: ${{ needs.plan.outputs.tap-sha }} run: | set -euo pipefail + producer_args=() + campaign_commit="$KANDELO_SHA" + if [ "$CANDIDATE_PROMOTION_MODE" = true ]; then + producer_args+=(--kandelo-root \ + "$GITHUB_WORKSPACE/bottle-producer") + campaign_commit="$BOTTLE_PRODUCER_SHA" + fi python3 kandelo/scripts/homebrew-prefix-campaign-publisher.py \ prepare \ --tap-root "$GITHUB_WORKSPACE/tap" \ - --kandelo-commit "$KANDELO_SHA" \ + "${producer_args[@]}" \ + --kandelo-commit "$campaign_commit" \ --tap-repository "$TAP_REPOSITORY" \ --tap-name "$TAP_NAME" \ --source-tap-commit "$TAP_SHA" \ @@ -2153,15 +2559,90 @@ jobs: name: ${{ needs.plan.outputs.artifact-name-prefix }}homebrew-oci-child-${{ matrix.formula }}-${{ matrix.arch }}-attempt-${{ github.run_attempt }} path: ${{ runner.temp }}/homebrew-oci-child + - name: Download exact candidate promotion admission + id: candidate-promotion + if: ${{ needs.plan.outputs.candidate-promotion-mode == 'true' }} + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: >- + homebrew-candidate-promotion-${{ matrix.formula }}-${{ matrix.arch }}-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/homebrew-candidate-promotion + + - name: Bind promotion artifacts to the exact merged candidate + id: validate-candidate-promotion + if: >- + ${{ + needs.plan.outputs.candidate-promotion-mode == 'true' && + steps.build-handoff.outcome == 'success' && + steps.oci-child.outcome == 'success' && + steps.candidate-promotion.outcome == 'success' + }} + shell: bash + env: + ABI: ${{ needs.plan.outputs.abi }} + ARCH: ${{ matrix.arch }} + CAMPAIGN_LAYOUT: >- + ${{ needs.plan.outputs.prefix-campaign-layout-sha256 }} + CAMPAIGN_TAG: ${{ needs.plan.outputs.prefix-campaign-tag }} + CANDIDATE_TAG: ${{ inputs.candidate-promotion-tag }} + FORMULA: ${{ matrix.formula }} + MERGE_COMMIT: ${{ needs.plan.outputs.kandelo-sha }} + PRODUCER_COMMIT: ${{ needs.plan.outputs.bottle-producer-sha }} + TAP_CHECKOUT_COMMIT: >- + ${{ needs.plan.outputs.prefix-campaign-prepared-tap-commit }} + TAP_COMMIT: ${{ needs.plan.outputs.tap-sha }} + run: | + set -euo pipefail + [ -z "${GH_TOKEN:-}" ] && [ -z "${GITHUB_TOKEN:-}" ] || { + echo "::error::promotion admission received credentials" + exit 2 + } + python3 kandelo/scripts/homebrew-candidate-campaign.py \ + validate-admission \ + --receipt \ + "$RUNNER_TEMP/homebrew-candidate-promotion/candidate-campaign-admission.json" \ + --candidate-tag "$CAMPAIGN_TAG" \ + --producer-commit "$PRODUCER_COMMIT" \ + --merge-commit "$MERGE_COMMIT" \ + --source-tap-commit "$TAP_COMMIT" \ + --abi "$ABI" \ + --guest-layout-sha256 "$CAMPAIGN_LAYOUT" + python3 kandelo/scripts/homebrew-bottle-candidate.py \ + validate-promotion \ + --receipt \ + "$RUNNER_TEMP/homebrew-candidate-promotion/candidate-promotion.json" \ + --candidate-tag "$CANDIDATE_TAG" \ + --producer-commit "$PRODUCER_COMMIT" \ + --merge-commit "$MERGE_COMMIT" \ + --tap-commit "$TAP_COMMIT" \ + --tap-checkout-commit "$TAP_CHECKOUT_COMMIT" \ + --campaign-tag "$CAMPAIGN_TAG" \ + --campaign-layout-sha256 "$CAMPAIGN_LAYOUT" \ + --formula "$FORMULA" --arch "$ARCH" \ + --build-handoff "$RUNNER_TEMP/homebrew-build-handoff" \ + --oci-child "$RUNNER_TEMP/homebrew-oci-child" \ + --package-input \ + "$RUNNER_TEMP/homebrew-candidate-promotion/candidate-package-input.json" + - name: Validate build data before exposing upload credentials id: validate-build - if: ${{ steps.build-handoff.outcome == 'success' && steps.oci-child.outcome == 'success' }} + if: >- + ${{ + steps.build-handoff.outcome == 'success' && + steps.oci-child.outcome == 'success' && + ( + needs.plan.outputs.candidate-promotion-mode != 'true' || + steps.validate-candidate-promotion.outcome == 'success' + ) + }} shell: bash env: KANDELO_HOMEBREW_ARCH: ${{ matrix.arch }} KANDELO_HOMEBREW_BOTTLE_ROOT_URL: ${{ needs.plan.outputs.bottle-root-prefix }} KANDELO_HOMEBREW_FORMULA: ${{ matrix.formula }} - KANDELO_HOMEBREW_KANDELO_COMMIT: ${{ needs.plan.outputs.kandelo-sha }} + KANDELO_HOMEBREW_KANDELO_COMMIT: >- + ${{ needs.plan.outputs.bottle-producer-sha }} KANDELO_HOMEBREW_RELEASE_TAG: ${{ needs.plan.outputs.release-tag }} KANDELO_HOMEBREW_TAP_COMMIT: ${{ needs.plan.outputs.tap-sha }} KANDELO_HOMEBREW_TAP_REPOSITORY: ${{ inputs.tap-repository }} @@ -2266,7 +2747,8 @@ jobs: KANDELO_HOMEBREW_ARCH: ${{ matrix.arch }} KANDELO_HOMEBREW_BOTTLE_ROOT_URL: ${{ needs.plan.outputs.bottle-root-prefix }} KANDELO_HOMEBREW_FORMULA: ${{ matrix.formula }} - KANDELO_HOMEBREW_KANDELO_COMMIT: ${{ needs.plan.outputs.kandelo-sha }} + KANDELO_HOMEBREW_KANDELO_COMMIT: >- + ${{ needs.plan.outputs.bottle-producer-sha }} KANDELO_HOMEBREW_RELEASE_TAG: ${{ needs.plan.outputs.release-tag }} KANDELO_HOMEBREW_TAP_COMMIT: ${{ needs.plan.outputs.tap-sha }} KANDELO_HOMEBREW_TAP_REPOSITORY: ${{ inputs.tap-repository }} @@ -2309,7 +2791,18 @@ jobs: retention-days: 2 - name: Fail when the matching immutable handoff is absent - if: ${{ always() && (steps.build-handoff.outcome != 'success' || steps.oci-child.outcome != 'success') }} + if: >- + ${{ + always() && + ( + steps.build-handoff.outcome != 'success' || + steps.oci-child.outcome != 'success' || + ( + needs.plan.outputs.candidate-promotion-mode == 'true' && + steps.validate-candidate-promotion.outcome != 'success' + ) + ) + }} shell: bash run: | echo "::error::the matching build or OCI child handoff was not produced" @@ -2345,6 +2838,16 @@ jobs: path: kandelo submodules: false + - name: Checkout promoted bottle producer for index publication + if: ${{ needs.plan.outputs.candidate-promotion-mode == 'true' }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ needs.plan.outputs.bottle-producer-sha }} + path: bottle-producer + submodules: false + - name: Checkout exact tap source for index validation uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -2372,15 +2875,26 @@ jobs: CAMPAIGN_TAG: ${{ needs.plan.outputs.prefix-campaign-tag }} FORMULA: ${{ matrix.formula }} KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + BOTTLE_PRODUCER_SHA: ${{ needs.plan.outputs.bottle-producer-sha }} + CANDIDATE_PROMOTION_MODE: >- + ${{ needs.plan.outputs.candidate-promotion-mode }} TAP_NAME: ${{ inputs.tap-name }} TAP_REPOSITORY: ${{ inputs.tap-repository }} TAP_SHA: ${{ needs.plan.outputs.tap-sha }} run: | set -euo pipefail + producer_args=() + campaign_commit="$KANDELO_SHA" + if [ "$CANDIDATE_PROMOTION_MODE" = true ]; then + producer_args+=(--kandelo-root \ + "$GITHUB_WORKSPACE/bottle-producer") + campaign_commit="$BOTTLE_PRODUCER_SHA" + fi python3 kandelo/scripts/homebrew-prefix-campaign-publisher.py \ prepare \ --tap-root "$GITHUB_WORKSPACE/tap" \ - --kandelo-commit "$KANDELO_SHA" \ + "${producer_args[@]}" \ + --kandelo-commit "$campaign_commit" \ --tap-repository "$TAP_REPOSITORY" \ --tap-name "$TAP_NAME" \ --source-tap-commit "$TAP_SHA" \ @@ -2432,6 +2946,42 @@ jobs: cd kandelo bash scripts/dev-shell.sh true + - name: Download exact candidate campaign admission for index + if: ${{ needs.plan.outputs.candidate-promotion-mode == 'true' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: >- + homebrew-candidate-promotion-${{ matrix.formula }}-wasm32-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/homebrew-candidate-index-admission + + - name: Bind index write to the admitted candidate campaign + if: ${{ needs.plan.outputs.candidate-promotion-mode == 'true' }} + shell: bash + env: + ABI: ${{ needs.plan.outputs.abi }} + CAMPAIGN_LAYOUT: >- + ${{ needs.plan.outputs.prefix-campaign-layout-sha256 }} + CAMPAIGN_TAG: ${{ needs.plan.outputs.prefix-campaign-tag }} + MERGE_COMMIT: ${{ needs.plan.outputs.kandelo-sha }} + PRODUCER_COMMIT: ${{ needs.plan.outputs.bottle-producer-sha }} + TAP_COMMIT: ${{ needs.plan.outputs.tap-sha }} + run: | + set -euo pipefail + [ -z "${GH_TOKEN:-}" ] && [ -z "${GITHUB_TOKEN:-}" ] || { + echo "::error::candidate index admission received credentials" + exit 2 + } + python3 kandelo/scripts/homebrew-candidate-campaign.py \ + validate-admission \ + --receipt \ + "$RUNNER_TEMP/homebrew-candidate-index-admission/candidate-campaign-admission.json" \ + --candidate-tag "$CAMPAIGN_TAG" \ + --producer-commit "$PRODUCER_COMMIT" \ + --merge-commit "$MERGE_COMMIT" \ + --source-tap-commit "$TAP_COMMIT" \ + --abi "$ABI" \ + --guest-layout-sha256 "$CAMPAIGN_LAYOUT" + - name: Download immutable OCI child layouts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -2691,6 +3241,16 @@ jobs: path: kandelo submodules: false + - name: Checkout promoted bottle producer for verification + if: ${{ needs.plan.outputs.candidate-promotion-mode == 'true' }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ needs.plan.outputs.bottle-producer-sha }} + path: bottle-producer + submodules: false + - name: Checkout exact Kandelo sysroot build source uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -2700,6 +3260,16 @@ jobs: path: kandelo-sysroot-build submodules: false + - name: Checkout protected candidate verifier package-reader authority + if: ${{ needs.plan.outputs.candidate-mode == 'true' }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ needs.plan.outputs.candidate-workflow-authority-sha }} + path: candidate-authority + submodules: false + - name: Checkout exact tap source uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -2723,6 +3293,9 @@ jobs: shell: bash env: KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + BOTTLE_PRODUCER_SHA: ${{ needs.plan.outputs.bottle-producer-sha }} + CANDIDATE_PROMOTION_MODE: >- + ${{ needs.plan.outputs.candidate-promotion-mode }} TAP_SHA: ${{ needs.plan.outputs.tap-sha }} run: | set -euo pipefail @@ -2767,7 +3340,11 @@ jobs: run: | set -euo pipefail cd kandelo - host="$(bash scripts/dev-shell.sh rustc -vV | sed -n 's/^host: //p')" + host="$(env -u GH_TOKEN -u GITHUB_TOKEN \ + -u HOMEBREW_GITHUB_API_TOKEN \ + -u HOMEBREW_GITHUB_PACKAGES_TOKEN \ + -u HOMEBREW_DOCKER_REGISTRY_TOKEN \ + bash scripts/dev-shell.sh rustc -vV | sed -n 's/^host: //p')" [ -n "$host" ] || { echo "::error::unable to resolve the Rust host target"; exit 2; } @@ -2795,6 +3372,54 @@ jobs: } echo "WASM_POSIX_BINARY_INDEX_URL=$index_url" >> "$GITHUB_ENV" + - name: Re-materialize immutable candidate verification packages + if: ${{ needs.plan.outputs.candidate-mode == 'true' }} + shell: bash + env: + CANDIDATE_PR_NUMBER: ${{ needs.plan.outputs.candidate-pr-number }} + CANDIDATE_STAGING_TAG: ${{ needs.plan.outputs.candidate-package-staging-tag }} + KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + EXPECTED_ABI: ${{ needs.plan.outputs.abi }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + cd candidate-authority + [[ "$CANDIDATE_STAGING_TAG" =~ ^pr-[1-9][0-9]*-staging-run-([1-9][0-9]*)-attempt-([1-9][0-9]*)$ ]] + staging_run="${BASH_REMATCH[1]}" + staging_attempt="${BASH_REMATCH[2]}" + host="$(env -u GH_TOKEN -u GITHUB_TOKEN \ + bash scripts/dev-shell.sh rustc -vV | sed -n 's/^host: //p')" + [ -n "$host" ] || { + echo "::error::unable to resolve the Rust host target"; exit 2; + } + env -u GH_TOKEN -u GITHUB_TOKEN \ + bash scripts/dev-shell.sh bash \ + .github/scripts/prepare-homebrew-package-materializer.sh \ + --host-target "$host" + authority_xtask="$PWD/target/$host/release/xtask" + cd ../kandelo + output="$RUNNER_TEMP/homebrew-candidate-verifier-packages" + # The protected wrapper and xtask treat this checkout only as data. + bash ../candidate-authority/.github/scripts/materialize-homebrew-candidate-package-input.sh \ + --tag "$CANDIDATE_STAGING_TAG" \ + --pr-number "$CANDIDATE_PR_NUMBER" \ + --run-id "$staging_run" \ + --run-attempt "$staging_attempt" \ + --producer-sha "$KANDELO_SHA" \ + --expected-abi "$EXPECTED_ABI" \ + --exclude erlang-vfs,perl,perl-vfs,python-vfs,redis,texlive \ + --consumer-root "$PWD" \ + --consumer-sha "$KANDELO_SHA" \ + --xtask "$authority_xtask" \ + --output-dir "$output" + index_url="$(cat "$output/resolver/index-url.txt")" + [ "$index_url" = \ + "file://$output/resolver/archives/index.toml" ] || { + echo "::error::candidate verifier resolver is not frozen" + exit 2 + } + echo "WASM_POSIX_BINARY_INDEX_URL=$index_url" >>"$GITHUB_ENV" + - name: Prepare sealed campaign dependencies for verification if: ${{ needs.plan.outputs.prefix-campaign-mode == 'true' }} shell: bash @@ -2804,16 +3429,27 @@ jobs: CAMPAIGN_TAG: ${{ needs.plan.outputs.prefix-campaign-tag }} FORMULA: ${{ matrix.formula }} KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + BOTTLE_PRODUCER_SHA: ${{ needs.plan.outputs.bottle-producer-sha }} + CANDIDATE_PROMOTION_MODE: >- + ${{ needs.plan.outputs.candidate-promotion-mode }} TAP_NAME: ${{ inputs.tap-name }} TAP_REPOSITORY: ${{ inputs.tap-repository }} TAP_SHA: ${{ needs.plan.outputs.tap-sha }} run: | set -euo pipefail + producer_args=() + campaign_commit="$KANDELO_SHA" + if [ "$CANDIDATE_PROMOTION_MODE" = true ]; then + producer_args+=(--kandelo-root \ + "$GITHUB_WORKSPACE/bottle-producer") + campaign_commit="$BOTTLE_PRODUCER_SHA" + fi bash kandelo/scripts/dev-shell.sh \ python3 kandelo/scripts/homebrew-prefix-campaign-publisher.py \ prepare \ --tap-root "$GITHUB_WORKSPACE/tap" \ - --kandelo-commit "$KANDELO_SHA" \ + "${producer_args[@]}" \ + --kandelo-commit "$campaign_commit" \ --tap-repository "$TAP_REPOSITORY" \ --tap-name "$TAP_NAME" \ --source-tap-commit "$TAP_SHA" \ @@ -2976,7 +3612,8 @@ jobs: KANDELO_HOMEBREW_ARCH: ${{ matrix.arch }} KANDELO_HOMEBREW_BOTTLE_ROOT_URL: ${{ needs.plan.outputs.bottle-root-prefix }} KANDELO_HOMEBREW_FORMULA: ${{ matrix.formula }} - KANDELO_HOMEBREW_KANDELO_COMMIT: ${{ needs.plan.outputs.kandelo-sha }} + KANDELO_HOMEBREW_KANDELO_COMMIT: >- + ${{ needs.plan.outputs.bottle-producer-sha }} KANDELO_HOMEBREW_RELEASE_TAG: ${{ needs.plan.outputs.release-tag }} KANDELO_HOMEBREW_TAP_COMMIT: ${{ needs.plan.outputs.tap-sha }} KANDELO_HOMEBREW_TAP_REPOSITORY: ${{ inputs.tap-repository }} @@ -3045,7 +3682,8 @@ jobs: KANDELO_HOMEBREW_DRY_RUN: ${{ inputs.dry-run }} KANDELO_HOMEBREW_BOTTLE_ROOT_URL: ${{ needs.plan.outputs.bottle-root-prefix }} KANDELO_HOMEBREW_FORMULA: ${{ matrix.formula }} - KANDELO_HOMEBREW_KANDELO_COMMIT: ${{ needs.plan.outputs.kandelo-sha }} + KANDELO_HOMEBREW_KANDELO_COMMIT: >- + ${{ needs.plan.outputs.bottle-producer-sha }} KANDELO_HOMEBREW_RELEASE_TAG: ${{ needs.plan.outputs.release-tag }} KANDELO_HOMEBREW_TAP_COMMIT: ${{ needs.plan.outputs.tap-sha }} KANDELO_HOMEBREW_TAP_REPOSITORY: ${{ inputs.tap-repository }} @@ -3810,17 +4448,28 @@ jobs: CAMPAIGN_TAG: ${{ needs.plan.outputs.prefix-campaign-tag }} FORMULA: ${{ matrix.formula }} KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + BOTTLE_PRODUCER_SHA: ${{ needs.plan.outputs.bottle-producer-sha }} + CANDIDATE_PROMOTION_MODE: >- + ${{ needs.plan.outputs.candidate-promotion-mode }} TAP_NAME: ${{ inputs.tap-name }} TAP_REPOSITORY: ${{ inputs.tap-repository }} TAP_SHA: ${{ needs.plan.outputs.tap-sha }} run: | set -euo pipefail + producer_args=() + campaign_commit="$KANDELO_SHA" + if [ "$CANDIDATE_PROMOTION_MODE" = true ]; then + producer_args+=(--kandelo-root \ + "$GITHUB_WORKSPACE/bottle-producer") + campaign_commit="$BOTTLE_PRODUCER_SHA" + fi bash kandelo-postverify/scripts/dev-shell.sh \ python3 \ kandelo-postverify/scripts/homebrew-prefix-campaign-publisher.py \ prepare \ --tap-root "$GITHUB_WORKSPACE/tap-postverify" \ - --kandelo-commit "$KANDELO_SHA" \ + "${producer_args[@]}" \ + --kandelo-commit "$campaign_commit" \ --tap-repository "$TAP_REPOSITORY" \ --tap-name "$TAP_NAME" \ --source-tap-commit "$TAP_SHA" \ @@ -4586,7 +5235,8 @@ jobs: KANDELO_HOMEBREW_BOTTLE_ROOT_URL: ${{ needs.plan.outputs.bottle-root-prefix }} KANDELO_HOMEBREW_DRY_RUN: ${{ inputs.dry-run }} KANDELO_HOMEBREW_FORMULA: ${{ matrix.formula }} - KANDELO_HOMEBREW_KANDELO_COMMIT: ${{ needs.plan.outputs.kandelo-sha }} + KANDELO_HOMEBREW_KANDELO_COMMIT: >- + ${{ needs.plan.outputs.bottle-producer-sha }} KANDELO_HOMEBREW_RELEASE_TAG: ${{ needs.plan.outputs.release-tag }} KANDELO_HOMEBREW_TAP_COMMIT: ${{ needs.plan.outputs.tap-sha }} KANDELO_HOMEBREW_TAP_REPOSITORY: ${{ inputs.tap-repository }} @@ -4967,6 +5617,502 @@ jobs: shell: bash run: exit 1 + seal-bottle-candidate: + needs: [plan, build-and-test, verify-bottle] + if: ${{ always() && !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.candidate-mode == 'true' && needs.build-and-test.result == 'success' && needs.verify-bottle.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 90 + # Candidate Formula code has already stopped running. This job checks out + # reviewed authority separately and treats all downloaded files as data. + permissions: + actions: read + contents: write + steps: + - name: Checkout protected candidate validator authority + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ needs.plan.outputs.candidate-workflow-authority-sha }} + path: authority + submodules: false + + - name: Checkout inert candidate producer + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ needs.plan.outputs.kandelo-sha }} + path: producer + fetch-depth: 0 + submodules: false + + - name: Checkout exact candidate tap source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ${{ inputs.tap-repository }} + ref: ${{ needs.plan.outputs.tap-sha }} + path: tap-source + fetch-depth: 0 + + - name: Checkout tap workflow authority + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + repository: ${{ inputs.tap-repository }} + ref: ${{ needs.plan.outputs.candidate-tap-workflow-authority-sha }} + path: tap-authority + fetch-depth: 0 + + - name: Download candidate build handoff + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: homebrew-build-handoff-${{ inputs.formulae }}-${{ inputs.arches }}-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/candidate-build + + - name: Download candidate OCI child + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: homebrew-oci-child-${{ inputs.formulae }}-${{ inputs.arches }}-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/candidate-oci + + - name: Download candidate package-input identity + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: homebrew-candidate-package-input-${{ inputs.formulae }}-${{ inputs.arches }}-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/candidate-packages + + - name: Install Nix for reviewed data validation + uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 + with: + github-token: "" + + - name: Cache Nix store + flake eval + uses: DeterminateSystems/magic-nix-cache-action@908b263ff629f4cc17666315b7fd3ec127c6244d # v14 + with: + use-gha-cache: false + use-flakehub: false + + - name: Recreate collision-planned candidate tap source + shell: bash + env: + ARCH: ${{ inputs.arches }} + CAMPAIGN_DEPENDENCIES: ${{ needs.plan.outputs.prefix-campaign-dependencies }} + CAMPAIGN_TAG: ${{ needs.plan.outputs.prefix-campaign-tag }} + FORMULA: ${{ inputs.formulae }} + KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + TAP_NAME: ${{ inputs.tap-name }} + TAP_REPOSITORY: ${{ inputs.tap-repository }} + TAP_SHA: ${{ needs.plan.outputs.tap-sha }} + run: | + set -euo pipefail + [ "$CAMPAIGN_DEPENDENCIES" = \ + '{"dependencies":[],"schema":1}' ] || { + echo "::error::candidate bottle v1 is restricted to leaf Formulae" + exit 2 + } + python3 authority/scripts/homebrew-prefix-campaign-publisher.py \ + prepare \ + --tap-root "$GITHUB_WORKSPACE/tap-source" \ + --kandelo-root "$GITHUB_WORKSPACE/producer" \ + --kandelo-commit "$KANDELO_SHA" \ + --tap-repository "$TAP_REPOSITORY" \ + --tap-name "$TAP_NAME" \ + --source-tap-commit "$TAP_SHA" \ + --campaign-tag "$CAMPAIGN_TAG" \ + --dependencies "$CAMPAIGN_DEPENDENCIES" \ + --formula "$FORMULA" \ + --arch "$ARCH" \ + --work-root "$RUNNER_TEMP/candidate-campaign" \ + --receipt-out "$RUNNER_TEMP/candidate-campaign.json" \ + --github-env "$GITHUB_ENV" + + - name: Capture immutable candidate evidence + id: evidence + shell: bash + env: + ABI: ${{ needs.plan.outputs.abi }} + ARCH: ${{ inputs.arches }} + AUTHORITY_SHA: ${{ needs.plan.outputs.candidate-workflow-authority-sha }} + BOTTLE_ROOT_URL: ${{ needs.plan.outputs.bottle-root-prefix }} + CAMPAIGN_LAYOUT_SHA256: ${{ needs.plan.outputs.prefix-campaign-layout-sha256 }} + CAMPAIGN_TAG: ${{ needs.plan.outputs.prefix-campaign-tag }} + FORMULA: ${{ inputs.formulae }} + GH_TOKEN: ${{ github.token }} + KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + PR_NUMBER: ${{ needs.plan.outputs.candidate-pr-number }} + RELEASE_TAG: ${{ needs.plan.outputs.release-tag }} + TAP_NAME: ${{ inputs.tap-name }} + TAP_REPOSITORY: ${{ inputs.tap-repository }} + TAP_SHA: ${{ needs.plan.outputs.tap-sha }} + TAP_AUTHORITY_SHA: ${{ needs.plan.outputs.candidate-tap-workflow-authority-sha }} + run: | + set -euo pipefail + for pair in \ + "authority:$AUTHORITY_SHA" \ + "producer:$KANDELO_SHA" \ + "tap-source:$KANDELO_HOMEBREW_PREPARED_TAP_COMMIT" \ + "tap-authority:$TAP_AUTHORITY_SHA"; do + root="${pair%%:*}" + sha="${pair#*:}" + if [ "$(git -C "$root" rev-parse HEAD)" != "$sha" ] || + [ -n "$(git -C "$root" status \ + --porcelain=v1 --untracked-files=all)" ]; then + echo "::error::$root is not its exact clean source" + exit 2 + fi + done + if [ "$(git -C tap-source rev-parse 'HEAD^{tree}')" != \ + "$KANDELO_HOMEBREW_PREPARED_TAP_TREE" ] || + ! git -C tap-source merge-base --is-ancestor \ + "$TAP_SHA" "$KANDELO_HOMEBREW_PREPARED_TAP_COMMIT"; then + echo "::error::candidate prepared tap is not derived from the admitted source" + exit 2 + fi + + pr="$RUNNER_TEMP/candidate-pr-rest.json" + gh api "/repos/Automattic/kandelo/pulls/$PR_NUMBER" >"$pr" + base_sha="$(jq -er '.base.sha' "$pr")" + [ "$(jq -er '.base.ref' "$pr")" = main ] && + [ "$(jq -er '.head.sha' "$pr")" = "$KANDELO_SHA" ] && + [ "$(jq -er '.state' "$pr")" = open ] || { + echo "::error::candidate PR no longer names the exact open head"; exit 1; + } + current_main="$(gh api /repos/Automattic/kandelo/git/ref/heads/main --jq .object.sha)" + [ "$base_sha" = "$current_main" ] || { + echo "::error::candidate PR base moved; rebuild from the current base"; exit 1; + } + authority_status="$(gh api "/repos/Automattic/kandelo/compare/$AUTHORITY_SHA...main" --jq .status)" + case "$authority_status" in ahead|identical) ;; *) exit 1 ;; esac + tap_source_status="$(gh api "/repos/$TAP_REPOSITORY/compare/$TAP_SHA...$TAP_AUTHORITY_SHA" --jq .status)" + case "$tap_source_status" in ahead|identical) ;; *) exit 1 ;; esac + tap_authority_status="$(gh api "/repos/$TAP_REPOSITORY/compare/$TAP_AUTHORITY_SHA...main" --jq .status)" + case "$tap_authority_status" in ahead|identical) ;; *) exit 1 ;; esac + + run_json="$RUNNER_TEMP/candidate-run-live.json" + artifacts_pages="$RUNNER_TEMP/candidate-artifact-pages.json" + artifacts_json="$RUNNER_TEMP/candidate-artifacts.json" + gh api "/repos/$TAP_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >"$run_json" + gh api --paginate --slurp \ + "/repos/$TAP_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100" \ + >"$artifacts_pages" + jq -e '[.[].artifacts[]]' "$artifacts_pages" >"$artifacts_json" + expected_names="$(jq -nc \ + --arg build "homebrew-build-handoff-$FORMULA-$ARCH-attempt-$GITHUB_RUN_ATTEMPT" \ + --arg oci "homebrew-oci-child-$FORMULA-$ARCH-attempt-$GITHUB_RUN_ATTEMPT" \ + --arg packages "homebrew-candidate-package-input-$FORMULA-$ARCH-attempt-$GITHUB_RUN_ATTEMPT" \ + '[$build,$oci,$packages] | sort')" + jq -e --argjson names "$expected_names" ' + map(select(.name as $name | $names | index($name))) as $selected | + ($selected | length) == 3 and + ([$selected[].name] | sort) == $names and + ([$selected[].id] | length == (unique | length)) and + all($selected[]; + .expired == false and + (.id | type == "number" and . > 0) and + (.size_in_bytes | type == "number" and . > 0) and + (.digest | type == "string" and + test("^sha256:[0-9a-f]{64}$"))) + ' "$artifacts_json" >/dev/null || { + echo "::error::candidate run lacks one exact retained artifact set" + exit 1 + } + jq -nS \ + --arg repository "$TAP_REPOSITORY" \ + --arg caller "$TAP_AUTHORITY_SHA" \ + --argjson run_id "$GITHUB_RUN_ID" \ + --argjson attempt "$GITHUB_RUN_ATTEMPT" \ + --slurpfile run "$run_json" \ + --slurpfile artifacts "$artifacts_json" \ + --argjson names "$expected_names" ' + { + schema:1, + repository:$repository, + workflow_path:".github/workflows/candidate-bottles.yml", + caller_commit:$caller, + event:"repository_dispatch", + run_id:$run_id, + run_attempt:$attempt, + status:$run[0].status, + conclusion:$run[0].conclusion, + artifacts:($artifacts[0] | + map(select(.name as $name | $names | index($name))) | + map({ + id, + name, + bytes:.size_in_bytes, + digest, + run_id:$run_id, + run_attempt:$attempt + }) | sort_by(.name)) + } + ' >"$RUNNER_TEMP/candidate-run.json" + + env -u GH_TOKEN -u GITHUB_TOKEN \ + python3 authority/scripts/homebrew-bottle-candidate.py \ + describe-source --root "$GITHUB_WORKSPACE/producer" \ + --producer-commit "$KANDELO_SHA" \ + --out "$RUNNER_TEMP/candidate-source-description.json" + producer_tree="$(jq -er '.producer_tree' \ + "$RUNNER_TEMP/candidate-source-description.json")" + snapshot_sha="$(jq -er '.abi_snapshot_sha256' \ + "$RUNNER_TEMP/candidate-source-description.json")" + layout_sha="$(jq -er '.guest_layout_sha256' \ + "$RUNNER_TEMP/candidate-source-description.json")" + jq -nS \ + --arg base "$base_sha" \ + --arg producer "$KANDELO_SHA" \ + --arg producer_tree "$producer_tree" \ + --arg authority "$AUTHORITY_SHA" \ + --arg snapshot "$snapshot_sha" \ + --arg layout "$layout_sha" \ + --arg release "$RELEASE_TAG" \ + --arg tap_repository "$TAP_REPOSITORY" \ + --arg tap_name "$TAP_NAME" \ + --arg tap "$TAP_SHA" \ + --arg tap_checkout "$KANDELO_HOMEBREW_PREPARED_TAP_COMMIT" \ + --arg tap_checkout_tree "$KANDELO_HOMEBREW_PREPARED_TAP_TREE" \ + --arg campaign "$CAMPAIGN_TAG" \ + --arg campaign_layout "$CAMPAIGN_LAYOUT_SHA256" \ + --argjson pr "$PR_NUMBER" \ + --argjson abi "$ABI" ' + { + kandelo_repository:"Automattic/kandelo", + workflow_authority_commit:$authority, + base_commit:$base, + producer_commit:$producer, + producer_tree:$producer_tree, + merge_method:"merge", + pr_number:$pr, + abi:$abi, + abi_snapshot_sha256:$snapshot, + guest_layout:{ + path:"homebrew/kandelo-guest-layout.json", + sha256:$layout + }, + release_tag:$release, + tap_repository:$tap_repository, + tap_name:$tap_name, + tap_commit:$tap, + tap_checkout_commit:$tap_checkout, + tap_checkout_tree:$tap_checkout_tree, + prefix_campaign_tag:$campaign, + prefix_campaign_layout_sha256:$campaign_layout + } + ' >"$RUNNER_TEMP/candidate-source.json" + printf '[]\n' >"$RUNNER_TEMP/candidate-dependencies.json" + echo "base-sha=$base_sha" >>"$GITHUB_OUTPUT" + + - name: Validate candidate handoffs with reviewed code + shell: bash + env: + ARCH: ${{ inputs.arches }} + BOTTLE_ROOT_URL: ${{ needs.plan.outputs.bottle-root-prefix }} + FORMULA: ${{ inputs.formulae }} + KANDELO_SHA: ${{ needs.plan.outputs.kandelo-sha }} + RELEASE_TAG: ${{ needs.plan.outputs.release-tag }} + TAP_NAME: ${{ inputs.tap-name }} + TAP_REPOSITORY: ${{ inputs.tap-repository }} + TAP_SHA: ${{ needs.plan.outputs.tap-sha }} + run: | + set -euo pipefail + for secret_name in GH_TOKEN GITHUB_TOKEN \ + HOMEBREW_GITHUB_API_TOKEN HOMEBREW_GITHUB_PACKAGES_TOKEN \ + HOMEBREW_DOCKER_REGISTRY_TOKEN; do + [ -z "${!secret_name:-}" ] || { + echo "::error::candidate data validator received $secret_name" + exit 2 + } + done + python3 authority/scripts/homebrew-oci-layout.py validate-child \ + --layout "$RUNNER_TEMP/candidate-oci/layout" \ + --receipt "$RUNNER_TEMP/candidate-oci/receipt.json" + resolved="$RUNNER_TEMP/candidate-resolved-taps.json" + python3 authority/scripts/homebrew-dependency-taps.py resolve \ + --tap-root "$GITHUB_WORKSPACE/tap-source" \ + --tap-name "$TAP_NAME" \ + --tap-repository "$TAP_REPOSITORY" \ + --tap-commit "$TAP_SHA" \ + --checkout-commit "$KANDELO_HOMEBREW_PREPARED_TAP_COMMIT" \ + --out "$resolved" + mkdir "$RUNNER_TEMP/candidate-validated" + bash authority/scripts/dev-shell.sh env \ + KANDELO_HOMEBREW_RESOLVED_TAPS_FILE="$resolved" \ + bash authority/scripts/homebrew-validate-build-handoff.sh \ + --handoff "$RUNNER_TEMP/candidate-build" \ + --formula "$FORMULA" \ + --arch "$ARCH" \ + --release-tag "$RELEASE_TAG" \ + --tap-repository "$TAP_REPOSITORY" \ + --tap-name "$TAP_NAME" \ + --tap-commit "$TAP_SHA" \ + --tap-checkout-commit \ + "$KANDELO_HOMEBREW_PREPARED_TAP_COMMIT" \ + --kandelo-commit "$KANDELO_SHA" \ + --bottle-root-url "$BOTTLE_ROOT_URL" \ + --tap-root "$GITHUB_WORKSPACE/tap-source" \ + --forbidden-root "$GITHUB_WORKSPACE" \ + --forbidden-root "$(dirname "$GITHUB_WORKSPACE")" \ + --forbidden-root "$RUNNER_TEMP" \ + --out-env "$RUNNER_TEMP/candidate-validated/build.env" \ + --out-bottle-json \ + "$RUNNER_TEMP/candidate-validated/bottle.json" + # WHY: v1 deliberately stages only leaf Formulae. The build's + # independently captured pour receipt is the authority for leafness; + # an empty caller-supplied dependency list is not evidence. + jq -e '.dependencies == []' \ + "$RUNNER_TEMP/candidate-build/dependency-provenance.json" \ + >/dev/null || { + echo "::error::candidate bottle v1 rejected a dependency-bearing Formula" + exit 2 + } + + - name: Prove candidate refs are collision-free + shell: bash + env: + FORMULA: ${{ inputs.formulae }} + TAP_SHA: ${{ needs.plan.outputs.tap-sha }} + TAP_REPOSITORY: ${{ inputs.tap-repository }} + run: | + set -euo pipefail + [ -z "${GH_TOKEN:-}" ] && [ -z "${GITHUB_TOKEN:-}" ] || { + echo "::error::candidate destination probe received credentials" + exit 2 + } + receipt="$RUNNER_TEMP/candidate-oci/receipt.json" + remote="ghcr.io/$(printf '%s' "$TAP_REPOSITORY" | tr '[:upper:]' '[:lower:]')/$FORMULA" + child="$(jq -er '.oci.transport_tag' "$receipt")" + homebrew_ref="$(jq -er '.oci.homebrew_ref' "$receipt")" + expected_child_digest="$(jq -er '.oci.manifest.digest' "$receipt")" + top="$(jq -er '.top_ref' "$receipt")" + config="$RUNNER_TEMP/candidate-anonymous-oras.json" + printf '{"auths":{}}\n' >"$config" + child_result="$RUNNER_TEMP/candidate-child-probe.json" + bash authority/scripts/dev-shell.sh env -u GH_TOKEN -u GITHUB_TOKEN \ + python3 authority/scripts/homebrew-oci-layout.py probe-registry \ + --kind manifest --remote "$remote" --reference "$child" \ + --registry-config "$config" --out-result "$child_result" + jq -e --arg digest "$expected_child_digest" ' + (.status == "missing" and .digest == null) or + (.status == "present" and .digest == $digest) + ' "$child_result" >/dev/null || { + echo "::error::candidate transport ref contains different bytes" + exit 1 + } + + existing="$RUNNER_TEMP/candidate-existing-index" + top_result="$RUNNER_TEMP/candidate-top-probe.json" + bash authority/scripts/dev-shell.sh env -u GH_TOKEN -u GITHUB_TOKEN \ + python3 authority/scripts/homebrew-oci-layout.py \ + import-public-index --remote "$remote" --reference "$top" \ + --registry-config "$config" --out-layout "$existing" \ + --out-result "$top_result" + merge_args=( + --child-layout "$RUNNER_TEMP/candidate-oci/layout" + --child-receipt "$receipt" + ) + case "$(jq -er .status "$top_result")" in + present) merge_args+=(--existing-layout "$existing") ;; + missing) ;; + *) echo "::error::candidate top index is ambiguous"; exit 1 ;; + esac + # WHY: the immutable child uses a content-addressed transport tag, + # but Homebrew selects through homebrew_ref in this top index. This + # dry merge rejects an older ABI at the same version/rebuild ref. + bash authority/scripts/dev-shell.sh env -u GH_TOKEN -u GITHUB_TOKEN \ + python3 authority/scripts/homebrew-oci-layout.py merge-index \ + "${merge_args[@]}" --tap-commit "$TAP_SHA" \ + --out-layout "$RUNNER_TEMP/candidate-merged-index" \ + --out-receipt "$RUNNER_TEMP/candidate-merged-index.json" + + child_status="$(jq -r .status "$child_result")" + child_digest="$(jq -c '.digest' "$child_result")" + top_status="$(jq -r .status "$top_result")" + top_digest="$(jq -c '.digest // null' "$top_result")" + jq -nS \ + --arg formula "$FORMULA" \ + --arg remote "$remote" \ + --arg child "$child" \ + --arg child_status "$child_status" \ + --argjson child_digest "$child_digest" \ + --arg homebrew_ref "$homebrew_ref" \ + --arg top "$top" \ + --arg top_status "$top_status" \ + --argjson top_digest "$top_digest" \ + --arg observed "$(date -u +%Y-%m-%dT%H:%M:%SZ)" ' + { + formula:$formula, + remote:$remote, + child_ref:$child, + child_status:$child_status, + child_digest:$child_digest, + homebrew_ref:$homebrew_ref, + homebrew_ref_status:"available", + top_ref:$top, + top_status:$top_status, + top_digest:$top_digest, + observed_at:$observed + } + ' >"$RUNNER_TEMP/candidate-destination.json" + + - name: Prepare inert immutable candidate release + shell: bash + run: | + set -euo pipefail + [ -z "${GH_TOKEN:-}" ] && [ -z "${GITHUB_TOKEN:-}" ] || { + echo "::error::candidate release preparation received credentials" + exit 2 + } + python3 authority/scripts/homebrew-bottle-candidate.py prepare \ + --source "$RUNNER_TEMP/candidate-source.json" \ + --run-evidence "$RUNNER_TEMP/candidate-run.json" \ + --destination "$RUNNER_TEMP/candidate-destination.json" \ + --dependencies "$RUNNER_TEMP/candidate-dependencies.json" \ + --package-input \ + "$RUNNER_TEMP/candidate-packages/package-input.json" \ + --build-handoff "$RUNNER_TEMP/candidate-build" \ + --oci-child "$RUNNER_TEMP/candidate-oci" \ + --out "$RUNNER_TEMP/prepared-candidate" + + - name: Publish and anonymously read back candidate release + shell: bash + env: + AUTHORITY_SHA: ${{ needs.plan.outputs.candidate-workflow-authority-sha }} + GH_TOKEN: ${{ github.token }} + PRODUCER_SHA: ${{ needs.plan.outputs.kandelo-sha }} + STATE_LOCK_OWNER_DETAIL: immutable Homebrew bottle candidate + TAP_AUTHORITY_SHA: ${{ needs.plan.outputs.candidate-tap-workflow-authority-sha }} + run: | + set -euo pipefail + bash authority/scripts/publish-immutable-github-release.sh \ + --manifest \ + "$RUNNER_TEMP/prepared-candidate/release-manifest.json" \ + --asset-root "$RUNNER_TEMP/prepared-candidate/assets" \ + --lock-root "$GITHUB_WORKSPACE/tap-authority" \ + --receipt "$RUNNER_TEMP/candidate-release-receipt.json" \ + --kandelo-main-contains-sha "$AUTHORITY_SHA" \ + --target-main-contains-sha "$TAP_AUTHORITY_SHA" + tag="$(cat "$RUNNER_TEMP/prepared-candidate/tag.txt")" + echo "candidate-tag=$tag" >>"$GITHUB_OUTPUT" + { + echo "### Immutable Homebrew bottle candidate" + echo + echo "Candidate: \`$tag\`" + echo + echo "Producer: \`$PRODUCER_SHA\`" + echo + echo "This release is noncanonical until exact-head promotion." + } >>"$GITHUB_STEP_SUMMARY" + + - name: Retain candidate publication receipt + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: homebrew-candidate-release-receipt-${{ inputs.formulae }}-${{ inputs.arches }}-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/candidate-release-receipt.json + if-no-files-found: error + retention-days: 90 + publish-vfs-release: needs: [plan, verify-bottle, finalize-tap] if: ${{ always() && !cancelled() && !inputs.dry-run && !inputs.defer-tap-finalization && inputs.require-vfs-acceptance && needs.plan.result == 'success' && needs.verify-bottle.result == 'success' && needs.finalize-tap.result == 'success' && needs.plan.outputs.vfs-acceptance-formula != '' }} diff --git a/.github/workflows/reusable-homebrew-candidate-campaign.yml b/.github/workflows/reusable-homebrew-candidate-campaign.yml new file mode 100644 index 0000000000..be43c876ef --- /dev/null +++ b/.github/workflows/reusable-homebrew-candidate-campaign.yml @@ -0,0 +1,538 @@ +name: Seal an unmerged Homebrew campaign candidate + +on: + workflow_call: + inputs: + kandelo-repository: + type: string + required: true + producer-sha: + type: string + required: true + pr-number: + type: number + required: true + tap-repository: + type: string + required: true + tap-name: + type: string + required: true + tap-sha: + type: string + required: true + outputs: + candidate-tag: + value: ${{ jobs.seal.outputs.candidate-tag }} + +jobs: + admit: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + base-sha: ${{ steps.admit.outputs.base-sha }} + caller-sha: ${{ steps.admit.outputs.caller-sha }} + steps: + - name: Bind the request to current protected branches + id: admit + shell: bash + env: + CALLER_REF: ${{ github.ref }} + CALLER_REPOSITORY: ${{ github.repository }} + CALLER_SHA: ${{ github.sha }} + CALLER_WORKFLOW_REF: ${{ github.workflow_ref }} + GH_TOKEN: ${{ github.token }} + KANDELO_REPOSITORY: ${{ inputs.kandelo-repository }} + PR_NUMBER: ${{ inputs.pr-number }} + PRODUCER_SHA: ${{ inputs.producer-sha }} + TAP_NAME: ${{ inputs.tap-name }} + TAP_REPOSITORY: ${{ inputs.tap-repository }} + TAP_SHA: ${{ inputs.tap-sha }} + run: | + set -euo pipefail + expected_caller="$CALLER_REPOSITORY/.github/workflows/" + expected_caller+="candidate-campaign.yml@refs/heads/main" + [ "${CALLER_REPOSITORY,,}" = \ + "kandelo-dev/homebrew-tap-core" ] && + [ "$CALLER_REF" = refs/heads/main ] && + [ "$CALLER_WORKFLOW_REF" = "$expected_caller" ] && + [ "${KANDELO_REPOSITORY,,}" = automattic/kandelo ] && + [ "${TAP_REPOSITORY,,}" = \ + kandelo-dev/homebrew-tap-core ] && + [ "${TAP_NAME,,}" = kandelo-dev/tap-core ] && + [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] && + [[ "$PRODUCER_SHA" =~ ^[0-9a-f]{40}$ ]] && + [[ "$TAP_SHA" =~ ^[0-9a-f]{40}$ ]] || { + echo "::error::candidate campaign caller is not exact" + exit 2 + } + tap_main="$(gh api \ + "/repos/$TAP_REPOSITORY/git/ref/heads/main" \ + --jq .object.sha)" + [ "$CALLER_SHA" = "$tap_main" ] || { + echo "::error::candidate campaign requires current tap main" + exit 1 + } + tap_status="$(gh api \ + "/repos/$TAP_REPOSITORY/compare/$TAP_SHA...$tap_main" \ + --jq .status)" + case "$tap_status" in ahead|identical) ;; *) + echo "::error::candidate campaign tap source is not on main" + exit 1 + esac + pr="$RUNNER_TEMP/candidate-campaign-pr.json" + gh api "/repos/$KANDELO_REPOSITORY/pulls/$PR_NUMBER" >"$pr" + base_sha="$(jq -er .base.sha "$pr")" + current_main="$(gh api \ + "/repos/$KANDELO_REPOSITORY/git/ref/heads/main" \ + --jq .object.sha)" + jq -e \ + --arg base "$current_main" \ + --arg head "$PRODUCER_SHA" ' + .state == "open" and .base.ref == "main" and + .base.sha == $base and .head.sha == $head + ' "$pr" >/dev/null || { + echo "::error::candidate PR is not based on current main" + exit 1 + } + [ "$base_sha" = "$current_main" ] || exit 1 + { + echo "base-sha=$base_sha" + echo "caller-sha=$CALLER_SHA" + } >>"$GITHUB_OUTPUT" + + - name: Checkout protected campaign caller validator + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ steps.admit.outputs.base-sha }} + path: caller-validator + submodules: false + + - name: Checkout exact campaign caller as inert data + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: ${{ inputs.tap-repository }} + ref: ${{ steps.admit.outputs.caller-sha }} + path: caller-data + submodules: false + + - name: Require the campaign caller to pin its exact base + shell: bash + env: + BASE_SHA: ${{ steps.admit.outputs.base-sha }} + run: | + set -euo pipefail + # WHY: GitHub resolves a reusable workflow ref before Kandelo code + # starts. The caller at exact tap commit C must name B literally; + # a mutable @main ref could execute code that was never reviewed. + python3 caller-validator/scripts/homebrew-candidate-caller-pins.py \ + validate --tap-root "$GITHUB_WORKSPACE/caller-data" \ + --mode campaign --kandelo-sha "$BASE_SHA" + + derive: + needs: [admit] + runs-on: ubuntu-latest + timeout-minutes: 180 + permissions: + contents: read + steps: + - name: Checkout protected campaign wrapper authority + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ needs.admit.outputs.base-sha }} + path: authority + submodules: false + + - name: Checkout exact candidate as untrusted source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ inputs.producer-sha }} + path: producer + submodules: false + + - name: Checkout exact tap campaign source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: ${{ inputs.tap-repository }} + ref: ${{ inputs.tap-sha }} + path: tap + fetch-depth: 0 + + - name: Describe exact candidate inputs with protected code + id: source + shell: bash + env: + BASE_SHA: ${{ needs.admit.outputs.base-sha }} + CALLER_SHA: ${{ needs.admit.outputs.caller-sha }} + KANDELO_REPOSITORY: ${{ inputs.kandelo-repository }} + PR_NUMBER: ${{ inputs.pr-number }} + PRODUCER_SHA: ${{ inputs.producer-sha }} + TAP_NAME: ${{ inputs.tap-name }} + TAP_REPOSITORY: ${{ inputs.tap-repository }} + TAP_SHA: ${{ inputs.tap-sha }} + run: | + set -euo pipefail + for secret_name in GH_TOKEN GITHUB_TOKEN \ + HOMEBREW_GITHUB_API_TOKEN \ + HOMEBREW_GITHUB_PACKAGES_TOKEN \ + HOMEBREW_DOCKER_REGISTRY_TOKEN; do + [ -z "${!secret_name:-}" ] || { + echo "::error::candidate campaign source received $secret_name" + exit 2 + } + done + python3 authority/scripts/homebrew-candidate-campaign.py \ + describe-source \ + --kandelo-root "$GITHUB_WORKSPACE/producer" \ + --kandelo-repository "$KANDELO_REPOSITORY" \ + --base-commit "$BASE_SHA" \ + --producer-commit "$PRODUCER_SHA" \ + --workflow-authority-commit "$BASE_SHA" \ + --pr-number "$PR_NUMBER" \ + --tap-root "$GITHUB_WORKSPACE/tap" \ + --tap-repository "$TAP_REPOSITORY" \ + --tap-name "$TAP_NAME" \ + --source-tap-commit "$TAP_SHA" \ + --tap-workflow-authority-commit "$CALLER_SHA" \ + --out "$RUNNER_TEMP/candidate-campaign-source.json" + echo "native-commit=$(jq -er .native_homebrew_commit \ + "$RUNNER_TEMP/candidate-campaign-source.json")" \ + >>"$GITHUB_OUTPUT" + + - name: Checkout reviewed native Homebrew implementation + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: Homebrew/brew + ref: ${{ steps.source.outputs.native-commit }} + path: native-homebrew + + - name: Install Nix for credential-free campaign derivation + uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 + with: + github-token: "" + + - name: Cache Nix store and flake evaluation + uses: DeterminateSystems/magic-nix-cache-action@908b263ff629f4cc17666315b7fd3ec127c6244d # v14 + with: + use-gha-cache: false + use-flakehub: false + + - name: Derive the candidate campaign without credentials + shell: bash + env: + PRODUCER_SHA: ${{ inputs.producer-sha }} + TAP_SHA: ${{ inputs.tap-sha }} + run: | + set -euo pipefail + for secret_name in GH_TOKEN GITHUB_TOKEN \ + HOMEBREW_GITHUB_API_TOKEN \ + HOMEBREW_GITHUB_PACKAGES_TOKEN \ + HOMEBREW_DOCKER_REGISTRY_TOKEN; do + [ -z "${!secret_name:-}" ] || { + echo "::error::candidate campaign derivation received $secret_name" + exit 2 + } + done + source="$RUNNER_TEMP/candidate-campaign-source.json" + metadata_sha="$(jq -er .old_metadata.sha256 "$source")" + layout_sha="$(jq -er .guest_layout.sha256 "$source")" + native_sha="$(jq -er .native_homebrew_commit "$source")" + env -u GH_TOKEN -u GITHUB_TOKEN \ + -u HOMEBREW_GITHUB_API_TOKEN \ + -u HOMEBREW_GITHUB_PACKAGES_TOKEN \ + -u HOMEBREW_DOCKER_REGISTRY_TOKEN \ + bash producer/scripts/dev-shell.sh \ + python3 producer/scripts/homebrew-prefix-campaign.py derive \ + --kandelo-root "$GITHUB_WORKSPACE/producer" \ + --kandelo-commit "$PRODUCER_SHA" \ + --old-tap-root "$GITHUB_WORKSPACE/tap" \ + --old-tap-commit "$TAP_SHA" \ + --source-tap-root "$GITHUB_WORKSPACE/tap" \ + --source-tap-commit "$TAP_SHA" \ + --native-brew-root "$GITHUB_WORKSPACE/native-homebrew" \ + --native-brew-commit "$native_sha" \ + --metadata-sha256 "$metadata_sha" \ + --guest-layout-sha256 "$layout_sha" \ + --out "$RUNNER_TEMP/campaign.json" + + - name: Retain one exact credential-free derivation + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: >- + homebrew-candidate-campaign-derivation-attempt-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/candidate-campaign-source.json + ${{ runner.temp }}/campaign.json + compression-level: 0 + if-no-files-found: error + retention-days: 2 + + seal: + needs: [admit, derive] + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + actions: read + contents: write + outputs: + candidate-tag: ${{ steps.publish.outputs.candidate-tag }} + steps: + - name: Checkout protected Kandelo sealer authority + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ needs.admit.outputs.base-sha }} + path: authority + submodules: false + + - name: Checkout protected tap release authority + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + ref: ${{ needs.admit.outputs.caller-sha }} + fetch-depth: 0 + path: tap-authority + + - name: Checkout candidate only as inert source data + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: ${{ inputs.kandelo-repository }} + ref: ${{ inputs.producer-sha }} + path: producer-data + submodules: false + + - name: Checkout exact tap only as inert source data + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: ${{ inputs.tap-repository }} + ref: ${{ inputs.tap-sha }} + path: tap-data + fetch-depth: 0 + + - name: Download exact credential-free derivation + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: >- + homebrew-candidate-campaign-derivation-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/candidate-campaign-derivation + + - name: Revalidate run, source, and branch authority + shell: bash + env: + BASE_SHA: ${{ needs.admit.outputs.base-sha }} + CALLER_SHA: ${{ needs.admit.outputs.caller-sha }} + GH_TOKEN: ${{ github.token }} + KANDELO_REPOSITORY: ${{ inputs.kandelo-repository }} + PR_NUMBER: ${{ inputs.pr-number }} + PRODUCER_SHA: ${{ inputs.producer-sha }} + TAP_REPOSITORY: ${{ inputs.tap-repository }} + TAP_SHA: ${{ inputs.tap-sha }} + run: | + set -euo pipefail + kandelo_main="$(gh api \ + "/repos/$KANDELO_REPOSITORY/git/ref/heads/main" \ + --jq .object.sha)" + tap_main="$(gh api \ + "/repos/$TAP_REPOSITORY/git/ref/heads/main" \ + --jq .object.sha)" + [ "$kandelo_main" = "$BASE_SHA" ] || { + echo "::error::Kandelo main moved during campaign derivation" + exit 1 + } + # WHY: the exact caller commit owns this run. Later append-only tap + # commits do not change its code and should not discard completed + # derivation work. + caller_status="$(gh api \ + "/repos/$TAP_REPOSITORY/compare/$CALLER_SHA...$tap_main" \ + --jq .status)" + case "$caller_status" in + ahead|identical) ;; + *) + echo "::error::candidate caller left protected tap main" + exit 1 + ;; + esac + gh api "/repos/$KANDELO_REPOSITORY/pulls/$PR_NUMBER" \ + >"$RUNNER_TEMP/candidate-campaign-pr-live.json" + jq -e \ + --arg base "$BASE_SHA" \ + --arg head "$PRODUCER_SHA" ' + .state == "open" and .base.ref == "main" and + .base.sha == $base and .head.sha == $head + ' "$RUNNER_TEMP/candidate-campaign-pr-live.json" >/dev/null || { + echo "::error::candidate PR changed during campaign derivation" + exit 1 + } + source="$RUNNER_TEMP/candidate-campaign-derivation/" + source+="candidate-campaign-source.json" + jq -e \ + --arg base "$BASE_SHA" \ + --arg producer "$PRODUCER_SHA" \ + --arg caller "$CALLER_SHA" \ + --arg tap "$TAP_SHA" \ + --argjson pr "$PR_NUMBER" ' + .base_commit == $base and + .workflow_authority_commit == $base and + .producer_commit == $producer and + .tap_workflow_authority_commit == $caller and + .source_tap_commit == $tap and .pr_number == $pr + ' "$source" >/dev/null || { + echo "::error::candidate campaign source changed" + exit 1 + } + env -u GH_TOKEN -u GITHUB_TOKEN \ + python3 authority/scripts/homebrew-candidate-campaign.py \ + describe-source \ + --kandelo-root "$GITHUB_WORKSPACE/producer-data" \ + --kandelo-repository "$KANDELO_REPOSITORY" \ + --base-commit "$BASE_SHA" \ + --producer-commit "$PRODUCER_SHA" \ + --workflow-authority-commit "$BASE_SHA" \ + --pr-number "$PR_NUMBER" \ + --tap-root "$GITHUB_WORKSPACE/tap-data" \ + --tap-repository "$TAP_REPOSITORY" \ + --tap-name kandelo-dev/tap-core \ + --source-tap-commit "$TAP_SHA" \ + --tap-workflow-authority-commit "$CALLER_SHA" \ + --out "$RUNNER_TEMP/rederived-candidate-campaign-source.json" + cmp -s "$source" \ + "$RUNNER_TEMP/rederived-candidate-campaign-source.json" || { + echo "::error::candidate execution changed protected source evidence" + exit 1 + } + run_json="$RUNNER_TEMP/candidate-campaign-run-live.json" + pages="$RUNNER_TEMP/candidate-campaign-artifact-pages.json" + artifacts="$RUNNER_TEMP/candidate-campaign-artifacts.json" + gh api "/repos/$TAP_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ + >"$run_json" + gh api --paginate --slurp \ + "/repos/$TAP_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100" \ + >"$pages" + jq -e '[.[].artifacts[]]' "$pages" >"$artifacts" + artifact_name="homebrew-candidate-campaign-derivation-" + artifact_name+="attempt-$GITHUB_RUN_ATTEMPT" + jq -e \ + --arg caller "$CALLER_SHA" \ + --arg repository "$TAP_REPOSITORY" \ + --argjson run_id "$GITHUB_RUN_ID" \ + --argjson attempt "$GITHUB_RUN_ATTEMPT" ' + .id == $run_id and .run_attempt == $attempt and + .head_sha == $caller and + .path == ".github/workflows/candidate-campaign.yml" and + (.repository.full_name | ascii_downcase) == + ($repository | ascii_downcase) and + .event == "repository_dispatch" and + (.status == "in_progress" or + (.status == "completed" and .conclusion == "success")) + ' "$run_json" >/dev/null || { + echo "::error::candidate campaign live run is not exact" + exit 1 + } + jq -e --arg name "$artifact_name" ' + [ .[] | select(.name == $name) ] as $selected | + ($selected | length) == 1 and + $selected[0].expired == false and + ($selected[0].id | type == "number" and . > 0) and + ($selected[0].size_in_bytes | type == "number" and . > 0) and + ($selected[0].digest | type == "string" and + test("^sha256:[0-9a-f]{64}$")) + ' "$artifacts" >/dev/null || { + echo "::error::candidate campaign derivation artifact is absent" + exit 1 + } + jq -nS \ + --arg repository "$TAP_REPOSITORY" \ + --arg caller "$CALLER_SHA" \ + --arg name "$artifact_name" \ + --argjson run_id "$GITHUB_RUN_ID" \ + --argjson attempt "$GITHUB_RUN_ATTEMPT" \ + --slurpfile run "$run_json" \ + --slurpfile artifacts "$artifacts" ' + ($artifacts[0] | map(select(.name == $name))[0]) as $artifact | + { + schema:1, + repository:$repository, + workflow_path:".github/workflows/candidate-campaign.yml", + caller_commit:$caller, + event:"repository_dispatch", + run_id:$run_id, + run_attempt:$attempt, + status:$run[0].status, + conclusion:$run[0].conclusion, + artifacts:[{ + id:$artifact.id, + name:$artifact.name, + bytes:$artifact.size_in_bytes, + digest:$artifact.digest, + run_id:$run_id, + run_attempt:$attempt + }] + } + ' >"$RUNNER_TEMP/candidate-campaign-run.json" + + - name: Prepare inert noncanonical campaign release + shell: bash + run: | + set -euo pipefail + [ -z "${GH_TOKEN:-}" ] && [ -z "${GITHUB_TOKEN:-}" ] || { + echo "::error::candidate campaign preparation received credentials" + exit 2 + } + python3 authority/scripts/homebrew-candidate-campaign.py prepare \ + --source \ + "$RUNNER_TEMP/candidate-campaign-derivation/candidate-campaign-source.json" \ + --run-evidence "$RUNNER_TEMP/candidate-campaign-run.json" \ + --campaign \ + "$RUNNER_TEMP/candidate-campaign-derivation/campaign.json" \ + --out "$RUNNER_TEMP/prepared-candidate-campaign" + + - name: Publish immutable candidate campaign + id: publish + shell: bash + env: + BASE_SHA: ${{ needs.admit.outputs.base-sha }} + CALLER_SHA: ${{ needs.admit.outputs.caller-sha }} + GH_TOKEN: ${{ github.token }} + STATE_LOCK_OWNER_DETAIL: immutable Homebrew candidate campaign + run: | + set -euo pipefail + bash authority/scripts/publish-immutable-github-release.sh \ + --manifest \ + "$RUNNER_TEMP/prepared-candidate-campaign/release-manifest.json" \ + --asset-root \ + "$RUNNER_TEMP/prepared-candidate-campaign/assets" \ + --lock-root "$GITHUB_WORKSPACE/tap-authority" \ + --receipt "$RUNNER_TEMP/candidate-campaign-release.json" \ + --exact-kandelo-main-sha "$BASE_SHA" \ + --target-main-contains-sha "$CALLER_SHA" + tag="$(cat "$RUNNER_TEMP/prepared-candidate-campaign/tag.txt")" + echo "candidate-tag=$tag" >>"$GITHUB_OUTPUT" + { + echo "### Immutable Homebrew candidate campaign" + echo + printf "Candidate: \`%s\`\n\n" "$tag" + echo "This release cannot publish canonical bottles before merge." + } >>"$GITHUB_STEP_SUMMARY" + + - name: Retain candidate campaign publication evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: >- + homebrew-candidate-campaign-release-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/candidate-campaign-release.json + if-no-files-found: error + retention-days: 90 diff --git a/docs/agent-guidance/packages-and-builds.md b/docs/agent-guidance/packages-and-builds.md index b8e565f0b9..16f4f6dd49 100644 --- a/docs/agent-guidance/packages-and-builds.md +++ b/docs/agent-guidance/packages-and-builds.md @@ -91,6 +91,40 @@ reachability, a tag, a merge method, or equality of only selected files is not this proof. Existing v1 generations remain readable, but new preparation uses v2. +The transitional Homebrew candidate lane is the only path that may +retain bottle bytes produced by an unmerged commit `S`. Version 1 is +limited to one dependency-free `wasm32` Formula. Candidate execution +must have no registry or tap write credential, and its immutable +release must remain noncanonical. + +Derive a noncanonical candidate campaign before building the bottle. +It must bind protected base `B`, producer `S`, protected tap caller and +source, native Homebrew source, package catalog, ABI snapshot, guest +layout, workflow run, and artifact. A catalog at an older ABI is valid +input to a newer candidate ABI only when every ABI-mismatched variant +is forced to rebuild. Reject a catalog newer than the candidate ABI. + +Promotion requires one merge commit `M` with exact parents `[B, S]`, +`tree(M) == tree(S)`, a byte-identical regenerated package ledger, and +protected-main code that revalidates the run, artifacts, prepared tap, +and public destinations. Preserve `built_from = S`; record +`validated_against_main = M` separately. A squash, rebase, changed base, +conflict resolution, or changed input invalidates the candidate. + +Treat each 90-day protected sealer receipt as the durable promotion +anchor. Require the exact successful run and attempt, exactly one live +receipt artifact, exact release and asset metadata, and anonymous +readback of every asset. Two-day build artifacts may expire. Require a +full workflow rerun for a new attempt; never combine partial-rerun +evidence. Candidate tap callers must contain literal reviewed Kandelo +SHAs. Never replace their base or merge pins with `@main`. + +The candidate lane's Kandelo pull-request staging release is migration +debt. Do not generalize it into the final bottle architecture. Remove +that bridge once tap-owned Formula builds can derive all inputs from +immutable bottles. Kandelo should then supply platform tools and ABI +inputs while the tap owns build, seal, and artifact publication. + The retired migration-only `identical-package-cache-projection-v1` method described distinct complete trees without claiming payload equivalence. Its historical reader and diff --git a/docs/binary-releases.md b/docs/binary-releases.md index 3cdfbb6185..ddf2e6fd5b 100644 --- a/docs/binary-releases.md +++ b/docs/binary-releases.md @@ -84,6 +84,47 @@ These releases are inert campaign inputs; they do not select a Formula or update tap Git state. Only the final complete handoff set may produce the atomic tap commit. +One transitional release kind seals an unmerged campaign under the +`homebrew-prefix-campaign-candidate-pr-...-sha256-...` namespace. +It contains exactly `campaign.json` and +`candidate-campaign.json`. Those assets bind the protected base, +pull-request producer, protected tap caller and source, native Homebrew +source, package catalog, ABI snapshot, guest layout, exact workflow +run, and derivation artifact. It is noncanonical and does not select a +Formula or write a package. + +A second transitional release stages one unmerged leaf bottle under +`homebrew-bottle-candidate-pr--run--attempt--sha256-`. +It binds the candidate campaign, exact pull-request producer, protected +validator and tap caller, package ledger, prepared tap, build handoff, +OCI child, and run artifacts. It is immutable and publicly readable, +but it is deliberately noncanonical: it does not update GHCR, a Formula +bottle block, or tap state. + +Each candidate sealer retains a small protected-code release receipt +for 90 days. The receipt binds the release identity, target commit, +immutability, and complete public asset inventory. The bulky build and +derivation artifacts may expire after two days; promotion reconstructs +their exact bytes from the immutable public releases. Promotion is +therefore available while both 90-day receipt artifacts remain live. + +After an exact merge, current-main code may reconstruct those same +bytes and publish them through the normal GHCR path. The merge commit +must have the recorded base and producer as its two exact parents, and +its complete tree must equal the producer tree. Provenance keeps the +pull-request head as `built_from` and records the merge commit as +`validated_against_main`. Squash, rebase, conflict resolution, or any +input change invalidates the candidate. See [candidate promotion] for +the operational protocol and version-1 limits. + +Candidate caller templates use exact Kandelo SHA placeholders rather +than `@main`. Before a build they are rendered with the protected base +SHA. After merge the promotion caller is rendered again with the exact +merge SHA. Protected reusable workflows inspect the exact tap caller +commit and reject mutable, unresolved, or different authority refs. + +[candidate promotion]: homebrew-publishing.md#candidate-promotion + The campaign release binds the path and SHA-256 of `homebrew/kandelo-guest-layout.json`. That digest selects `/opt/kandelo/homebrew` and its Cellar throughout bottle build, diff --git a/docs/homebrew-packaging-system.md b/docs/homebrew-packaging-system.md index 4ec4d23439..ca98389287 100644 --- a/docs/homebrew-packaging-system.md +++ b/docs/homebrew-packaging-system.md @@ -348,38 +348,169 @@ An ABI bump changes the contract between Kandelo programs and the kernel. Every bottle for the new ABI must therefore be rebuilt, even when the upstream software version did not change. -### There is no canonical candidate-bottle lane today - -The Kandelo merge-candidate workflow can build package-registry archives -for the proposed ABI, create an isolated candidate index, and run -kernel, libc, POSIX, Node, and browser validation against the synthetic -merge. These are candidate artifacts. They are valuable test evidence, -but Prepare Merge does not publish public canonical Homebrew bottles. - -The complete Homebrew publisher intentionally cannot run from -PR-controlled workflow code. Ordinary canonical publication also -requires exact protected `main` authority. A pull-request head, a -synthetic merge, or a commit that may become an ancestor later does not -satisfy that rule. Tree equality is used after merge to admit -package-generation inputs; it does not turn a pre-merge Homebrew bottle -into a canonical publication. - -The reviewed prefix-campaign mode is a narrow exception for a sealed -source commit that is already in protected `main` history. It may -continue while that exact source remains an ancestor, and it preserves -that source in bottle provenance. It never admits a PR-only commit. - -A workflow already reviewed on protected tap `main` may select an -unmerged Kandelo SHA as read-only input for a no-write dry run. The -trusted workflow, not the selected candidate, still owns the job graph -and permissions. This is useful for ABI-neutral publisher or Formula -testing when all required package inputs already exist. It does not make -the candidate protected or canonical. - -That dry-run path is not a complete new-ABI bottle prebuild. A new ABI -does not yet have the durable package generations needed by the bottle -builder, and dry runs are not allowed to create or substitute those -generations. +### Transitional pre-merge candidate bottles + +Kandelo has a narrow candidate lane for building a bottle before its +producer pull request merges. It exists to move ABI testing earlier +without letting pull-request code publish a canonical package. + +As of 2026-08-01, this repository contains the implementation and +caller templates. The lane is not live until a coordinated tap commit +installs rendered exact-SHA callers. Do not dispatch the candidate +events while the live tap lacks those rendered files. + +The first version supports one leaf Formula for `wasm32`. A leaf has no +Homebrew runtime dependencies in the selected campaign. Dependency +Formulae and `wasm64` need a later version of the protocol. + +The protocol gives each commit one role: + +- `B` is the protected Kandelo `main` commit used as the pull request's + base; +- `S` is the exact pull-request head that produces the candidate bottle; +- `A` is the protected Kandelo workflow and validator authority, which + must equal `B` in version 1; +- `C` is the exact protected tap commit that owns a caller run; +- `T` is the protected tap source used to prepare the Formula; and +- `M` is the later merge commit that may admit the candidate. + +The candidate caller lives on protected tap `main`. Its build jobs have +read-only permissions. Candidate Formula and Kandelo code can run there, +but no registry or tap write credential is present. After that execution +has stopped, code from `A` validates the results and seals them in one +immutable, run-bound candidate release. The release tag says that it is +a candidate. It is not a Homebrew version tag, a Formula update, or a +canonical bottle reference. + +Before a bottle is built, a separate candidate-campaign run binds `B`, +`S`, `C`, `T`, the native Homebrew source, ABI snapshot, guest layout, +package catalog, and complete Formula plan. Its immutable release uses +the noncanonical +`homebrew-prefix-campaign-candidate-pr-...-sha256-...` namespace. The +release contains only `campaign.json` and +`candidate-campaign.json`. Neither asset selects a public Formula. + +The candidate campaign may use a package catalog from the same ABI or +an older ABI. It must reject a catalog from a newer ABI. When the +catalog is older, every catalog variant is marked for rebuild because +an older-ABI archive cannot be reused under the candidate ABI. This +lets an ABI pull request plan its first bottles without pretending that +old archives are compatible. + +The bottle candidate then binds that campaign, `B`, `S`, protected +validator authority `A`, `C`, `T`, the prepared tap tree, Formula, +package input ledger, build handoff, OCI child, workflow run, and +artifact identities. Version 1 also binds an empty dependency list. A +rerun gets a distinct tag and cannot replace an earlier candidate. + +The successful sealer also uploads one small release receipt for 90 +days. The receipt binds the public release ID, tag, target commit, +immutability, and the complete asset inventory by ID, name, size, URL, +and SHA-256. This receipt is the durable proof that protected code +accepted the release. The larger derivation and build artifacts may +expire after two days because the immutable public release retains the +bytes needed for promotion. + +The package input currently comes from Kandelo's immutable pull-request +staging release. This is an explicit migration bridge, not the final +Homebrew ownership model. It is read as inert data by protected +validator code. The bridge must be removed once the tap-owned candidate +builder can derive every build input from Formulae and immutable +bottles. At that point the Formula build, seal, and candidate artifacts +should all be owned by the tap. + +### Exact merge and promotion + +A candidate can be promoted only after GitHub reports the pull request +as merged and all of these statements are true: + +```text +parents(M) = [B, S] +tree(M) = tree(S) +M and S are in protected main history +``` + +This requires a merge commit that preserves the exact pull-request head. +A squash merge, rebase merge, conflict resolution, changed pull-request +head, or changed base invalidates the candidate. Build another candidate +instead of claiming that different source produced the old bytes. + +The trusted workflow at `M` then regenerates the complete package +ledger and compares it byte for byte with the candidate input. It also +rederives the campaign from `S`, `T`, and the native Homebrew source. +Public registry observations recorded while the candidate was sealed +are replayed because unrelated bottles may have been published since +then. The exact bottle being promoted still receives a live collision +check before every public write. + +The workflow locates the exact original run and attempt recorded in the +candidate. The run must be complete and successful, and every job in +that attempt must have completed successfully or been skipped. Exactly +one live 90-day sealer-receipt artifact must match the recorded run and +caller commit. Promotion validates that artifact's ID, name, size, +digest, and workflow-run identity before downloading it by artifact ID. + +Protected code then exact-key validates the receipt and compares it +with the live immutable release. The repository, tag, target commit, +release ID, immutability flag, and complete asset inventory must all +match. Every release asset is downloaded anonymously and rehashed. The +workflow reconstructs the prepared tap and the exact build and OCI +handoffs from those release bytes. It does not rebuild the bottle and +does not depend on the two-day build artifacts still existing. + +Promotion is available only while both 90-day receipt artifacts remain +live. If a candidate workflow needs another attempt, use **Re-run all +jobs**. Do not use a partial rerun or combine evidence from different +attempts. The new complete attempt receives its own candidate tag and +receipts. + +The canonical publisher receives the reconstructed bytes only after +those checks. Bottle provenance continues to say `built_from = S`. +Separate admission evidence records `validated_against_main = M`. This +is more accurate than rewriting the producer to `M`, which never built +the archive. + +Successful promotion publishes the exact OCI bytes and an immutable +Formula handoff for the campaign. The bottle is then durable and can be +selected by its immutable digest. The campaign's later tap finalization +still updates the public Formula bottle block and normal `brew` +selection. An unrelated failed Formula does not invalidate a promoted +bottle. The candidate release itself remains candidate evidence; it is +never renamed or treated as the canonical package. + +### Installing exact candidate callers + +The three caller files stored under +`homebrew/homebrew-tap-core/.github/workflows/` are templates. They +contain deliberate placeholders and must not be copied directly into +the live tap. GitHub does not allow an expression in a reusable +workflow `uses:` reference, so the caller files must contain literal +Kandelo commit SHAs. + +Before a pre-merge candidate build, render the callers with `B` as both +inputs: + +```sh +python3 scripts/homebrew-candidate-caller-pins.py render \ + --template-root homebrew/homebrew-tap-core \ + --base-sha "$B" \ + --merge-sha "$B" \ + --out rendered-candidate-callers +``` + +Install the three files from +`rendered-candidate-callers/.github/workflows/` in one protected tap +commit. The campaign and bottle callers now execute reusable workflows +from exact `B`. The promotion caller is intentionally pinned to `B`, so +it cannot admit a later merge accidentally. + +After the exact merge creates `M`, render again with `--base-sha "$B"` +and `--merge-sha "$M"`. Install all three files in one new protected tap +commit. The campaign and bottle callers remain pinned to `B`; the +promotion caller now executes the materializer and publisher from exact +`M`. Each reusable workflow checks the literal pins in the exact tap +caller commit `C` that GitHub reports for that run. A mutable `@main` +reference, an unresolved placeholder, or a different SHA fails closed. ### Supported ABI-bump sequence @@ -393,26 +524,38 @@ Use this sequence for an ABI candidate: the synthetic merge, creates an isolated candidate index, and runs the relevant kernel, libc, POSIX, Node, and browser suites. Fix the platform before adding package-specific workarounds. -3. Merge the exact prepared tree. The post-merge +3. Render and install the pre-merge callers with exact `B` as described + above. For an early bottle test, dispatch the protected tap's + `prepare-kandelo-candidate-campaign` event with exact `S`, `T`, and + pull-request number. It creates the noncanonical candidate-campaign + tag. Keep `B`, `S`, and `T` unchanged while it derives and seals. +4. Dispatch `build-kandelo-bottle-candidate` with exact `S`, `T`, + Formula, candidate-campaign tag, pull-request number, and staging + tag. Independent version-1 leaf Formulae may run in parallel. Skip + this step for Formulae outside the `wasm32` leaf scope. +5. Merge the exact prepared tree. The post-merge `activate-merge-candidate.yml` workflow verifies that the tested producer tree equals the resulting `main` tree, creates the new `binaries-abi-v` release, copies the complete tested closure, commits one canonical index transaction, and publishes the release once. `force-rebuild.yml` is not the initializer for a new ABI release. -4. Read the final `main` commit `M` and the immutable archive producer +6. Read the final `main` commit `M` and the immutable archive producer `S` from activation evidence. Promote the required roots with `promote-package-generation.yml`, using `identical-git-tree-v1` to prove that the complete `S` tree equals `M`. The archives keep truthful `S` provenance even when `S` and `M` are different commit identities. -5. Rotate the tap's pinned Kandelo workflow trust to `M` and run a +7. Render the callers again with exact `B` and `M`, then install that + coordinated tap commit. If step 4 created a candidate, dispatch + `promote-kandelo-bottle-candidate` with `S`, `M`, its candidate tag, + Formula, and exact `M`-bound rootfs generation. Otherwise run a no-write bottle canary. -6. Publish Formulae in dependency order. Run independent branches in +8. Publish Formulae in dependency order. Run independent branches in parallel. Each anonymously verified bottle becomes usable immediately; do not wait for unrelated failures before consuming it by immutable digest. -7. Recompose and validate the selected VFS image. Deploy only the image +9. Recompose and validate the selected VFS image. Deploy only the image and guest lifecycle claims that passed both Node and browser evidence. @@ -421,31 +564,19 @@ transaction. The dependency graph imposes ordering, but independent leaves can publish at the same time and successful results remain useful. -### Why not publish candidate bottles before merge? - -Publishing before merge would make unmerged code a public package -authority. Preserving a PR commit with a merge commit only proves that -it became an ancestor; it does not prove that the bottle was produced -from the final protected-main identity. Kandelo does not use that -history trick as the normal release model. - -A future safe prebuild lane is possible, but it needs an explicit -design. At minimum it must: - -- build without registry or tap write credentials; -- store candidate bottles in a quarantined, run-bound namespace; -- bind the exact Formula, dependencies, SDK, sysroot, ABI, and synthetic - merge tree; -- after merge, prove that every output-affecting input is identical to - final `main`; -- preserve the candidate producer in immutable `built_from` provenance - and record final-main validation as separate admission evidence; and -- publish through a trusted default-branch workflow that rechecks all - public destinations before mutation. - -Until that promotion contract exists, build and test the ABI and package -candidate before merge, activate its complete package closure after -merge, then build Homebrew bottles through exact-main publication. +### Why candidates stay noncanonical before merge + +Publishing directly from `S` would make unmerged code a package +authority. The candidate lane instead separates three actions: + +1. untrusted source produces bytes without write credentials; +2. protected code seals those bytes under a candidate-only identity; +3. protected post-merge code may publish the same bytes after proving + the exact merge and every bound input. + +The third action is what makes the bottle canonical. A public immutable +candidate release is only quarantined evidence; public readability does +not grant package authority. ## VFS images and lazy bottles diff --git a/docs/homebrew-publishing.md b/docs/homebrew-publishing.md index 4eb303613c..682e5ec58a 100644 --- a/docs/homebrew-publishing.md +++ b/docs/homebrew-publishing.md @@ -292,13 +292,18 @@ frozen build and verifier realms without uploading packages, writing an index or tap, or publishing a release. Do not call the complete publisher from merge-candidate pull-request -code. GitHub validates every permission requested by the reusable -workflow, including write-capable jobs that dry-run conditions skip. -Granting those permissions to the pull request would let changed -workflow bytes make a write job reachable. The exact Linux input and -TLS proof is therefore the read-only pre-merge gate; the complete -publisher-realm dry-run is the immediate post-merge gate from reviewed -tap-main workflow bytes. +workflow code. GitHub validates every permission requested by the +reusable workflow, including write-capable jobs that conditions skip. +Granting those permissions to a pull request would let changed workflow +bytes make a write job reachable. + +The candidate-bottle lane is a bounded exception owned by protected tap +`main`, not by the pull request. Its Formula and producer execution jobs +are read-only. A separate final job uses protected validator code and a +contents-only token to seal inert candidate assets. It cannot write a +package or tap state. Exact Linux input and TLS validation remain the +general read-only pre-merge gates; the candidate lane adds one early +`wasm32` leaf-bottle proof. Those two checks are deliberately compositional. The pre-merge Ruby lifecycle proves the signed install plan and certificate output in the @@ -1793,6 +1798,213 @@ use the same exact source contract. Rollback does not consume `tap_sha`; it refreshes and mutates the current protected branch under the tap-wide state lock. +### Candidate Promotion + +Kandelo carries templates for three additional protected-tap callers: + +```text +.github/workflows/candidate-campaign.yml +.github/workflows/candidate-bottles.yml +.github/workflows/promote-candidate-bottle.yml +``` + +They implement a three-stage trust boundary. The campaign and bottle +stages can run the unmerged Formula and Kandelo producer without a +registry token. The promotion stage runs only after an exact merge and +is the first stage allowed to publish the retained OCI bytes. + +As of 2026-08-01, these files are not installed in the live tap. The +candidate lane remains unavailable until a coordinated tap change +installs rendered exact-SHA callers. + +The repository copies of these callers are templates, not deployable +workflows. Their reusable-workflow references contain exact-SHA +placeholders. Before candidate work starts, render them with protected +base `B` as both the base and merge SHA, and install all three files in +one protected tap commit: + +```sh +python3 scripts/homebrew-candidate-caller-pins.py render \ + --template-root homebrew/homebrew-tap-core \ + --base-sha "$B" \ + --merge-sha "$B" \ + --out rendered-candidate-callers +``` + +Install the three files from +`rendered-candidate-callers/.github/workflows/`. Do not install a +template containing a placeholder or replace a literal SHA with +`@main`. The candidate campaign and bottle workflows check that the +exact tap caller commit reported by GitHub pins their protected Kandelo +base. + +Version 1 accepts one `wasm32` leaf Formula. It rejects a Formula whose +actual build receipt has dependencies, even if the request claims an +empty list. It also rejects VFS publication and live tap finalization. +Those restrictions keep the first protocol small enough to audit. + +Before dispatching a candidate, record these immutable values: + +- pull-request number; +- pull-request head `S` and current base `B`; +- current source tap commit `T`; +- exact pull-request staging release tag; +- Formula name and architecture. + +First dispatch `prepare-kandelo-candidate-campaign` to the tap: + +```json +{ + "event_type": "prepare-kandelo-candidate-campaign", + "client_payload": { + "kandelo_ref": "", + "pr_number": , + "tap_ref": "" + } +} +``` + +The read-only derivation job runs candidate code. A separate sealer +runs protected code from `B`, treats the `S` and `T` checkouts as inert +data, and byte-compares a protected re-description of the source. It +also proves the exact completed workflow run and derivation artifact. +The sealer then publishes exactly two assets in an immutable release: + +```text +campaign.json +candidate-campaign.json +``` + +Its tag starts with +`homebrew-prefix-campaign-candidate-pr-`. The candidate package catalog +may have the same ABI as `S` or an older ABI. A newer-ABI catalog is +rejected. Every older-ABI variant is classified as a required rebuild; +none can be reused under the new ABI. + +Keep `B`, `S`, and `T` unchanged during derivation and sealing. Then +use the reported candidate-campaign tag to dispatch +`build-kandelo-bottle-candidate`: + +Dispatch `build-kandelo-bottle-candidate` to the tap with this payload +shape: + +```json +{ + "event_type": "build-kandelo-bottle-candidate", + "client_payload": { + "kandelo_ref": "", + "tap_ref": "", + "formula": "", + "arch": "wasm32", + "pr_number": , + "package_staging_tag": "", + "prefix_campaign_tag": "", + "prefix_campaign_dependencies": + "{\"dependencies\":[],\"schema\":1}" + } +} +``` + +The run fails if `B` is not still current Kandelo `main`. It produces an +immutable release whose tag starts with +`homebrew-bottle-candidate-pr-`. The run summary reports the complete +tag. That release is public evidence, but it is not a canonical bottle +and does not change GHCR or the tap. + +Each successful sealer retains one small Actions artifact for 90 days. +It is a protected-code receipt for the public release. The receipt +binds the repository, tag, target commit, release ID, immutability, and +the full public asset inventory by ID, name, size, URL, and SHA-256. The +large derivation and build artifacts retain only two days because the +immutable releases contain the bytes needed later. Promotion therefore +has a 90-day window; it does not require those large Actions artifacts. + +If either workflow must be rerun, use **Re-run all jobs**. A partial +rerun is not a supported evidence source. Do not combine a sealer from +one attempt with derivation or build evidence from another. A complete +new attempt receives a distinct tag and receipt. + +Do not update the pull-request head or base after this point. Merge with +a true merge commit `M`. Promotion requires both exact Git statements: + +```text +parents(M) = [B, S] +tree(M) = tree(S) +``` + +The merge commit and producer must also remain in protected `main` +history. Squash merge, rebase merge, conflict resolution, or another +base commit makes the candidate ineligible. + +Once `M` exists, render the callers again with exact `B` and `M`, and +install all three rendered files in one new protected tap commit. The +campaign and bottle callers remain pinned to `B`. The promotion caller +now pins both reusable workflows to exact `M`. Protected promotion code +checks those literal pins against the exact caller commit before it +uses any read or write authority. + +After merge, create the exact `M`-bound `rootfs` package generation. +Then dispatch `promote-kandelo-bottle-candidate` with this payload: + +```json +{ + "event_type": "promote-kandelo-bottle-candidate", + "client_payload": { + "candidate_tag": "", + "formula": "", + "merge_commit": "", + "producer_sha": "", + "rootfs_generation": "" + } +} +``` + +Protected code at `M` reads each candidate's recorded run ID and attempt +from its immutable release. It requires the original run to be complete +and successful, and every job in that attempt to be successful or +skipped. It requires exactly one live 90-day receipt artifact for that +run, checks its ID, name, size, digest, run ID, and caller SHA, and then +downloads that exact artifact. + +The receipt is exact-key validated. Its repository, tag, target commit, +release ID, immutability, and complete asset inventory must equal the +live release. Protected code anonymously downloads and rehashes every +asset. It validates `candidate.json` before downloading larger bottle +assets, regenerates the package ledger, recreates the prepared tap, and +proves the merge. The package-input bytes must be present in the exact +release inventory and their SHA-256 must equal the promotion receipt. + +Promotion also rederives the campaign from `S`, `T`, and the native +Homebrew source. It replays the sealed public-registry observations +because successful sibling bottles may have changed registry state +after sealing. The bottle being promoted still receives a live +collision check before every public write. It copies the exact build +and OCI handoffs from the immutable release into the normal publisher. +It does not rebuild the bottle. + +The uploader rechecks both the content-addressed transport tag and the +Homebrew version index. The former prevents replacement of immutable +bytes. The latter prevents a different ABI bottle from occupying the +same Homebrew version and rebuild selection. Only then may the normal +package token enter the upload step. + +Published metadata preserves `built_from = S` and separately records +`validated_against_main = M`. The tap also publishes an immutable +Formula handoff. Campaign finalization later updates the public Formula +and makes normal `brew` resolution select it. The handoff and bottle +remain useful by immutable digest while unrelated Formulae fail or wait. + +The pull-request staging release used by version 1 is migration debt. +It lets the existing Kandelo package builder supply an exact dependency +ledger while the Homebrew migration is incomplete. Remove this bridge +when the tap can build candidate Formulae entirely from tap-owned +Formula and bottle inputs. The intended final ownership is: + +- Kandelo supplies the SDK, ABI contract, and reusable platform tools; +- the tap owns Formula selection and build execution; +- the tap seals candidate and canonical bottle evidence; and +- no Kandelo package-registry staging release is needed for a bottle. + ### Prefix Campaign Publisher Mode The guest-prefix campaign is a narrow mode of the same reusable diff --git a/docs/plans/2026-07-29-homebrew-guest-prefix-cutover.md b/docs/plans/2026-07-29-homebrew-guest-prefix-cutover.md index 61f50bd5c1..245db358cd 100644 --- a/docs/plans/2026-07-29-homebrew-guest-prefix-cutover.md +++ b/docs/plans/2026-07-29-homebrew-guest-prefix-cutover.md @@ -44,6 +44,39 @@ snapshot. Package-changing resolution instead uses the candidate index, whose unchanged entries were derived from the captured base. Do not restore a GitHub token to synthetic materialization. +Candidate-bottle update, 2026-08-01: the transitional version-1 lane +may build one dependency-free `wasm32` Formula from a pull-request head +before merge. Protected tap code seals the result in an immutable, +noncanonical release without a package token. After an exact merge +commit, current-main code can prove exact parents and complete tree +equality, regenerate the package ledger, and promote those same bytes. +Bottle provenance remains attached to the pull-request producer; the +merge commit is separate admission evidence. + +The lane first seals a separate candidate campaign from exact `B`, `S`, +tap caller and source, native Homebrew source, package catalog, ABI +snapshot, guest layout, workflow run, and artifact identities. A +catalog from an older ABI may plan the newer candidate, but every +ABI-mismatched variant must rebuild. A future-ABI catalog is invalid. +The candidate campaign and bottle releases remain noncanonical until +protected code at the exact merge commit revalidates and promotes one +bottle at a time. + +Promotion uses the small protected sealer receipts retained for 90 +days, not the bulky two-day derivation and build artifacts. It proves +the exact completed run attempt, validates the complete immutable +release inventory, and anonymously rehashes every release asset. A new +attempt requires a full workflow rerun. Candidate tap callers are +rendered from placeholders to literal base and merge SHAs; mutable +`@main` reusable-workflow references are not accepted. + +This lane is intended for early ABI-candidate testing. It does not turn +candidate releases into public Formula state and does not yet handle +dependency waves or `wasm64`. Kandelo pull-request staging supplies its +package input only as a migration bridge. Before this plan is complete, +replace that bridge with tap-owned Formula and bottle inputs so the tap +owns build, seal, and artifacts end to end. + ## Accelerated Usable Cutover Checkpoint: 2026-07-31 This checkpoint separates the first usable in-guest Homebrew delivery diff --git a/homebrew/homebrew-tap-core/.github/workflows/candidate-bottles.yml b/homebrew/homebrew-tap-core/.github/workflows/candidate-bottles.yml new file mode 100644 index 0000000000..760792e163 --- /dev/null +++ b/homebrew/homebrew-tap-core/.github/workflows/candidate-bottles.yml @@ -0,0 +1,29 @@ +name: Build Kandelo bottle candidate + +on: + repository_dispatch: + types: [build-kandelo-bottle-candidate] + +jobs: + candidate: + # Formula and PR code run only in read-only jobs. The reusable workflow's + # reviewed final job receives contents:write solely to seal inert bytes in + # a run-bound release after all candidate execution has stopped. + permissions: + actions: read + contents: write + uses: Automattic/kandelo/.github/workflows/reusable-homebrew-bottle-publish.yml@__KANDELO_CANDIDATE_BASE_SHA__ + with: + kandelo-repository: Automattic/kandelo + kandelo-ref: ${{ github.event.client_payload.kandelo_ref }} + tap-repository: kandelo-dev/homebrew-tap-core + tap-name: kandelo-dev/tap-core + tap-ref: ${{ github.event.client_payload.tap_ref }} + formulae: ${{ github.event.client_payload.formula }} + arches: ${{ github.event.client_payload.arch }} + force: true + dry-run: true + candidate-pr-number: ${{ github.event.client_payload.pr_number }} + candidate-package-staging-tag: ${{ github.event.client_payload.package_staging_tag }} + prefix-campaign-tag: ${{ github.event.client_payload.prefix_campaign_tag }} + prefix-campaign-dependencies: ${{ github.event.client_payload.prefix_campaign_dependencies }} diff --git a/homebrew/homebrew-tap-core/.github/workflows/candidate-campaign.yml b/homebrew/homebrew-tap-core/.github/workflows/candidate-campaign.yml new file mode 100644 index 0000000000..d34ffbbcdb --- /dev/null +++ b/homebrew/homebrew-tap-core/.github/workflows/candidate-campaign.yml @@ -0,0 +1,19 @@ +name: Prepare Kandelo candidate bottle campaign + +on: + repository_dispatch: + types: [prepare-kandelo-candidate-campaign] + +jobs: + campaign: + permissions: + actions: read + contents: write + uses: Automattic/kandelo/.github/workflows/reusable-homebrew-candidate-campaign.yml@__KANDELO_CANDIDATE_BASE_SHA__ + with: + kandelo-repository: Automattic/kandelo + producer-sha: ${{ github.event.client_payload.kandelo_ref }} + pr-number: ${{ github.event.client_payload.pr_number }} + tap-repository: kandelo-dev/homebrew-tap-core + tap-name: kandelo-dev/tap-core + tap-sha: ${{ github.event.client_payload.tap_ref }} diff --git a/homebrew/homebrew-tap-core/.github/workflows/promote-candidate-bottle.yml b/homebrew/homebrew-tap-core/.github/workflows/promote-candidate-bottle.yml new file mode 100644 index 0000000000..cd2d5cb12e --- /dev/null +++ b/homebrew/homebrew-tap-core/.github/workflows/promote-candidate-bottle.yml @@ -0,0 +1,276 @@ +name: Promote exact merged Kandelo bottle candidate + +on: + repository_dispatch: + types: [promote-kandelo-bottle-candidate] + +jobs: + admit: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + candidate-tag: ${{ steps.admit.outputs.candidate-tag }} + formula: ${{ steps.admit.outputs.formula }} + merge-commit: ${{ steps.admit.outputs.merge-commit }} + producer-sha: ${{ steps.admit.outputs.producer-sha }} + rootfs-generation: ${{ steps.admit.outputs.rootfs-generation }} + steps: + - name: Admit one exact promotion request + id: admit + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TAP_REPOSITORY: ${{ github.repository }} + TAP_SHA: ${{ github.sha }} + run: | + set -euo pipefail + payload="$RUNNER_TEMP/candidate-promotion-request.json" + jq -eS ' + .client_payload | + select( + type == "object" and + keys == [ + "candidate_tag", + "formula", + "merge_commit", + "producer_sha", + "rootfs_generation" + ] and + (.candidate_tag | type == "string" and test( + "^homebrew-bottle-candidate-pr-[1-9][0-9]*-run-[1-9][0-9]*-attempt-[1-9][0-9]*-sha256-[0-9a-f]{64}$" + )) and + (.formula | type == "string" and + test("^[a-z0-9][a-z0-9._-]{0,254}$")) and + (.merge_commit | type == "string" and + test("^[0-9a-f]{40}$")) and + (.producer_sha | type == "string" and + test("^[0-9a-f]{40}$")) and + (.rootfs_generation | type == "string" and test( + "^package-generation-rootfs-wasm32-abi-v[1-9][0-9]*-sha256-[0-9a-f]{64}$" + )) + ) + ' "$GITHUB_EVENT_PATH" >"$payload" + current_main="$(gh api \ + "/repos/$TAP_REPOSITORY/git/ref/heads/main" \ + --jq .object.sha)" + [ "$TAP_SHA" = "$current_main" ] || { + echo "::error::candidate promotion requires current tap main" + exit 1 + } + { + echo "candidate-tag=$(jq -r .candidate_tag "$payload")" + echo "formula=$(jq -r .formula "$payload")" + echo "merge-commit=$(jq -r .merge_commit "$payload")" + echo "producer-sha=$(jq -r .producer_sha "$payload")" + echo "rootfs-generation=$(jq -r .rootfs_generation "$payload")" + } >>"$GITHUB_OUTPUT" + + materialize: + needs: [admit] + permissions: + actions: read + contents: read + uses: Automattic/kandelo/.github/workflows/reusable-homebrew-bottle-candidate-materialize.yml@__KANDELO_CANDIDATE_MERGE_SHA__ + with: + candidate-tag: ${{ needs.admit.outputs.candidate-tag }} + formula: ${{ needs.admit.outputs.formula }} + producer-sha: ${{ needs.admit.outputs.producer-sha }} + merge-commit: ${{ needs.admit.outputs.merge-commit }} + + publish: + needs: [admit, materialize] + permissions: + actions: read + contents: read + packages: write + uses: Automattic/kandelo/.github/workflows/reusable-homebrew-bottle-publish.yml@__KANDELO_CANDIDATE_MERGE_SHA__ + with: + kandelo-repository: Automattic/kandelo + kandelo-ref: ${{ needs.admit.outputs.merge-commit }} + tap-repository: kandelo-dev/homebrew-tap-core + tap-name: kandelo-dev/tap-core + tap-ref: ${{ needs.materialize.outputs.source-tap-commit }} + formulae: ${{ needs.admit.outputs.formula }} + arches: wasm32 + release-tag: ${{ needs.materialize.outputs.release-tag }} + expected-cache-keys: "" + package-generation-wasm32: >- + ${{ needs.admit.outputs.rootfs-generation }} + force: true + dry-run: false + require-vfs-acceptance: false + defer-tap-finalization: true + prefix-campaign-tag: ${{ needs.materialize.outputs.campaign-tag }} + prefix-campaign-dependencies: '{"dependencies":[],"schema":1}' + candidate-promotion-tag: ${{ needs.admit.outputs.candidate-tag }} + candidate-producer-sha: ${{ needs.admit.outputs.producer-sha }} + + seal-formula-handoff: + needs: [admit, materialize, publish] + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + actions: read + contents: write + steps: + - name: Checkout protected tap release authority + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + fetch-depth: 0 + path: tap-authority + + - name: Checkout exact merged Kandelo handoff authority + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: Automattic/kandelo + ref: ${{ needs.admit.outputs.merge-commit }} + path: kandelo + submodules: false + + - name: Checkout exact bottle producer as inert data + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: Automattic/kandelo + ref: ${{ needs.admit.outputs.producer-sha }} + path: producer + submodules: false + + - name: Checkout exact campaign source tap + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + repository: kandelo-dev/homebrew-tap-core + ref: ${{ needs.materialize.outputs.source-tap-commit }} + path: source-tap + fetch-depth: 0 + + - name: Download the exact validated publication handoff + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: >- + homebrew-publish-handoff-${{ needs.admit.outputs.formula }}-wasm32-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/candidate-publication + + - name: Download exact candidate campaign admission + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: >- + homebrew-candidate-promotion-${{ needs.admit.outputs.formula }}-wasm32-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/candidate-promotion + + - name: Install Nix for protected handoff validation + uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 + with: + github-token: "" + + - name: Cache Nix store and flake evaluation + uses: DeterminateSystems/magic-nix-cache-action@908b263ff629f4cc17666315b7fd3ec127c6244d # v14 + with: + use-gha-cache: false + use-flakehub: false + + - name: Derive the immutable campaign Formula handoff + id: handoff + shell: bash + env: + ABI: ${{ needs.materialize.outputs.abi }} + CAMPAIGN_TAG: ${{ needs.materialize.outputs.campaign-tag }} + FORMULA: ${{ needs.admit.outputs.formula }} + MERGE_COMMIT: ${{ needs.admit.outputs.merge-commit }} + PRODUCER_SHA: ${{ needs.admit.outputs.producer-sha }} + TAP_SHA: ${{ needs.materialize.outputs.source-tap-commit }} + run: | + set -euo pipefail + campaign="$RUNNER_TEMP/campaign.json" + python3 kandelo/scripts/homebrew-candidate-campaign.py \ + fetch-release \ + --repository kandelo-dev/homebrew-tap-core \ + --tag "$CAMPAIGN_TAG" --out "$campaign" \ + --candidate-out \ + "$RUNNER_TEMP/candidate-campaign-manifest.json" \ + --receipt-out "$RUNNER_TEMP/campaign-readback.json" + layout_sha="$(sha256sum \ + "$GITHUB_WORKSPACE/producer/homebrew/kandelo-guest-layout.json" | + awk '{print $1}')" + python3 kandelo/scripts/homebrew-candidate-campaign.py \ + validate-admission \ + --receipt \ + "$RUNNER_TEMP/candidate-promotion/candidate-campaign-admission.json" \ + --candidate-tag "$CAMPAIGN_TAG" \ + --producer-commit "$PRODUCER_SHA" \ + --merge-commit "$MERGE_COMMIT" \ + --source-tap-commit "$TAP_SHA" \ + --abi "$ABI" \ + --guest-layout-sha256 "$layout_sha" + bash kandelo/scripts/dev-shell.sh \ + python3 kandelo/scripts/homebrew-prefix-campaign-publisher.py \ + prepare --tap-root "$GITHUB_WORKSPACE/source-tap" \ + --kandelo-root "$GITHUB_WORKSPACE/producer" \ + --kandelo-commit "$PRODUCER_SHA" \ + --tap-repository kandelo-dev/homebrew-tap-core \ + --tap-name kandelo-dev/tap-core \ + --source-tap-commit "$TAP_SHA" \ + --campaign-tag "$CAMPAIGN_TAG" \ + --dependencies '{"dependencies":[],"schema":1}' \ + --formula "$FORMULA" --arch wasm32 \ + --work-root "$RUNNER_TEMP/prepared-campaign-tap" \ + --receipt-out "$RUNNER_TEMP/prepared-campaign.json" + bash kandelo/scripts/dev-shell.sh \ + python3 kandelo/scripts/homebrew-prefix-campaign-executor.py \ + derive-build --campaign "$campaign" \ + --source-tap-root "$GITHUB_WORKSPACE/source-tap" \ + --formula "$FORMULA" \ + --publication \ + "wasm32=$RUNNER_TEMP/candidate-publication" \ + --out "$RUNNER_TEMP/candidate-formula-handoff" + bash kandelo/scripts/dev-shell.sh \ + python3 kandelo/scripts/homebrew-prefix-campaign-executor.py \ + prepare-release --campaign "$campaign" \ + --handoff "$RUNNER_TEMP/candidate-formula-handoff" \ + --out "$RUNNER_TEMP/prepared-formula-release" + tag="$(jq -er .tag \ + "$RUNNER_TEMP/prepared-formula-release/release-manifest.json")" + echo "tag=$tag" >>"$GITHUB_OUTPUT" + + - name: Publish the now-main-reachable Formula handoff + shell: bash + env: + FORMULA: ${{ needs.admit.outputs.formula }} + GH_TOKEN: ${{ github.token }} + HANDOFF_TAG: ${{ steps.handoff.outputs.tag }} + PRODUCER_SHA: ${{ needs.admit.outputs.producer-sha }} + SOURCE_TAP_SHA: >- + ${{ needs.materialize.outputs.source-tap-commit }} + STATE_LOCK_OWNER_DETAIL: promoted candidate Formula handoff + run: | + set -euo pipefail + bash kandelo/scripts/publish-immutable-github-release.sh \ + --manifest \ + "$RUNNER_TEMP/prepared-formula-release/release-manifest.json" \ + --asset-root \ + "$RUNNER_TEMP/prepared-formula-release/assets" \ + --lock-root "$GITHUB_WORKSPACE/tap-authority" \ + --receipt "$RUNNER_TEMP/formula-handoff-release.json" \ + --kandelo-main-contains-sha "$PRODUCER_SHA" \ + --target-main-contains-sha "$SOURCE_TAP_SHA" + { + echo "### Promoted Kandelo Homebrew bottle" + echo + printf "Formula: \`%s\`\n\n" "$FORMULA" + printf "Handoff: \`%s\`\n\n" "$HANDOFF_TAG" + printf "Built from: \`%s\`\n" "$PRODUCER_SHA" + } >>"$GITHUB_STEP_SUMMARY" + + - name: Retain handoff publication evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: >- + promoted-formula-handoff-${{ needs.admit.outputs.formula }}-wasm32-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/formula-handoff-release.json + if-no-files-found: error + retention-days: 90 diff --git a/scripts/check-homebrew-publish-workflow-trust.rb b/scripts/check-homebrew-publish-workflow-trust.rb index 404ee1a8c7..c9498fd6fc 100644 --- a/scripts/check-homebrew-publish-workflow-trust.rb +++ b/scripts/check-homebrew-publish-workflow-trust.rb @@ -21,6 +21,14 @@ REPO_ROOT, ".github/workflows/reusable-homebrew-prefix-first-child-publish.yml" ) +CANDIDATE_MATERIALIZER_PATH = File.join( + REPO_ROOT, + ".github/workflows/reusable-homebrew-bottle-candidate-materialize.yml" +) +CANDIDATE_CAMPAIGN_PATH = File.join( + REPO_ROOT, + ".github/workflows/reusable-homebrew-candidate-campaign.yml" +) WORKFLOW_ROOT = File.join(REPO_ROOT, ".github/workflows") HOST_RUNTIME_PREPARER_PATH = File.join( REPO_ROOT, "scripts/prepare-homebrew-recipe-host-runtime.py" @@ -52,6 +60,7 @@ # read-only PR workflow follows the repository-wide v7 pin independently. NATIVE_COMPATIBILITY_CHECKOUT_ACTION = "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" +STANDARD_CHECKOUT_ACTION = NATIVE_COMPATIBILITY_CHECKOUT_ACTION NIX_ACTION = "DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25" MAGIC_NIX_ACTION = "DeterminateSystems/magic-nix-cache-action@908b263ff629f4cc17666315b7fd3ec127c6244d" UPLOAD_ACTION = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" @@ -67,13 +76,21 @@ "c8192c2521864005b34e9eaa39d44d11d580997db39d6e64f2afe30fe447eb91" NATIVE_CA_VALIDATION_RUN_SHA256 = "7cb1417ec6df08daefa71c2ee6a364be76737b9d7f7ed4aa4022d3d7ca90a8b9" -PUBLISHER_PLAN_DIGEST = "4064b0f7ae61f01fbe0db68c2ce02cae25b57ae6d3ca55bd15178a609775e48f" -PUBLISHER_BUILD_DIGEST = "b50114b394faf7efb56818fd421ff15acb9630ad838ab10b2da6c4d259075cb0" -PUBLISHER_UPLOAD_DIGEST = "ec78200c83223dec9e8d85d1b25e19e7e748ac7a7c5a2e0ce1f9cafc2ebadcbb" -PUBLISHER_INDEX_DIGEST = "08a8c2360535d9e3dea92af39c4039d2966434049f9a2333d3d118fc6c4a1936" -PUBLISHER_VERIFY_DIGEST = "7755e34bfdb25bd775eb321f3b5588467f8a599440d324710fd9b984cb43a7ef" +PUBLISHER_PLAN_DIGEST = "9fc98b365a0ac90fab0a7eeccbaa029b3e76721b746d2a3229f6c9c888bfc349" +PUBLISHER_BUILD_DIGEST = "8c6e0417ac49ff62ef2bdb2e249ba37706b1a19ea20034b2d81dd5fcea40d5c7" +PUBLISHER_UPLOAD_DIGEST = "5d8592f405480e268ef718160056e2bd423b0d18d46a8c20b5103ea3c0289d90" +PUBLISHER_INDEX_DIGEST = "665ca4762b97ec79d65c12d4854e92e7df95a8abca486774592be35f20f785b2" +PUBLISHER_VERIFY_DIGEST = "e89c081fadbc82b9cf029c4c873d5b10bcadeef35cea90ae4c4562584194eb87" PUBLISHER_FINALIZE_DIGEST = "b17e7bf5d0a5ef512e49f74c224a94958642dfdd80a27439f2a0335816a0886b" PUBLISHER_VFS_RELEASE_DIGEST = "2db9ec075edf382e326066d5f49a32947f5a584fce26a966fb9fff23bbbe3c26" +PUBLISHER_CANDIDATE_DIGEST = + "ca086782ffd59b45f891767fb50d69e46f91f94055de7b86f4b0393d8814ebb3" +CANDIDATE_MATERIALIZER_STEPS_DIGEST = + "976d6c9d1c7b605527d2746e5e6311d7c20ea3937142872cf7c0e7a1ded11aa5" +CANDIDATE_CAMPAIGN_DIGEST = + "c208050bb3c7b2a113e22c7640a59374e127cfefabf8ad45d91ae77852f430d0" +TAP_CANDIDATE_PROMOTION_DIGEST = + "61775eabc3c015231811e5eef2a48d4731b1e834e59c0c5272a4697e2266ed91" MAINTENANCE_VALIDATE_DIGEST = "30ebccd5d44e004e37f168e81284d7ceb18accfa067c05248c1cc19398a7515f" MAINTENANCE_ROLLBACK_DIGEST = "f82d9f351202c3a20824e4525eb88ce7f75879740014d3232e69f3d585ed5781" FIRST_PUBLICATION_STEPS_DIGEST = "cf1c41bbfb91a1e5de6e7e0bfe7c16406dd3d022ad66dec7188cb31240168c7e" @@ -213,6 +230,7 @@ def caller_validation_result(source, overrides = {}) "CALLER_EVENT_NAME" => "repository_dispatch", "CALLER_REF" => "refs/heads/main", "CALLER_REPOSITORY" => "kandelo-dev/homebrew-tap-core", + "CALLER_SHA" => "d" * 40, "CALLER_WORKFLOW_REF" => "kandelo-dev/homebrew-tap-core/.github/workflows/dry-run-bottles.yml@refs/heads/main", "DEFER_TAP_FINALIZATION" => "false", @@ -231,6 +249,10 @@ def caller_validation_result(source, overrides = {}) "TAP_REPOSITORY" => "kandelo-dev/homebrew-tap-core", "TAP_REF" => "main", "BOTTLE_ROOT_URL" => "", + "CANDIDATE_PACKAGE_STAGING_TAG" => "", + "CANDIDATE_PR_NUMBER" => "", + "CANDIDATE_PROMOTION_TAG" => "", + "CANDIDATE_PRODUCER_SHA" => "", }.merge(overrides) Tempfile.create("kandelo-homebrew-trust-output") do |output| @@ -334,7 +356,9 @@ def tap_source_binding_result(source, overrides = {}) def expected_caller_outputs( kandelo_ref, tap_ref, wasm32: "", wasm64: "", kind: "none", - campaign_mode: "false", campaign_tag: "", campaign_dependencies: "" + campaign_mode: "false", campaign_tag: "", campaign_dependencies: "", + candidate_mode: "false", candidate_pr: "", candidate_staging: "", + candidate_tap_authority: "", candidate_promotion_mode: "false" ) "kandelo-ref=#{kandelo_ref}\n" \ "tap-ref=#{tap_ref}\n" \ @@ -343,7 +367,12 @@ def expected_caller_outputs( "package-generation-kind=#{kind}\n" \ "prefix-campaign-mode=#{campaign_mode}\n" \ "prefix-campaign-tag=#{campaign_tag}\n" \ - "prefix-campaign-dependencies=#{campaign_dependencies}\n" + "prefix-campaign-dependencies=#{campaign_dependencies}\n" \ + "candidate-mode=#{candidate_mode}\n" \ + "candidate-pr-number=#{candidate_pr}\n" \ + "candidate-package-staging-tag=#{candidate_staging}\n" \ + "candidate-tap-workflow-authority-sha=#{candidate_tap_authority}\n" \ + "candidate-promotion-mode=#{candidate_promotion_mode}\n" end def check_caller_validation_behavior(workflow) @@ -379,6 +408,157 @@ def check_caller_validation_behavior(workflow) expected_caller_outputs(kandelo_sha, tap_sha), "publisher dry-run does not accept exact source commits") + candidate_tag = "pr-77-staging-run-900-attempt-2" + candidate_campaign_tag = + "homebrew-prefix-campaign-candidate-pr-77-run-899-attempt-2-" \ + "sha256-#{"1" * 64}" + candidate_campaign_dependencies = '{"dependencies":[],"schema":1}' + candidate_tap_authority = "d" * 40 + candidate = caller_validation_result(source, { + "CALLER_WORKFLOW_REF" => + "kandelo-dev/homebrew-tap-core/.github/workflows/candidate-bottles.yml@refs/heads/main", + "FORCE_REBUILD" => "true", + "KANDELO_REF" => kandelo_sha, + "TAP_REF" => tap_sha, + "CANDIDATE_PR_NUMBER" => "77", + "CANDIDATE_PACKAGE_STAGING_TAG" => candidate_tag, + "CALLER_SHA" => candidate_tap_authority, + "PREFIX_CAMPAIGN_TAG" => candidate_campaign_tag, + "PREFIX_CAMPAIGN_DEPENDENCIES" => candidate_campaign_dependencies, + }) + check(candidate["status"] == 0 && candidate["outputs"] == + expected_caller_outputs( + kandelo_sha, + tap_sha, + candidate_mode: "true", + candidate_pr: "77", + candidate_staging: candidate_tag, + candidate_tap_authority: candidate_tap_authority, + campaign_mode: "true", + campaign_tag: candidate_campaign_tag, + campaign_dependencies: candidate_campaign_dependencies + ), "publisher candidate caller does not bind exact authority") + candidate_with_branch = caller_validation_result(source, { + "CALLER_WORKFLOW_REF" => + "kandelo-dev/homebrew-tap-core/.github/workflows/candidate-bottles.yml@refs/heads/main", + "FORCE_REBUILD" => "true", + "KANDELO_REF" => "candidate-branch", + "TAP_REF" => tap_sha, + "CANDIDATE_PR_NUMBER" => "77", + "CANDIDATE_PACKAGE_STAGING_TAG" => candidate_tag, + "CALLER_SHA" => candidate_tap_authority, + "PREFIX_CAMPAIGN_TAG" => candidate_campaign_tag, + "PREFIX_CAMPAIGN_DEPENDENCIES" => candidate_campaign_dependencies, + }) + check(candidate_with_branch["status"] == 2, + "publisher candidate caller accepts a mutable source ref") + { + "write mode" => { "DRY_RUN" => "false" }, + "unforced build" => { "FORCE_REBUILD" => "false" }, + "deferred finalization" => { "DEFER_TAP_FINALIZATION" => "true" }, + "VFS acceptance" => { "REQUIRE_VFS_ACCEPTANCE" => "true" }, + "canonical package generation" => { + "PACKAGE_GENERATION_WASM32" => SELF_TEST_PACKAGE_GENERATION_ROOTFS, + }, + "wasm64 target" => { "ARCHES" => "wasm64" }, + "promotion authority" => { + "CANDIDATE_PROMOTION_TAG" => + "homebrew-bottle-candidate-pr-77-run-900-attempt-2-sha256-#{"2" * 64}", + "CANDIDATE_PRODUCER_SHA" => kandelo_sha, + }, + "dependency-bearing campaign" => { + "PREFIX_CAMPAIGN_DEPENDENCIES" => + '{"dependencies":[{"formula":"zlib","tag":"homebrew-prefix-handoff-sha256-' \ + "#{"3" * 64}" + '"}],"schema":1}', + }, + "canonical campaign namespace" => { + "PREFIX_CAMPAIGN_TAG" => + "homebrew-prefix-campaign-sha256-#{"1" * 64}", + }, + }.each do |label, overrides| + rejected = caller_validation_result(source, { + "CALLER_WORKFLOW_REF" => + "kandelo-dev/homebrew-tap-core/.github/workflows/candidate-bottles.yml@refs/heads/main", + "FORCE_REBUILD" => "true", + "KANDELO_REF" => kandelo_sha, + "TAP_REF" => tap_sha, + "CANDIDATE_PR_NUMBER" => "77", + "CANDIDATE_PACKAGE_STAGING_TAG" => candidate_tag, + "CALLER_SHA" => candidate_tap_authority, + "PREFIX_CAMPAIGN_TAG" => candidate_campaign_tag, + "PREFIX_CAMPAIGN_DEPENDENCIES" => candidate_campaign_dependencies, + }.merge(overrides)) + check(rejected["status"] == 2, + "publisher candidate caller accepts #{label}") + end + + promotion_tag = + "homebrew-bottle-candidate-pr-77-run-900-attempt-2-sha256-#{"2" * 64}" + promotion = { + "CALLER_WORKFLOW_REF" => + "kandelo-dev/homebrew-tap-core/.github/workflows/" \ + "promote-candidate-bottle.yml@refs/heads/main", + "DEFER_TAP_FINALIZATION" => "true", + "DRY_RUN" => "false", + "FORCE_REBUILD" => "true", + "KANDELO_REF" => kandelo_sha, + "TAP_REF" => tap_sha, + "CANDIDATE_PROMOTION_TAG" => promotion_tag, + "CANDIDATE_PRODUCER_SHA" => "c" * 40, + "PACKAGE_GENERATION_WASM32" => SELF_TEST_PACKAGE_GENERATION_ROOTFS, + "PREFIX_CAMPAIGN_TAG" => candidate_campaign_tag, + "PREFIX_CAMPAIGN_DEPENDENCIES" => candidate_campaign_dependencies, + }.freeze + promoted = caller_validation_result(source, promotion) + check(promoted["status"] == 0 && promoted["outputs"] == + expected_caller_outputs( + kandelo_sha, + tap_sha, + wasm32: SELF_TEST_PACKAGE_GENERATION_ROOTFS, + kind: "rootfs-wasm32", + campaign_mode: "true", + campaign_tag: candidate_campaign_tag, + campaign_dependencies: candidate_campaign_dependencies, + candidate_promotion_mode: "true" + ), "publisher promotion caller does not bind its exact merged candidate") + { + "candidate caller path" => { + "CALLER_WORKFLOW_REF" => + "kandelo-dev/homebrew-tap-core/.github/workflows/candidate-bottles.yml@refs/heads/main", + }, + "dry-run mode" => { "DRY_RUN" => "true" }, + "unforced publication" => { "FORCE_REBUILD" => "false" }, + "immediate tap finalization" => { "DEFER_TAP_FINALIZATION" => "false" }, + "VFS acceptance" => { "REQUIRE_VFS_ACCEPTANCE" => "true" }, + "mutable merge ref" => { "KANDELO_REF" => "main" }, + "mutable tap ref" => { "TAP_REF" => "main" }, + "mutable producer" => { "CANDIDATE_PRODUCER_SHA" => "main" }, + "wasm64 target" => { "ARCHES" => "wasm64" }, + "browser package generation" => { + "PACKAGE_GENERATION_WASM32" => SELF_TEST_PACKAGE_GENERATION_WASM32, + }, + "wasm64 package generation" => { + "PACKAGE_GENERATION_WASM64" => SELF_TEST_PACKAGE_GENERATION_WASM64, + }, + "candidate staging identity" => { + "CANDIDATE_PR_NUMBER" => "77", + "CANDIDATE_PACKAGE_STAGING_TAG" => candidate_tag, + }, + "dependency-bearing campaign" => { + "PREFIX_CAMPAIGN_DEPENDENCIES" => + '{"dependencies":[{"formula":"zlib","tag":"homebrew-prefix-handoff-sha256-' \ + "#{"3" * 64}" + '"}],"schema":1}', + }, + "canonical campaign namespace" => { + "PREFIX_CAMPAIGN_TAG" => + "homebrew-prefix-campaign-sha256-#{"1" * 64}", + }, + }.each do |label, overrides| + rejected = caller_validation_result(source, promotion.merge(overrides)) + check(rejected["status"] == 2, + "publisher promotion caller accepts #{label}") + end + data_only = caller_validation_result(source, { "KANDELO_REF" => "review/homebrew;still-data", }) @@ -480,6 +660,14 @@ def check_caller_validation_behavior(workflow) campaign_dependencies: campaign_dependencies ), "publisher rejects the exact reviewed prefix-campaign contract") + candidate_namespace_campaign = caller_validation_result( + source, + campaign_caller.merge("PREFIX_CAMPAIGN_TAG" => + "homebrew-prefix-campaign-candidate-pr-77-run-899-attempt-2-" \ + "sha256-#{"1" * 64}") + ) + check(candidate_namespace_campaign["status"] == 2, + "canonical campaign caller accepts the candidate namespace") campaign_dry_run = caller_validation_result( source, @@ -831,7 +1019,12 @@ def check_common(workflow, label, allowed_secret_nodes: []) "#{label} secret contract changed") end -def check_tap_caller(path, expected_name:, event_type:, job_name:, reusable:, inputs:, secrets: {}) +def check_tap_caller( + path, expected_name:, event_type:, job_name:, reusable:, inputs:, + secrets: {}, permissions: { + "actions" => "read", "contents" => "write", "packages" => "write", + } +) workflow = load_workflow(path) top_keys = workflow.keys.map { |key| key == true ? "on" : key.to_s }.sort check(top_keys == %w[jobs name on], "#{File.basename(path)} has unexpected top-level configuration") @@ -846,9 +1039,8 @@ def check_tap_caller(path, expected_name:, event_type:, job_name:, reusable:, in expected_job_keys << "secrets" unless secrets.empty? check(job.keys.sort == expected_job_keys.sort, "#{File.basename(path)} caller job changed") - check(exact_permissions?(job["permissions"], { - "actions" => "read", "contents" => "write", "packages" => "write", - }), "#{File.basename(path)} permission ceiling changed") + check(exact_permissions?(job["permissions"], permissions), + "#{File.basename(path)} permission ceiling changed") check(job["uses"] == reusable, "#{File.basename(path)} reusable workflow target changed") check(job["with"] == inputs, "#{File.basename(path)} caller inputs changed") check(job.fetch("secrets", {}) == secrets, "#{File.basename(path)} caller secrets changed") @@ -859,6 +1051,1003 @@ def check_tap_caller(path, expected_name:, event_type:, job_name:, reusable:, in "#{File.basename(path)} may pass only its reviewed named secrets") end +def normalized_expression(value) + value.to_s.split.join(" ") +end + +def check_candidate_campaign(workflow) + top_keys = workflow.keys.map { |key| key == true ? "on" : key.to_s }.sort + check(top_keys == %w[jobs name on], + "candidate campaign has unexpected top-level configuration") + check(workflow["name"] == + "Seal an unmerged Homebrew campaign candidate", + "candidate campaign name changed") + check(workflow_events(workflow) == { + "workflow_call" => { + "inputs" => { + "kandelo-repository" => { + "type" => "string", "required" => true, + }, + "producer-sha" => { "type" => "string", "required" => true }, + "pr-number" => { "type" => "number", "required" => true }, + "tap-repository" => { "type" => "string", "required" => true }, + "tap-name" => { "type" => "string", "required" => true }, + "tap-sha" => { "type" => "string", "required" => true }, + }, + "outputs" => { + "candidate-tag" => { + "value" => "${{ jobs.seal.outputs.candidate-tag }}", + }, + }, + }, + }, "candidate campaign call contract changed") + check(!workflow.key?("permissions"), + "candidate campaign requests workflow-wide authority") + check_common(workflow, "candidate campaign") + + jobs = workflow_jobs(workflow) + check(jobs.keys == %w[admit derive seal], + "candidate campaign job graph changed") + admit = jobs.fetch("admit") + derive = jobs.fetch("derive") + seal = jobs.fetch("seal") + check(admit.keys.sort == + %w[outputs permissions runs-on steps timeout-minutes] && + admit["runs-on"] == "ubuntu-latest" && + admit["timeout-minutes"] == 10 && + exact_permissions?(admit["permissions"], { "contents" => "read" }) && + admit["outputs"] == { + "base-sha" => "${{ steps.admit.outputs.base-sha }}", + "caller-sha" => "${{ steps.admit.outputs.caller-sha }}", + }, "candidate campaign admission authority changed") + admit_steps = job_steps(admit, "candidate campaign admission") + check(admit_steps.filter_map { |step| step["uses"] } == [ + STANDARD_CHECKOUT_ACTION, STANDARD_CHECKOUT_ACTION, + ], "candidate campaign admission action set or pin changed") + admission = named_step( + admit_steps, "Bind the request to current protected branches" + ) + check(admission.keys.sort == %w[env id name run shell] && + admission["id"] == "admit" && admission["shell"] == "bash" && + admission["env"] == { + "CALLER_REF" => "${{ github.ref }}", + "CALLER_REPOSITORY" => "${{ github.repository }}", + "CALLER_SHA" => "${{ github.sha }}", + "CALLER_WORKFLOW_REF" => "${{ github.workflow_ref }}", + "GH_TOKEN" => "${{ github.token }}", + "KANDELO_REPOSITORY" => "${{ inputs.kandelo-repository }}", + "PR_NUMBER" => "${{ inputs.pr-number }}", + "PRODUCER_SHA" => "${{ inputs.producer-sha }}", + "TAP_NAME" => "${{ inputs.tap-name }}", + "TAP_REPOSITORY" => "${{ inputs.tap-repository }}", + "TAP_SHA" => "${{ inputs.tap-sha }}", + }, "candidate campaign admission mapping changed") + admission_run = admission.fetch("run") + [ + 'candidate-campaign.yml@refs/heads/main', + '"kandelo-dev/homebrew-tap-core"', + '[ "$CALLER_REF" = refs/heads/main ]', + '[ "$CALLER_WORKFLOW_REF" = "$expected_caller" ]', + '[ "${KANDELO_REPOSITORY,,}" = automattic/kandelo ]', + '[ "${TAP_NAME,,}" = kandelo-dev/tap-core ]', + '[[ "$PRODUCER_SHA" =~ ^[0-9a-f]{40}$ ]]', + '[[ "$TAP_SHA" =~ ^[0-9a-f]{40}$ ]]', + '"/repos/$TAP_REPOSITORY/git/ref/heads/main"', + '[ "$CALLER_SHA" = "$tap_main" ]', + '"/repos/$TAP_REPOSITORY/compare/$TAP_SHA...$tap_main"', + 'gh api "/repos/$KANDELO_REPOSITORY/pulls/$PR_NUMBER"', + '.state == "open" and .base.ref == "main"', + '.base.sha == $base and .head.sha == $head', + ].each do |fragment| + check(admission_run.include?(fragment), + "candidate campaign admission lacks #{fragment}") + end + admit_checkouts = admit_steps.select do |step| + step["uses"] == STANDARD_CHECKOUT_ACTION + end.to_h { |step| [step.fetch("name"), step.fetch("with")] } + check(admit_checkouts == { + "Checkout protected campaign caller validator" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => "${{ steps.admit.outputs.base-sha }}", + "path" => "caller-validator", "submodules" => false, + }, + "Checkout exact campaign caller as inert data" => { + "persist-credentials" => false, + "repository" => "${{ inputs.tap-repository }}", + "ref" => "${{ steps.admit.outputs.caller-sha }}", + "path" => "caller-data", "submodules" => false, + }, + }, "candidate campaign caller evidence checkout changed") + caller_pin = named_step( + admit_steps, "Require the campaign caller to pin its exact base" + ) + check(caller_pin.fetch("env") == { + "BASE_SHA" => "${{ steps.admit.outputs.base-sha }}", + } && caller_pin.fetch("run").include?( + "homebrew-candidate-caller-pins.py" + ) && caller_pin.fetch("run").include?( + '--mode campaign --kandelo-sha "$BASE_SHA"' + ), "candidate campaign caller does not prove its immutable base pin") + + check(derive.keys.sort == + %w[needs permissions runs-on steps timeout-minutes] && + derive["needs"] == ["admit"] && + derive["runs-on"] == "ubuntu-latest" && + derive["timeout-minutes"] == 180 && + exact_permissions?(derive["permissions"], { "contents" => "read" }), + "candidate campaign derivation authority changed") + derive_steps = job_steps(derive, "candidate campaign derivation") + check(derive_steps.filter_map { |step| step["uses"] }.sort == [ + *Array.new(4, STANDARD_CHECKOUT_ACTION), + NIX_ACTION, MAGIC_NIX_ACTION, UPLOAD_ACTION, + ].sort, "candidate campaign derivation action set or pin changed") + derive_checkouts = derive_steps.select do |step| + step["uses"] == STANDARD_CHECKOUT_ACTION + end.to_h { |step| [step.fetch("name"), step.fetch("with")] } + check(derive_checkouts == { + "Checkout protected campaign wrapper authority" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => "${{ needs.admit.outputs.base-sha }}", + "path" => "authority", "submodules" => false, + }, + "Checkout exact candidate as untrusted source" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => "${{ inputs.producer-sha }}", + "path" => "producer", "submodules" => false, + }, + "Checkout exact tap campaign source" => { + "persist-credentials" => false, + "repository" => "${{ inputs.tap-repository }}", + "ref" => "${{ inputs.tap-sha }}", + "path" => "tap", "fetch-depth" => 0, + }, + "Checkout reviewed native Homebrew implementation" => { + "persist-credentials" => false, + "repository" => "Homebrew/brew", + "ref" => "${{ steps.source.outputs.native-commit }}", + "path" => "native-homebrew", + }, + }, "candidate campaign derivation checkout authority changed") + credential_names = %w[ + GH_TOKEN GITHUB_TOKEN HOMEBREW_GITHUB_API_TOKEN + HOMEBREW_GITHUB_PACKAGES_TOKEN HOMEBREW_DOCKER_REGISTRY_TOKEN + ] + check(derive_steps.none? do |step| + !(step.fetch("env", {}).keys & credential_names).empty? + end, "candidate code receives a GitHub or Homebrew credential") + source = named_step( + derive_steps, "Describe exact candidate inputs with protected code" + ) + source_run = source.fetch("run") + [ + 'for secret_name in GH_TOKEN GITHUB_TOKEN \\', + 'python3 authority/scripts/homebrew-candidate-campaign.py \\', + 'describe-source', '--kandelo-root "$GITHUB_WORKSPACE/producer"', + '--base-commit "$BASE_SHA"', '--producer-commit "$PRODUCER_SHA"', + '--workflow-authority-commit "$BASE_SHA"', + '--source-tap-commit "$TAP_SHA"', + '--tap-workflow-authority-commit "$CALLER_SHA"', + ].each do |fragment| + check(source_run.include?(fragment), + "candidate campaign source description lacks #{fragment}") + end + derivation = named_step( + derive_steps, "Derive the candidate campaign without credentials" + ) + derivation_run = derivation.fetch("run") + [ + 'for secret_name in GH_TOKEN GITHUB_TOKEN \\', + 'env -u GH_TOKEN -u GITHUB_TOKEN \\', + '-u HOMEBREW_GITHUB_PACKAGES_TOKEN \\', + 'bash producer/scripts/dev-shell.sh \\', + 'python3 producer/scripts/homebrew-prefix-campaign.py derive \\', + '--kandelo-root "$GITHUB_WORKSPACE/producer"', + '--old-tap-root "$GITHUB_WORKSPACE/tap"', + '--native-brew-root "$GITHUB_WORKSPACE/native-homebrew"', + '--metadata-sha256 "$metadata_sha"', + '--guest-layout-sha256 "$layout_sha"', + ].each do |fragment| + check(derivation_run.include?(fragment), + "candidate campaign credential-free derivation lacks #{fragment}") + end + check(values_for_key(workflow, "run").count do |run| + run.include?("producer/scripts/") + end == 1, "candidate source code executes outside its read-only derivation") + derivation_upload = named_step( + derive_steps, "Retain one exact credential-free derivation" + ) + check(derivation_upload["uses"] == UPLOAD_ACTION && + derivation_upload["with"] == { + "name" => + "homebrew-candidate-campaign-derivation-attempt-" \ + "${{ github.run_attempt }}", + "path" => + "${{ runner.temp }}/candidate-campaign-source.json\n" \ + "${{ runner.temp }}/campaign.json\n", + "compression-level" => 0, + "if-no-files-found" => "error", "retention-days" => 2, + }, "candidate campaign derivation artifact changed") + + check(seal.keys.sort == + %w[needs outputs permissions runs-on steps timeout-minutes] && + seal["needs"] == %w[admit derive] && + seal["runs-on"] == "ubuntu-latest" && + seal["timeout-minutes"] == 90 && + exact_permissions?(seal["permissions"], { + "actions" => "read", "contents" => "write", + }) && seal["outputs"] == { + "candidate-tag" => "${{ steps.publish.outputs.candidate-tag }}", + }, "candidate campaign sealer authority changed") + seal_steps = job_steps(seal, "candidate campaign sealer") + check(seal_steps.filter_map { |step| step["uses"] }.sort == [ + *Array.new(4, STANDARD_CHECKOUT_ACTION), DOWNLOAD_ACTION, UPLOAD_ACTION, + ].sort, "candidate campaign sealer action set or pin changed") + seal_checkouts = seal_steps.select do |step| + step["uses"] == STANDARD_CHECKOUT_ACTION + end.to_h { |step| [step.fetch("name"), step.fetch("with")] } + check(seal_checkouts == { + "Checkout protected Kandelo sealer authority" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => "${{ needs.admit.outputs.base-sha }}", + "path" => "authority", "submodules" => false, + }, + "Checkout protected tap release authority" => { + "persist-credentials" => false, + "ref" => "${{ needs.admit.outputs.caller-sha }}", + "fetch-depth" => 0, "path" => "tap-authority", + }, + "Checkout candidate only as inert source data" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => "${{ inputs.producer-sha }}", + "path" => "producer-data", "submodules" => false, + }, + "Checkout exact tap only as inert source data" => { + "persist-credentials" => false, + "repository" => "${{ inputs.tap-repository }}", + "ref" => "${{ inputs.tap-sha }}", + "path" => "tap-data", "fetch-depth" => 0, + }, + }, "candidate campaign sealer checkout authority changed") + check(values_for_key(seal, "run").none? do |run| + run.include?("producer/scripts/") + end, "candidate code executes in the credentialed campaign sealer") + sealer_credentials = seal_steps.select do |step| + !(step.fetch("env", {}).keys & credential_names).empty? + end + check(sealer_credentials.map { |step| step["name"] } == [ + "Revalidate run, source, and branch authority", + "Publish immutable candidate campaign", + ] && sealer_credentials.all? do |step| + step.fetch("env").slice(*credential_names) == { + "GH_TOKEN" => "${{ github.token }}", + } + end, "candidate campaign sealer credential boundary changed") + revalidate = named_step( + seal_steps, "Revalidate run, source, and branch authority" + ) + [ + '[ "$kandelo_main" = "$BASE_SHA" ]', + 'compare/$CALLER_SHA...$tap_main', + 'case "$caller_status" in', + 'ahead|identical)', + 'candidate caller left protected tap main', + '.state == "open" and .base.ref == "main"', + '.base.sha == $base and .head.sha == $head', + '.base_commit == $base', '.producer_commit == $producer', + '.tap_workflow_authority_commit == $caller', + 'env -u GH_TOKEN -u GITHUB_TOKEN \\', + 'python3 authority/scripts/homebrew-candidate-campaign.py \\', + 'describe-source', + '--kandelo-root "$GITHUB_WORKSPACE/producer-data"', + '--tap-root "$GITHUB_WORKSPACE/tap-data"', + 'cmp -s "$source" \\', + 'candidate execution changed protected source evidence', + 'actions/runs/$GITHUB_RUN_ID', + 'actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100', + '.id == $run_id and .run_attempt == $attempt', + '.head_sha == $caller', + '.path == ".github/workflows/candidate-campaign.yml"', + '(.repository.full_name | ascii_downcase)', + '.event == "repository_dispatch"', + '($selected | length) == 1', + 'test("^sha256:[0-9a-f]{64}$")', + 'workflow_path:".github/workflows/candidate-campaign.yml"', + ].each do |fragment| + check(revalidate.fetch("run").include?(fragment), + "candidate campaign sealer revalidation lacks #{fragment}") + end + preparation = named_step( + seal_steps, "Prepare inert noncanonical campaign release" + ) + check((preparation.fetch("env", {}).keys & credential_names).empty? && + preparation.fetch("run").include?( + '[ -z "${GH_TOKEN:-}" ] && [ -z "${GITHUB_TOKEN:-}" ]' + ) && preparation.fetch("run").include?( + "python3 authority/scripts/homebrew-candidate-campaign.py prepare" + ), "candidate campaign preparation is not inert protected code") + publish = named_step(seal_steps, "Publish immutable candidate campaign") + [ + 'bash authority/scripts/publish-immutable-github-release.sh \\', + '--lock-root "$GITHUB_WORKSPACE/tap-authority"', + '--exact-kandelo-main-sha "$BASE_SHA"', + '--target-main-contains-sha "$CALLER_SHA"', + 'This release cannot publish canonical bottles before merge.', + ].each do |fragment| + check(publish.fetch("run").include?(fragment), + "candidate campaign publication lacks #{fragment}") + end + download = named_step( + seal_steps, "Download exact credential-free derivation" + ) + check(download["uses"] == DOWNLOAD_ACTION && download["with"] == { + "name" => + "homebrew-candidate-campaign-derivation-attempt-" \ + "${{ github.run_attempt }}", + "path" => "${{ runner.temp }}/candidate-campaign-derivation", + }, "candidate campaign derivation download changed") + retained = named_step( + seal_steps, "Retain candidate campaign publication evidence" + ) + check(retained["uses"] == UPLOAD_ACTION && retained["with"] == { + "name" => + "homebrew-candidate-campaign-release-attempt-" \ + "${{ github.run_attempt }}", + "path" => "${{ runner.temp }}/candidate-campaign-release.json", + "if-no-files-found" => "error", "retention-days" => 90, + }, "candidate campaign publication evidence changed") + + tool = File.read( + File.join(REPO_ROOT, "scripts/homebrew-candidate-campaign.py") + ) + [ + 'homebrew-prefix-campaign-candidate-pr-', + 'if parents != f"{source[\'base_commit\']} {source[\'producer_commit\']}":', + 'if producer_tree != source["producer_tree"] or merge_tree != producer_tree:', + 'recorded_probe_dependencies(campaign_module, campaign)', + 'regenerated = campaign_module.derive_campaign(', + 'if campaign_module.pretty_json(regenerated) != campaign_payload:', + 'fail("protected main regenerated a different candidate campaign")', + 'def validate_admission(arguments: argparse.Namespace) -> None:', + ].each do |fragment| + check(tool.include?(fragment), + "candidate campaign trusted tool lacks #{fragment}") + end + check(contract_digest(workflow) == CANDIDATE_CAMPAIGN_DIGEST, + "candidate campaign workflow contract changed") +end + +def check_candidate_materializer(workflow) + top_keys = workflow.keys.map { |key| key == true ? "on" : key.to_s }.sort + check(top_keys == %w[jobs name on], + "candidate materializer has unexpected top-level configuration") + check(workflow["name"] == + "Re-materialize an exact merged Homebrew bottle candidate", + "candidate materializer name changed") + events = workflow_events(workflow) + check(events == { + "workflow_call" => { + "inputs" => { + "candidate-tag" => { "type" => "string", "required" => true }, + "formula" => { "type" => "string", "required" => true }, + "producer-sha" => { "type" => "string", "required" => true }, + "merge-commit" => { "type" => "string", "required" => true }, + }, + "outputs" => { + "abi" => { "value" => "${{ jobs.materialize.outputs.abi }}" }, + "campaign-tag" => { + "value" => "${{ jobs.materialize.outputs.campaign-tag }}", + }, + "formula" => { + "value" => "${{ jobs.materialize.outputs.formula }}", + }, + "release-tag" => { + "value" => "${{ jobs.materialize.outputs.release-tag }}", + }, + "source-tap-commit" => { + "value" => "${{ jobs.materialize.outputs.source-tap-commit }}", + }, + }, + }, + }, "candidate materializer call contract changed") + check(!workflow.key?("permissions"), + "candidate materializer requests workflow-wide permissions") + check_common(workflow, "candidate materializer") + + jobs = workflow_jobs(workflow) + check(jobs.keys == ["materialize"], + "candidate materializer has an unexpected job set") + job = jobs.fetch("materialize") + check(job.keys.sort == + %w[outputs permissions runs-on steps timeout-minutes], + "candidate materializer job contract changed") + check(job["runs-on"] == "ubuntu-latest" && + job["timeout-minutes"] == 120 && + exact_permissions?(job["permissions"], { + "actions" => "read", "contents" => "read", + }), "candidate materializer authority changed") + check(job["outputs"] == { + "abi" => "${{ steps.candidate.outputs.abi }}", + "campaign-tag" => "${{ steps.candidate.outputs.campaign-tag }}", + "formula" => "${{ steps.candidate.outputs.formula }}", + "release-tag" => "${{ steps.candidate.outputs.release-tag }}", + "source-tap-commit" => + "${{ steps.candidate.outputs.source-tap-commit }}", + }, "candidate materializer output authority changed") + + steps = job_steps(job, "candidate materializer") + expected_uses = [ + *Array.new(5, STANDARD_CHECKOUT_ACTION), + NIX_ACTION, MAGIC_NIX_ACTION, + *Array.new(2, DOWNLOAD_ACTION), + *Array.new(3, UPLOAD_ACTION), + ].sort + check(steps.filter_map { |step| step["uses"] }.sort == expected_uses, + "candidate materializer action set or pin changed") + checkout_views = steps.select do |step| + step["uses"] == STANDARD_CHECKOUT_ACTION + end.to_h { |step| [step.fetch("name"), step.fetch("with")] } + check(checkout_views == { + "Checkout exact merged Kandelo authority" => { + "persist-credentials" => false, + "repository" => "Automattic/kandelo", + "ref" => "${{ inputs.merge-commit }}", + "path" => "kandelo-main", "fetch-depth" => 0, + "submodules" => false, + }, + "Checkout exact bottle producer as inert data" => { + "persist-credentials" => false, + "repository" => "Automattic/kandelo", + "ref" => "${{ inputs.producer-sha }}", + "path" => "producer", "fetch-depth" => 0, + "submodules" => false, + }, + "Checkout protected tap authority" => { + "persist-credentials" => false, + "ref" => "${{ github.sha }}", + "fetch-depth" => 0, "path" => "tap-authority", + }, + "Checkout exact candidate tap source" => { + "persist-credentials" => false, + "repository" => "${{ github.repository }}", + "ref" => "${{ steps.candidate.outputs.source-tap-commit }}", + "path" => "tap-source", "fetch-depth" => 0, + }, + "Checkout exact native Homebrew campaign input" => { + "persist-credentials" => false, + "repository" => "Homebrew/brew", + "ref" => "${{ steps.candidate.outputs.native-homebrew-commit }}", + "path" => "native-homebrew", + }, + }, "candidate materializer checkout authority changed") + + admit = named_step(steps, "Admit the protected tap promotion caller") + check(steps.first.equal?(admit) && admit.keys.sort == %w[env name run shell] && + admit["shell"] == "bash" && admit["env"] == { + "CANDIDATE_TAG" => "${{ inputs.candidate-tag }}", + "CALLER_REF" => "${{ github.ref }}", + "CALLER_REPOSITORY" => "${{ github.repository }}", + "CALLER_WORKFLOW_REF" => "${{ github.workflow_ref }}", + "FORMULA" => "${{ inputs.formula }}", + "MERGE_COMMIT" => "${{ inputs.merge-commit }}", + "PRODUCER_SHA" => "${{ inputs.producer-sha }}", + }, "candidate materializer caller admission mapping changed") + [ + 'promote-candidate-bottle.yml@refs/heads/main', + '[ "$CALLER_REPOSITORY" = \\', + 'kandelo-dev/homebrew-tap-core', + '[ "$CALLER_REF" = refs/heads/main ]', + '[ "$CALLER_WORKFLOW_REF" = "$expected_caller" ]', + '[[ "$FORMULA" =~ ^[a-z0-9][a-z0-9._-]{0,254}$ ]]', + '[[ "$PRODUCER_SHA" =~ ^[0-9a-f]{40}$ ]]', + '[[ "$MERGE_COMMIT" =~ ^[0-9a-f]{40}$ ]]', + '^homebrew-bottle-candidate-pr-[1-9][0-9]*-run-', + ].each do |fragment| + check(admit.fetch("run").include?(fragment), + "candidate materializer caller admission lacks #{fragment}") + end + + credential_names = %w[ + GH_TOKEN GITHUB_TOKEN HOMEBREW_GITHUB_API_TOKEN + HOMEBREW_GITHUB_PACKAGES_TOKEN HOMEBREW_DOCKER_REGISTRY_TOKEN + ] + credential_steps = steps.select do |step| + !(step.fetch("env", {}).keys & credential_names).empty? + end + check(credential_steps.map { |step| step["name"] } == [ + "Authenticate candidate runs and locate sealer receipts", + "Regenerate and admit exact package input on the merge tree", + "Admit the exact merge with protected-main code", + ] && credential_steps.all? do |step| + step.fetch("env").slice(*credential_names) == { + "GH_TOKEN" => "${{ github.token }}", + } + end, "candidate materializer read token escapes reviewed admission steps") + + describe = named_step( + steps, "Authenticate candidate runs and locate sealer receipts" + ) + check(describe["id"] == "candidate" && describe["env"] == { + "CANDIDATE_TAG" => "${{ inputs.candidate-tag }}", + "EXPECTED_FORMULA" => "${{ inputs.formula }}", + "EXPECTED_PRODUCER" => "${{ inputs.producer-sha }}", + "GH_TOKEN" => "${{ github.token }}", + "TAP_REPOSITORY" => "${{ github.repository }}", + }, "candidate materializer immutable release mapping changed") + describe_run = describe.fetch("run") + [ + 'releases/tags/$CANDIDATE_TAG', + '.draft == false', '.prerelease == false', '.immutable == true', + '.name == "candidate.json"', + 'curl --disable --fail --location --silent --show-error', + 'env -u GH_TOKEN -u GITHUB_TOKEN \\', + '-u ACTIONS_ID_TOKEN_REQUEST_TOKEN', + '-u ACTIONS_RUNTIME_TOKEN', + 'python3 kandelo-main/scripts/homebrew-bottle-candidate.py \\', + 'describe-release', '.manifest.formula.arch == "wasm32"', + '.manifest.dependencies == []', + '.manifest.source.producer_commit == $producer', + '.manifest.run.caller_commit ==', + 'homebrew-candidate-campaign.py', + 'fetch-release', + 'candidate-prefix-campaign.json', + 'candidate-campaign-manifest.json', + '.source.producer_commit ==', + '.source.pr_number ==', + '.source.source_tap_commit ==', + '.source.abi ==', + '.source.guest_layout.sha256 ==', + 'native-homebrew-commit=', + 'select(.path == "package-input.json")', + 'actions/runs/$run_id/attempts/$attempt"', + 'actions/runs/$run_id/attempts/$attempt/jobs?per_page=100', + '.status == "completed" and .conclusion == "success"', + 'homebrew-candidate-release-receipt-', + 'homebrew-candidate-campaign-release-', + 'workflow_run.id == $run_id', + 'workflow_run.head_sha == $caller', + ].each do |fragment| + check(describe_run.include?(fragment), + "candidate materializer immutable release admission lacks #{fragment}") + end + check(describe_run.scan("<= 16777216").length == 2, + "candidate materializer no longer bounds manifest or receipt bytes") + + promotion_pin = named_step( + steps, "Require the promotion caller to pin the exact merge" + ) + check(promotion_pin.keys.sort == %w[env name run shell] && + promotion_pin["env"] == { + "MERGE_COMMIT" => "${{ inputs.merge-commit }}", + } && promotion_pin.fetch("run").include?( + "homebrew-candidate-caller-pins.py" + ) && promotion_pin.fetch("run").include?( + '--mode promotion --kandelo-sha "$MERGE_COMMIT"' + ), "candidate materializer does not prove its immutable merge pin") + + { + "Download exact bottle sealer receipt" => { + "artifact-ids" => + "${{ steps.candidate.outputs.bottle-receipt-artifact-id }}", + "github-token" => "${{ github.token }}", + "repository" => "${{ github.repository }}", + "run-id" => "${{ steps.candidate.outputs.bottle-run-id }}", + "path" => "${{ runner.temp }}/bottle-sealer-receipt", + }, + "Download exact campaign sealer receipt" => { + "artifact-ids" => + "${{ steps.candidate.outputs.campaign-receipt-artifact-id }}", + "github-token" => "${{ github.token }}", + "repository" => "${{ github.repository }}", + "run-id" => "${{ steps.candidate.outputs.campaign-run-id }}", + "path" => "${{ runner.temp }}/campaign-sealer-receipt", + }, + }.each do |name, with| + download = named_step(steps, name) + check(download["uses"] == DOWNLOAD_ACTION && download["with"] == with, + "candidate materializer #{name} contract changed") + end + + bind_receipts = named_step( + steps, "Bind public candidate bytes to protected sealer receipts" + ) + check((bind_receipts.fetch("env", {}).keys & credential_names).empty?, + "candidate receipt validation received a credential") + [ + 'homebrew-candidate-release-receipt.py \\', 'plan \\', + 'candidate-release-receipt.json', 'candidate-campaign-release.json', + '--release-assets "$RUNNER_TEMP/candidate-release-assets.json"', + '--release-assets \\', '--target-commit "$bottle_target"', + '--target-commit "$campaign_target"', + 'curl --disable --fail --location --silent --show-error', + 'verify-readback \\', + '--asset-root "$RUNNER_TEMP/homebrew-candidate-release"', + '--asset-root "$RUNNER_TEMP/homebrew-candidate-campaign-release"', + ].each do |fragment| + check(bind_receipts.fetch("run").include?(fragment), + "candidate receipt validation lacks #{fragment}") + end + + package = named_step( + steps, "Regenerate and admit exact package input on the merge tree" + ) + package_run = package.fetch("run") + [ + 'cd kandelo-main', + 'materialize-homebrew-candidate-package-input.sh', + '--producer-sha "$PRODUCER_SHA"', + '--expected-abi "$EXPECTED_ABI"', + '--consumer-sha "$MERGE_COMMIT"', + 'env -u GH_TOKEN -u GITHUB_TOKEN \\', + 'python3 scripts/homebrew-bottle-candidate.py \\', + 'admit-package-input', + '--validated-main "$MERGE_COMMIT"', + ].each do |fragment| + check(package_run.include?(fragment), + "candidate package re-materialization lacks #{fragment}") + end + + live = named_step(steps, "Admit the exact merge with protected-main code") + check(live["env"] == { + "CANDIDATE_TAG" => "${{ inputs.candidate-tag }}", + "GH_TOKEN" => "${{ github.token }}", + "MERGE_COMMIT" => "${{ inputs.merge-commit }}", + "TAP_REPOSITORY" => "${{ github.repository }}", + }, "candidate live promotion evidence mapping changed") + live_run = live.fetch("run") + [ + 'completed-candidate-campaign-run.json', + '/repos/Automattic/kandelo/compare/$MERGE_COMMIT...$current_kandelo_main', + 'ahead|identical)', + 'env -u GH_TOKEN -u GITHUB_TOKEN \\', + 'homebrew-candidate-campaign.py \\', + 'admit \\', + '--candidate "$campaign_manifest"', + '--campaign "$RUNNER_TEMP/candidate-prefix-campaign.json"', + '--completed-run-evidence \\', + 'completed-candidate-campaign-run.json', + '--native-brew-root "$GITHUB_WORKSPACE/native-homebrew"', + '--out "$RUNNER_TEMP/candidate-campaign-admission.json"', + 'python3 kandelo-main/scripts/homebrew-bottle-candidate.py \\', + 'materialize', '--candidate-tag "$CANDIDATE_TAG"', + '--merge-commit "$MERGE_COMMIT"', + '--current-kandelo-main "$current_kandelo_main"', + '--current-tap-main "$current_tap_main"', + '--out-package-input \\', + '--out-receipt "$RUNNER_TEMP/candidate-promotion.json"', + ].each do |fragment| + check(live_run.include?(fragment), + "candidate live promotion evidence lacks #{fragment}") + end + campaign_admission = live_run.index("homebrew-candidate-campaign.py") + bottle_materialization = live_run.index( + 'python3 kandelo-main/scripts/homebrew-bottle-candidate.py' + ) + check(campaign_admission && bottle_materialization && + campaign_admission < bottle_materialization, + "candidate bottle materialization precedes campaign admission") + + candidate_tool = File.read( + File.join(REPO_ROOT, "scripts/homebrew-bottle-candidate.py") + ) + [ + 'value["workflow_authority_commit"] != value["base_commit"]', + '(package_input_path, "package-input.json", "package-input.json")', + '"package_input_sha256": package_record["sha256"]', + 'value["package_input_sha256"] != sha256_bytes(package_payload)', + '"package-input.json": package_path', + 'if set(current_files) != set(by_path):', + ].each do |fragment| + check(candidate_tool.include?(fragment), + "candidate materializer trusted tool lacks #{fragment}") + end + check(!candidate_tool.include?("headRefOid") && + !candidate_tool.include?("--pr-json"), + "candidate materializer depends on mutable pull-request state") + + receipt_tool = File.read( + File.join(REPO_ROOT, "scripts/homebrew-candidate-release-receipt.py") + ) + [ + '"assets",', '"immutable",', '"release_id",', '"target_commitish",', + 'value["visibility"] != "public-anonymous-readback"', + 'if set(by_name) != set(receipt_by_name):', + 'live["digest"] != f"sha256:{recorded[\'sha256\']}"', + 'if actual != expected:', + 'sha256_file(path) != asset["sha256"]', + ].each do |fragment| + check(receipt_tool.include?(fragment), + "candidate release receipt validator lacks #{fragment}") + end + + caller_pin_tool = File.read( + File.join(REPO_ROOT, "scripts/homebrew-candidate-caller-pins.py") + ) + [ + 'BASE_TOKEN = "__KANDELO_CANDIDATE_BASE_SHA__"', + 'MERGE_TOKEN = "__KANDELO_CANDIDATE_MERGE_SHA__"', + 'if found != wanted:', + 'if "@main" in text or BASE_TOKEN in text or MERGE_TOKEN in text:', + ].each do |fragment| + check(caller_pin_tool.include?(fragment), + "candidate caller pin validator lacks #{fragment}") + end + + probe = named_step(steps, "Re-probe the collision-sensitive child reference") + check((probe.fetch("env").keys & credential_names).empty? && + probe.fetch("run").include?( + 'env -u GH_TOKEN -u GITHUB_TOKEN \\' + ) && + probe.fetch("run").include?( + '(.status == "missing" and .digest == null) or' + ) && + probe.fetch("run").include?( + '(.status == "present" and .digest == $digest)' + ), "candidate destination re-probe is not anonymous and immutable") + + expected_artifacts = { + "Upload exact build handoff for the publisher" => { + "name" => + "homebrew-build-handoff-${{ steps.candidate.outputs.formula }}-" \ + "wasm32-attempt-${{ github.run_attempt }}", + "path" => "${{ runner.temp }}/homebrew-build-handoff", + "compression-level" => 0, + "if-no-files-found" => "error", "retention-days" => 2, + }, + "Upload exact OCI child for the publisher" => { + "name" => + "homebrew-oci-child-${{ steps.candidate.outputs.formula }}-" \ + "wasm32-attempt-${{ github.run_attempt }}", + "path" => "${{ runner.temp }}/homebrew-oci-child", + "compression-level" => 0, + "if-no-files-found" => "error", "retention-days" => 2, + }, + "Retain bounded promotion evidence" => { + "name" => + "homebrew-candidate-promotion-${{ steps.candidate.outputs.formula }}-" \ + "wasm32-attempt-${{ github.run_attempt }}", + "path" => "${{ runner.temp }}/candidate-promotion.json\n" \ + "${{ runner.temp }}/candidate-campaign-admission.json\n" \ + "${{ runner.temp }}/candidate-package-input.json\n" \ + "${{ runner.temp }}/live-child-probe.json\n", + "compression-level" => 0, + "if-no-files-found" => "error", "retention-days" => 90, + }, + } + expected_artifacts.each do |name, with| + step = named_step(steps, name) + check(step["uses"] == UPLOAD_ACTION && step["with"] == with, + "candidate materializer #{name} artifact contract changed") + end + check(contract_digest(steps) == CANDIDATE_MATERIALIZER_STEPS_DIGEST, + "candidate materializer step contract changed") +end + +def check_candidate_promotion_caller(path) + workflow = load_workflow(path) + top_keys = workflow.keys.map { |key| key == true ? "on" : key.to_s }.sort + check(top_keys == %w[jobs name on], + "candidate promotion caller has unexpected top-level configuration") + check(workflow["name"] == "Promote exact merged Kandelo bottle candidate", + "candidate promotion caller name changed") + check(workflow_events(workflow) == { + "repository_dispatch" => { + "types" => ["promote-kandelo-bottle-candidate"], + }, + }, "candidate promotion caller event changed") + check(!workflow.key?("permissions") && + values_for_key(workflow, "secrets").empty?, + "candidate promotion caller has workflow-wide authority or secrets") + + jobs = workflow_jobs(workflow) + check(jobs.keys == %w[admit materialize publish seal-formula-handoff], + "candidate promotion caller job set changed") + admit = jobs.fetch("admit") + materialize = jobs.fetch("materialize") + publish = jobs.fetch("publish") + seal = jobs.fetch("seal-formula-handoff") + check(admit.keys.sort == + %w[outputs permissions runs-on steps timeout-minutes] && + admit["runs-on"] == "ubuntu-latest" && + admit["timeout-minutes"] == 10 && + exact_permissions?(admit["permissions"], { "contents" => "read" }), + "candidate promotion admission authority changed") + check(admit["outputs"] == { + "candidate-tag" => "${{ steps.admit.outputs.candidate-tag }}", + "formula" => "${{ steps.admit.outputs.formula }}", + "merge-commit" => "${{ steps.admit.outputs.merge-commit }}", + "producer-sha" => "${{ steps.admit.outputs.producer-sha }}", + "rootfs-generation" => + "${{ steps.admit.outputs.rootfs-generation }}", + }, "candidate promotion admission outputs changed") + admit_steps = job_steps(admit, "candidate promotion admission") + check(admit_steps.length == 1, + "candidate promotion admission executes extra steps") + admission = named_step(admit_steps, "Admit one exact promotion request") + check(admission.keys.sort == %w[env id name run shell] && + admission["id"] == "admit" && admission["shell"] == "bash" && + admission["env"] == { + "GH_TOKEN" => "${{ github.token }}", + "TAP_REPOSITORY" => "${{ github.repository }}", + "TAP_SHA" => "${{ github.sha }}", + }, "candidate promotion dispatch admission mapping changed") + admission_run = admission.fetch("run") + [ + 'keys == [', '"candidate_tag"', '"formula"', '"merge_commit"', + '"producer_sha"', '"rootfs_generation"', + '^homebrew-bottle-candidate-pr-[1-9][0-9]*-run-', + 'test("^[0-9a-f]{40}$")', + '^package-generation-rootfs-wasm32-abi-v[1-9][0-9]*-sha256-', + '"/repos/$TAP_REPOSITORY/git/ref/heads/main"', + '[ "$TAP_SHA" = "$current_main" ]', + ].each do |fragment| + check(admission_run.include?(fragment), + "candidate promotion dispatch admission lacks #{fragment}") + end + + check(materialize.keys.sort == %w[needs permissions uses with] && + materialize["needs"] == ["admit"] && + exact_permissions?(materialize["permissions"], { + "actions" => "read", "contents" => "read", + }) && + materialize["uses"] == + "Automattic/kandelo/.github/workflows/" \ + "reusable-homebrew-bottle-candidate-materialize.yml@" \ + "__KANDELO_CANDIDATE_MERGE_SHA__" && + materialize["with"] == { + "candidate-tag" => "${{ needs.admit.outputs.candidate-tag }}", + "formula" => "${{ needs.admit.outputs.formula }}", + "producer-sha" => "${{ needs.admit.outputs.producer-sha }}", + "merge-commit" => "${{ needs.admit.outputs.merge-commit }}", + }, "candidate promotion materializer call changed") + check(publish.keys.sort == %w[needs permissions uses with] && + publish["needs"] == %w[admit materialize] && + exact_permissions?(publish["permissions"], { + "actions" => "read", "contents" => "read", "packages" => "write", + }) && + publish["uses"] == + "Automattic/kandelo/.github/workflows/" \ + "reusable-homebrew-bottle-publish.yml@" \ + "__KANDELO_CANDIDATE_MERGE_SHA__" && + publish["with"] == { + "kandelo-repository" => "Automattic/kandelo", + "kandelo-ref" => "${{ needs.admit.outputs.merge-commit }}", + "tap-repository" => "kandelo-dev/homebrew-tap-core", + "tap-name" => "kandelo-dev/tap-core", + "tap-ref" => + "${{ needs.materialize.outputs.source-tap-commit }}", + "formulae" => "${{ needs.admit.outputs.formula }}", + "arches" => "wasm32", + "release-tag" => "${{ needs.materialize.outputs.release-tag }}", + "expected-cache-keys" => "", + "package-generation-wasm32" => + "${{ needs.admit.outputs.rootfs-generation }}", + "force" => true, "dry-run" => false, + "require-vfs-acceptance" => false, + "defer-tap-finalization" => true, + "prefix-campaign-tag" => + "${{ needs.materialize.outputs.campaign-tag }}", + "prefix-campaign-dependencies" => + '{"dependencies":[],"schema":1}', + "candidate-promotion-tag" => + "${{ needs.admit.outputs.candidate-tag }}", + "candidate-producer-sha" => + "${{ needs.admit.outputs.producer-sha }}", + }, "candidate promotion publisher call changed") + + check(seal.keys.sort == + %w[needs permissions runs-on steps timeout-minutes] && + seal["needs"] == %w[admit materialize publish] && + seal["runs-on"] == "ubuntu-latest" && + seal["timeout-minutes"] == 90 && + exact_permissions?(seal["permissions"], { + "actions" => "read", "contents" => "write", + }), "candidate promotion Formula handoff authority changed") + seal_steps = job_steps(seal, "candidate promotion Formula handoff") + expected_actions = [ + *Array.new(4, STANDARD_CHECKOUT_ACTION), + *Array.new(2, DOWNLOAD_ACTION), + NIX_ACTION, MAGIC_NIX_ACTION, UPLOAD_ACTION, + ].sort + check(seal_steps.filter_map { |step| step["uses"] }.sort == + expected_actions, + "candidate promotion Formula handoff action set or pin changed") + check(seal_steps.select do |step| + step["uses"] == STANDARD_CHECKOUT_ACTION + end.all? { |step| step.dig("with", "persist-credentials") == false }, + "candidate promotion Formula handoff persists checkout credentials") + check(values_for_key(seal, "run").none? do |source| + source.is_a?(String) && source.include?("${{") + end, "candidate promotion Formula handoff interpolates expressions into shell") + + handoff_download = named_step( + seal_steps, "Download the exact validated publication handoff" + ) + check(handoff_download["uses"] == DOWNLOAD_ACTION && + handoff_download["with"] == { + "name" => + "homebrew-publish-handoff-" \ + "${{ needs.admit.outputs.formula }}-wasm32-attempt-" \ + "${{ github.run_attempt }}", + "path" => "${{ runner.temp }}/candidate-publication", + }, "candidate promotion publication handoff download changed") + campaign_admission_download = named_step( + seal_steps, "Download exact candidate campaign admission" + ) + check(campaign_admission_download["uses"] == DOWNLOAD_ACTION && + campaign_admission_download["with"] == { + "name" => + "homebrew-candidate-promotion-" \ + "${{ needs.admit.outputs.formula }}-wasm32-attempt-" \ + "${{ github.run_attempt }}", + "path" => "${{ runner.temp }}/candidate-promotion", + }, "candidate promotion campaign admission download changed") + derive = named_step( + seal_steps, "Derive the immutable campaign Formula handoff" + ) + check(derive.fetch("env") == { + "ABI" => "${{ needs.materialize.outputs.abi }}", + "CAMPAIGN_TAG" => "${{ needs.materialize.outputs.campaign-tag }}", + "FORMULA" => "${{ needs.admit.outputs.formula }}", + "MERGE_COMMIT" => "${{ needs.admit.outputs.merge-commit }}", + "PRODUCER_SHA" => "${{ needs.admit.outputs.producer-sha }}", + "TAP_SHA" => "${{ needs.materialize.outputs.source-tap-commit }}", + } && (derive.fetch("env").keys & %w[GH_TOKEN GITHUB_TOKEN]).empty?, + "candidate promotion handoff derivation received credentials") + [ + 'python3 kandelo/scripts/homebrew-candidate-campaign.py \\', + 'fetch-release', + 'validate-admission', + 'candidate-campaign-admission.json', + '--merge-commit "$MERGE_COMMIT"', + '--abi "$ABI"', + 'python3 kandelo/scripts/homebrew-prefix-campaign-publisher.py \\', + '--kandelo-root "$GITHUB_WORKSPACE/producer"', + '--kandelo-commit "$PRODUCER_SHA"', + '--dependencies \'{"dependencies":[],"schema":1}\'', + 'derive-build --campaign "$campaign"', + '"wasm32=$RUNNER_TEMP/candidate-publication"', + 'prepare-release --campaign "$campaign"', + ].each do |fragment| + check(derive.fetch("run").include?(fragment), + "candidate promotion Formula handoff derivation lacks #{fragment}") + end + admission_validation = derive.fetch("run").index("validate-admission") + formula_derivation = derive.fetch("run").index("derive-build") + check(admission_validation && formula_derivation && + admission_validation < formula_derivation, + "candidate Formula derivation precedes campaign admission") + write = named_step( + seal_steps, "Publish the now-main-reachable Formula handoff" + ) + check(write.fetch("env").slice("GH_TOKEN") == { + "GH_TOKEN" => "${{ github.token }}", + }, "candidate promotion Formula handoff credential changed") + [ + 'bash kandelo/scripts/publish-immutable-github-release.sh \\', + '--lock-root "$GITHUB_WORKSPACE/tap-authority"', + '--kandelo-main-contains-sha "$PRODUCER_SHA"', + '--target-main-contains-sha "$SOURCE_TAP_SHA"', + ].each do |fragment| + check(write.fetch("run").include?(fragment), + "candidate promotion Formula handoff write lacks #{fragment}") + end + retained = named_step(seal_steps, "Retain handoff publication evidence") + check(retained["uses"] == UPLOAD_ACTION && retained["with"] == { + "name" => + "promoted-formula-handoff-${{ needs.admit.outputs.formula }}-" \ + "wasm32-attempt-${{ github.run_attempt }}", + "path" => "${{ runner.temp }}/formula-handoff-release.json", + "if-no-files-found" => "error", "retention-days" => 90, + }, "candidate promotion Formula handoff evidence changed") + check(contract_digest(workflow) == TAP_CANDIDATE_PROMOTION_DIGEST, + "candidate promotion caller contract changed") +end + def check_tap_callers common_publish_inputs = { "kandelo-repository" => "Automattic/kandelo", @@ -888,10 +2077,59 @@ def check_tap_callers expected_name: "Publish Kandelo bottles", event_type: "publish-kandelo-bottles", job_name: "publish", - reusable: "Automattic/kandelo/.github/workflows/reusable-homebrew-bottle-publish.yml@main", + reusable: "Automattic/kandelo/.github/workflows/" \ + "reusable-homebrew-bottle-publish.yml@main", inputs: write_publish_inputs, ) + check_tap_caller( + File.join(TAP_CALLER_ROOT, "candidate-bottles.yml"), + expected_name: "Build Kandelo bottle candidate", + event_type: "build-kandelo-bottle-candidate", + job_name: "candidate", + reusable: "Automattic/kandelo/.github/workflows/" \ + "reusable-homebrew-bottle-publish.yml@" \ + "__KANDELO_CANDIDATE_BASE_SHA__", + permissions: { "actions" => "read", "contents" => "write" }, + inputs: { + "kandelo-repository" => "Automattic/kandelo", + "kandelo-ref" => "${{ github.event.client_payload.kandelo_ref }}", + "tap-repository" => "kandelo-dev/homebrew-tap-core", + "tap-name" => "kandelo-dev/tap-core", + "tap-ref" => "${{ github.event.client_payload.tap_ref }}", + "formulae" => "${{ github.event.client_payload.formula }}", + "arches" => "${{ github.event.client_payload.arch }}", + "force" => true, + "dry-run" => true, + "candidate-pr-number" => "${{ github.event.client_payload.pr_number }}", + "candidate-package-staging-tag" => + "${{ github.event.client_payload.package_staging_tag }}", + "prefix-campaign-tag" => + "${{ github.event.client_payload.prefix_campaign_tag }}", + "prefix-campaign-dependencies" => + "${{ github.event.client_payload.prefix_campaign_dependencies }}", + }, + ) + + check_tap_caller( + File.join(TAP_CALLER_ROOT, "candidate-campaign.yml"), + expected_name: "Prepare Kandelo candidate bottle campaign", + event_type: "prepare-kandelo-candidate-campaign", + job_name: "campaign", + reusable: "Automattic/kandelo/.github/workflows/" \ + "reusable-homebrew-candidate-campaign.yml@" \ + "__KANDELO_CANDIDATE_BASE_SHA__", + permissions: { "actions" => "read", "contents" => "write" }, + inputs: { + "kandelo-repository" => "Automattic/kandelo", + "producer-sha" => "${{ github.event.client_payload.kandelo_ref }}", + "pr-number" => "${{ github.event.client_payload.pr_number }}", + "tap-repository" => "kandelo-dev/homebrew-tap-core", + "tap-name" => "kandelo-dev/tap-core", + "tap-sha" => "${{ github.event.client_payload.tap_ref }}", + }, + ) + check_tap_caller( File.join(TAP_CALLER_ROOT, "dry-run-bottles.yml"), expected_name: "Dry run Kandelo bottles", @@ -937,6 +2175,10 @@ def check_tap_callers "deletion-reason" => "${{ github.event.client_payload.deletion_reason || '' }}", }, ) + + check_candidate_promotion_caller( + File.join(TAP_CALLER_ROOT, "promote-candidate-bottle.yml") + ) end def check_native_compatibility_workflow(workflow) @@ -957,7 +2199,9 @@ def check_native_compatibility_workflow(workflow) "pull_request" => { "paths" => [ ".github/workflows/homebrew-native-publisher-compatibility.yml", + ".github/workflows/reusable-homebrew-bottle-candidate-materialize.yml", ".github/workflows/reusable-homebrew-bottle-publish.yml", + ".github/workflows/reusable-homebrew-candidate-campaign.yml", ".github/workflows/reusable-homebrew-prefix-first-child-publish.yml", "flake.lock", "flake.nix", @@ -1738,6 +2982,10 @@ def check_publisher(workflow) "defer-tap-finalization" => { "type" => "boolean", "default" => false }, "prefix-campaign-tag" => { "type" => "string", "default" => "" }, "prefix-campaign-dependencies" => { "type" => "string", "default" => "" }, + "candidate-pr-number" => { "type" => "string", "default" => "" }, + "candidate-package-staging-tag" => { "type" => "string", "default" => "" }, + "candidate-promotion-tag" => { "type" => "string", "default" => "" }, + "candidate-producer-sha" => { "type" => "string", "default" => "" }, }, "publisher inputs changed") check(!workflow.key?("permissions"), "publisher requests workflow-wide permissions") check_common(workflow, "reusable publisher") @@ -1746,7 +2994,7 @@ def check_publisher(workflow) "publisher still accepts a caller secret") jobs = workflow_jobs(workflow) - check(jobs.keys.sort == %w[build-and-test finalize-tap plan publish-bottle-index publish-vfs-release upload-bottle verify-bottle], + check(jobs.keys.sort == %w[build-and-test finalize-tap plan publish-bottle-index publish-vfs-release seal-bottle-candidate upload-bottle verify-bottle], "publisher has an unexpected job set") plan = jobs.fetch("plan") build = jobs.fetch("build-and-test") @@ -1755,6 +3003,7 @@ def check_publisher(workflow) verify = jobs.fetch("verify-bottle") finalize = jobs.fetch("finalize-tap") vfs_release = jobs.fetch("publish-vfs-release") + candidate = jobs.fetch("seal-bottle-candidate") check(plan.keys.sort == %w[outputs permissions runs-on steps], "publisher plan contract changed") @@ -1772,6 +3021,13 @@ def check_publisher(workflow) "publisher version-index job contract changed") check(vfs_release.keys.sort == %w[if needs permissions runs-on steps timeout-minutes], "publisher VFS release job contract changed") + check(candidate.keys.sort == %w[if needs permissions runs-on steps timeout-minutes], + "publisher candidate sealer job contract changed") + check(candidate["runs-on"] == "ubuntu-latest" && + candidate["timeout-minutes"] == 90 && + exact_permissions?(candidate["permissions"], { + "actions" => "read", "contents" => "write", + }), "publisher candidate sealer authority changed") check(plan["runs-on"] == "ubuntu-latest" && exact_permissions?(plan["permissions"], { "contents" => "read" }), "publisher plan authority changed") @@ -1819,7 +3075,8 @@ def check_publisher(workflow) "matrix" => { "include" => "${{ fromJson(needs.plan.outputs.formula-matrix) }}" }, }, "publisher version-index job bypasses the validated Formula matrix") check(build["needs"] == ["plan"] && - build["if"] == "${{ needs.plan.outputs.matrix != '[]' }}", + build["if"] == "${{ needs.plan.outputs.matrix != '[]' && " \ + "needs.plan.outputs.candidate-promotion-mode != 'true' }}", "publisher build graph changed") check(upload["needs"] == %w[plan build-and-test] && upload["if"] == "${{ always() && !cancelled() && !inputs.dry-run && " \ @@ -1846,6 +3103,13 @@ def check_publisher(workflow) "needs.finalize-tap.result == 'success' && " \ "needs.plan.outputs.vfs-acceptance-formula != '' }}", "publisher VFS release graph or evidence gate changed") + check(candidate["needs"] == %w[plan build-and-test verify-bottle] && + candidate["if"] == "${{ always() && !cancelled() && " \ + "needs.plan.result == 'success' && " \ + "needs.plan.outputs.candidate-mode == 'true' && " \ + "needs.build-and-test.result == 'success' && " \ + "needs.verify-bottle.result == 'success' }}", + "publisher candidate sealer graph or read-only evidence gate changed") plan_steps = job_steps(plan, "publisher plan") build_steps = job_steps(build, "publisher build") @@ -1854,6 +3118,7 @@ def check_publisher(workflow) verify_steps = job_steps(verify, "publisher verification") finalize_steps = job_steps(finalize, "publisher finalization") vfs_release_steps = job_steps(vfs_release, "publisher VFS release") + candidate_steps = job_steps(candidate, "publisher candidate sealer") validation = named_step(plan_steps, "Validate caller trust boundary") check(plan_steps.first.equal?(validation), "publisher trust validation must be first") @@ -1863,6 +3128,7 @@ def check_publisher(workflow) "CALLER_EVENT_NAME" => "${{ github.event_name }}", "CALLER_REF" => "${{ github.ref }}", "CALLER_REPOSITORY" => "${{ github.repository }}", + "CALLER_SHA" => "${{ github.sha }}", "CALLER_WORKFLOW_REF" => "${{ github.workflow_ref }}", "DEFER_TAP_FINALIZATION" => "${{ inputs.defer-tap-finalization }}", "DRY_RUN" => "${{ inputs.dry-run }}", @@ -1882,6 +3148,13 @@ def check_publisher(workflow) "TAP_REPOSITORY" => "${{ inputs.tap-repository }}", "TAP_REF" => "${{ inputs.tap-ref }}", "BOTTLE_ROOT_URL" => "${{ inputs.bottle-root-url }}", + "CANDIDATE_PACKAGE_STAGING_TAG" => + "${{ inputs.candidate-package-staging-tag }}", + "CANDIDATE_PR_NUMBER" => "${{ inputs.candidate-pr-number }}", + "CANDIDATE_PROMOTION_TAG" => + "${{ inputs.candidate-promotion-tag }}", + "CANDIDATE_PRODUCER_SHA" => + "${{ inputs.candidate-producer-sha }}", }, "publisher caller validation mapping changed") validation_run = validation.fetch("run") [ @@ -1892,6 +3165,7 @@ def check_publisher(workflow) '[ "$CALLER_REF" = "refs/heads/main" ]', '[ "$CALLER_EVENT_NAME" = "repository_dispatch" ]', '"$CALLER_REPOSITORY/.github/workflows/dry-run-bottles.yml@refs/heads/main"', + '"$CALLER_REPOSITORY/.github/workflows/candidate-bottles.yml@refs/heads/main"', '"$CALLER_REPOSITORY/.github/workflows/publish-bottles.yml@refs/heads/main"', '"$CALLER_REPOSITORY/.github/workflows/maintain-bottles.yml@refs/heads/main"', '[ "$KANDELO_REPOSITORY" = "Automattic/kandelo" ]', @@ -1960,11 +3234,9 @@ def check_publisher(workflow) caller_index < dry_index && kandelo_index < dry_index && tap_name_index < dry_index, "publisher dry-run can bypass caller authority validation") check(dry_kandelo_ref_index && write_kandelo_ref_index && write_tap_ref_index && - dry_index < dry_kandelo_ref_index && dry_kandelo_ref_index < write_kandelo_ref_index && - dry_kandelo_ref_index < write_tap_ref_index, + dry_index < dry_kandelo_ref_index, "publisher does not separate selectable dry-run refs from reviewed write refs") check(write_generation_wasm32_index && write_generation_wasm64_index && - write_tap_ref_index < write_generation_wasm32_index && write_generation_wasm32_index < write_generation_wasm64_index, "publisher does not require both exact package generations on its write path") @@ -2272,8 +3544,24 @@ def check_publisher(workflow) "${{ steps.trust.outputs.prefix-campaign-dependencies }}", "prefix-campaign-layout-sha256" => "${{ steps.campaign-source.outputs.prefix-campaign-layout-sha256 }}", + "prefix-campaign-prepared-tap-commit" => + "${{ steps.campaign-source.outputs.prefix-campaign-prepared-tap-commit }}", + "prefix-campaign-prepared-tap-tree" => + "${{ steps.campaign-source.outputs.prefix-campaign-prepared-tap-tree }}", "artifact-name-prefix" => "${{ steps.artifact-scope.outputs.prefix }}", + "candidate-mode" => "${{ steps.trust.outputs.candidate-mode }}", + "candidate-pr-number" => + "${{ steps.trust.outputs.candidate-pr-number }}", + "candidate-package-staging-tag" => + "${{ steps.trust.outputs.candidate-package-staging-tag }}", + "candidate-workflow-authority-sha" => + "${{ steps.candidate-authority.outputs.kandelo-sha }}", + "candidate-tap-workflow-authority-sha" => + "${{ steps.trust.outputs.candidate-tap-workflow-authority-sha }}", + "candidate-promotion-mode" => + "${{ steps.trust.outputs.candidate-promotion-mode }}", + "bottle-producer-sha" => "${{ steps.bottle-producer.outputs.sha }}", }, "publisher plan outputs changed") campaign_materializations = [ @@ -2372,6 +3660,8 @@ def check_publisher(workflow) "ADMISSION_KIND" => "${{ steps.campaign-source.outputs." \ "prefix-campaign-destination-admission-kind }}", + "CANDIDATE_MODE" => + "${{ steps.trust.outputs.candidate-mode }}", "DRY_RUN" => "${{ inputs.dry-run }}", "PREFIX_CAMPAIGN_MODE" => "${{ steps.trust.outputs.prefix-campaign-mode }}", @@ -2381,6 +3671,7 @@ def check_publisher(workflow) [ '[ "$PREFIX_CAMPAIGN_MODE" = "true" ]', '[ "$DRY_RUN" = "true" ]', + '[ "$CANDIDATE_MODE" != "true" ]', 'first-package-namespace-bootstrap-required', 'prefix="prefix-campaign-bootstrap-dry-run-"', 'echo "prefix=$prefix" >>"$GITHUB_OUTPUT"', @@ -2439,14 +3730,17 @@ def check_publisher(workflow) source_commits["env"] == { "PREFIX_CAMPAIGN_MODE" => "${{ steps.trust.outputs.prefix-campaign-mode }}", + "CANDIDATE_MODE" => + "${{ steps.trust.outputs.candidate-mode }}", "DRY_RUN" => "${{ inputs.dry-run }}", "REQUESTED_KANDELO_SHA" => "${{ inputs.kandelo-ref }}", }, "publisher source-commit resolution mapping changed") [ '[ "$DRY_RUN" = "false" ]', + '[ "$CANDIDATE_MODE" = "true" ]', '[ "$PREFIX_CAMPAIGN_MODE" = "true" ]', '[ "$kandelo_sha" != "$REQUESTED_KANDELO_SHA" ]', - "Kandelo checkout differs from the exact admitted main commit", + "Kandelo checkout differs from the exact requested source", ].each do |fragment| check(source_commits.fetch("run").include?(fragment), "publisher source checkout binding lacks #{fragment}") @@ -2457,7 +3751,8 @@ def check_publisher(workflow) check(tap_source_binding.keys.sort == %w[env if name run shell] && tap_source_binding["if"] == "${{ !inputs.dry-run || " \ - "steps.trust.outputs.prefix-campaign-mode == 'true' }}" && + "steps.trust.outputs.prefix-campaign-mode == 'true' || " \ + "steps.trust.outputs.candidate-mode == 'true' }}" && tap_source_binding["shell"] == "bash" && tap_source_binding["env"] == { "GH_TOKEN" => "${{ github.token }}", @@ -2486,11 +3781,11 @@ def check_publisher(workflow) "publisher resolves immutable sources outside the planning boundary") expected_uses = [ - *Array.new(23, CHECKOUT_ACTION), - *Array.new(6, NIX_ACTION), - *Array.new(3, MAGIC_NIX_ACTION), - *Array.new(9, UPLOAD_ACTION), - *Array.new(10, DOWNLOAD_ACTION), + *Array.new(35, CHECKOUT_ACTION), + *Array.new(7, NIX_ACTION), + *Array.new(4, MAGIC_NIX_ACTION), + *Array.new(11, UPLOAD_ACTION), + *Array.new(15, DOWNLOAD_ACTION), ].sort check(values_for_key(workflow, "uses").sort == expected_uses, "publisher action set or pin changed") @@ -2527,6 +3822,39 @@ def check_publisher(workflow) "path" => "kandelo", "submodules" => false, }, }, + { + "name" => "Checkout protected candidate validator authority", + "if" => "${{ steps.trust.outputs.candidate-mode == 'true' }}", + "with" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => "${{ steps.candidate-authority.outputs.kandelo-sha }}", + "path" => "candidate-authority", "submodules" => false, + }, + }, + { + "name" => "Checkout exact candidate caller as inert data", + "if" => + "${{ steps.trust.outputs.candidate-mode == 'true' || " \ + "steps.trust.outputs.candidate-promotion-mode == 'true' }}", + "with" => { + "persist-credentials" => false, + "repository" => "${{ inputs.tap-repository }}", + "ref" => "${{ github.sha }}", + "path" => "candidate-caller-data", "submodules" => false, + }, + }, + { + "name" => "Checkout promoted bottle producer", + "if" => "${{ steps.trust.outputs.candidate-promotion-mode == 'true' }}", + "with" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => "${{ inputs.candidate-producer-sha }}", + "path" => "bottle-producer", "fetch-depth" => 0, + "submodules" => false, + }, + }, { "name" => "Checkout tap", "if" => nil, "with" => { @@ -2537,6 +3865,29 @@ def check_publisher(workflow) }, }, ], "publisher plan checkout wiring changed") + caller_pin = named_step( + plan_steps, "Require the candidate caller to pin exact Kandelo authority" + ) + check(caller_pin.keys.sort == %w[env if name run shell] && + normalized_expression(caller_pin["if"]) == + "${{ steps.trust.outputs.candidate-mode == 'true' || " \ + "steps.trust.outputs.candidate-promotion-mode == 'true' }}" && + caller_pin["env"] == { + "CANDIDATE_MODE" => + "${{ steps.trust.outputs.candidate-mode }}", + "CANDIDATE_SHA" => + "${{ steps.candidate-authority.outputs.kandelo-sha }}", + "MERGE_SHA" => "${{ steps.trust.outputs.kandelo-ref }}", + }, "publisher candidate caller pin mapping changed") + [ + 'authority_root=candidate-authority', 'mode=bottle', + 'authority_root=kandelo', 'mode=promotion', + 'homebrew-candidate-caller-pins.py', + '--mode "$mode" --kandelo-sha "$expected_sha"', + ].each do |fragment| + check(caller_pin.fetch("run").include?(fragment), + "publisher candidate caller pin proof lacks #{fragment}") + end check(checkout_view.call(build_steps) == [ { "name" => "Checkout Kandelo workflow source", "if" => nil, @@ -2547,6 +3898,17 @@ def check_publisher(workflow) "path" => "kandelo", "submodules" => false, }, }, + { + "name" => "Checkout protected candidate package-reader authority", + "if" => "${{ needs.plan.outputs.candidate-mode == 'true' }}", + "with" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => + "${{ needs.plan.outputs.candidate-workflow-authority-sha }}", + "path" => "candidate-authority", "submodules" => false, + }, + }, { "name" => "Checkout tap", "if" => nil, "with" => { @@ -2592,6 +3954,16 @@ def check_publisher(workflow) "path" => "kandelo", "submodules" => false, }, }, + { + "name" => "Checkout promoted bottle producer for upload", + "if" => "${{ needs.plan.outputs.candidate-promotion-mode == 'true' }}", + "with" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => "${{ needs.plan.outputs.bottle-producer-sha }}", + "path" => "bottle-producer", "submodules" => false, + }, + }, { "name" => "Checkout exact tap source for upload validation", "if" => nil, "with" => { @@ -2612,6 +3984,16 @@ def check_publisher(workflow) "path" => "kandelo", "submodules" => false, }, }, + { + "name" => "Checkout promoted bottle producer for index publication", + "if" => "${{ needs.plan.outputs.candidate-promotion-mode == 'true' }}", + "with" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => "${{ needs.plan.outputs.bottle-producer-sha }}", + "path" => "bottle-producer", "submodules" => false, + }, + }, { "name" => "Checkout exact tap source for index validation", "if" => nil, "with" => { @@ -2632,6 +4014,16 @@ def check_publisher(workflow) "path" => "kandelo", "submodules" => false, }, }, + { + "name" => "Checkout promoted bottle producer for verification", + "if" => "${{ needs.plan.outputs.candidate-promotion-mode == 'true' }}", + "with" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => "${{ needs.plan.outputs.bottle-producer-sha }}", + "path" => "bottle-producer", "submodules" => false, + }, + }, { "name" => "Checkout exact Kandelo sysroot build source", "if" => nil, "with" => { @@ -2641,6 +4033,18 @@ def check_publisher(workflow) "path" => "kandelo-sysroot-build", "submodules" => false, }, }, + { + "name" => + "Checkout protected candidate verifier package-reader authority", + "if" => "${{ needs.plan.outputs.candidate-mode == 'true' }}", + "with" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => + "${{ needs.plan.outputs.candidate-workflow-authority-sha }}", + "path" => "candidate-authority", "submodules" => false, + }, + }, { "name" => "Checkout exact tap source", "if" => nil, "with" => { @@ -2995,6 +4399,8 @@ def check_publisher(workflow) check(plan_credential_steps.map { |step| step["name"] } == [ "Admit exact Kandelo main source", "Admit prefix-campaign Kandelo main history", + "Bind candidate validator to the exact merge base", + "Admit promoted producer on exact-main tree", "Bind write tap source to protected main history", ] && plan_credential_steps.all? do |step| step.fetch("env").slice(*credential_names) == { @@ -3002,16 +4408,24 @@ def check_publisher(workflow) } end, "publisher plan credential escapes source validation") { - build_steps => build_generation, - verify_steps => verify_generations, - }.each do |steps, generation_step| + build_steps => [ + build_generation, + named_step(build_steps, "Materialize immutable candidate Formula runtime packages"), + ], + verify_steps => [ + verify_generations, + named_step(verify_steps, "Re-materialize immutable candidate verification packages"), + ], + }.each do |steps, generation_steps| credential_steps = steps.select do |step| !(step.fetch("env", {}).keys & credential_names).empty? end - check(credential_steps == [generation_step] && - generation_step.fetch("env").slice(*credential_names) == { - "GH_TOKEN" => "${{ github.token }}", - }, + check(credential_steps == generation_steps && + generation_steps.all? do |generation_step| + generation_step.fetch("env").slice(*credential_names) == { + "GH_TOKEN" => "${{ github.token }}", + } + end, "publisher read credential escapes exact public-generation metadata fetch") check(steps.select { |step| step["uses"] == CHECKOUT_ACTION }.all? do |step| step.dig("with", "persist-credentials") == false @@ -3075,6 +4489,130 @@ def check_publisher(workflow) step.dig("with", "persist-credentials") == false end, "publisher VFS release persists checkout credentials") + candidate_credential_steps = candidate_steps.select do |step| + !(step.fetch("env", {}).keys & credential_names).empty? + end + check(candidate_credential_steps.map { |step| step["name"] } == [ + "Capture immutable candidate evidence", + "Publish and anonymously read back candidate release", + ] && candidate_credential_steps.all? do |step| + step.fetch("env").slice(*credential_names) == { + "GH_TOKEN" => "${{ github.token }}", + } + end, "publisher candidate sealer credentials escape reviewed evidence steps") + candidate_checkout_views = candidate_steps.select do |step| + step["uses"] == CHECKOUT_ACTION + end.to_h { |step| [step.fetch("name"), step.fetch("with")] } + check(candidate_checkout_views == { + "Checkout protected candidate validator authority" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => + "${{ needs.plan.outputs.candidate-workflow-authority-sha }}", + "path" => "authority", "submodules" => false, + }, + "Checkout inert candidate producer" => { + "persist-credentials" => false, + "repository" => "${{ inputs.kandelo-repository }}", + "ref" => "${{ needs.plan.outputs.kandelo-sha }}", + "path" => "producer", "fetch-depth" => 0, "submodules" => false, + }, + "Checkout exact candidate tap source" => { + "persist-credentials" => false, + "repository" => "${{ inputs.tap-repository }}", + "ref" => "${{ needs.plan.outputs.tap-sha }}", + "path" => "tap-source", "fetch-depth" => 0, + }, + "Checkout tap workflow authority" => { + "persist-credentials" => false, + "repository" => "${{ inputs.tap-repository }}", + "ref" => + "${{ needs.plan.outputs.candidate-tap-workflow-authority-sha }}", + "path" => "tap-authority", "fetch-depth" => 0, + }, + }, "publisher candidate sealer checkout authority changed") + candidate_validation = named_step( + candidate_steps, "Validate candidate handoffs with reviewed code" + ) + candidate_validation_run = candidate_validation.fetch("run") + [ + 'for secret_name in GH_TOKEN GITHUB_TOKEN \\', + 'candidate data validator received $secret_name', + 'python3 authority/scripts/homebrew-oci-layout.py validate-child', + 'python3 authority/scripts/homebrew-dependency-taps.py resolve', + 'bash authority/scripts/homebrew-validate-build-handoff.sh \\', + '--tap-checkout-commit \\', + '"$KANDELO_HOMEBREW_PREPARED_TAP_COMMIT"', + 'jq -e \'.dependencies == []\'', + ].each do |fragment| + check(candidate_validation_run.include?(fragment), + "publisher candidate sealer validation lacks #{fragment}") + end + check_forbidden_root_args( + candidate_validation_run, + "publisher candidate sealer handoff validation", + [ + '--forbidden-root "$GITHUB_WORKSPACE"', + '--forbidden-root "$(dirname "$GITHUB_WORKSPACE")"', + '--forbidden-root "$RUNNER_TEMP"', + ] + ) + candidate_capture = named_step( + candidate_steps, "Capture immutable candidate evidence" + ) + [ + '[ "$(jq -er \'.head.sha\' "$pr")" = "$KANDELO_SHA" ]', + '[ "$(jq -er \'.state\' "$pr")" = open ]', + '[ "$base_sha" = "$current_main" ]', + 'authority_status="$(gh api', + 'tap_source_status="$(gh api', + 'tap_authority_status="$(gh api', + '(.digest | type == "string" and', + 'workflow_path:".github/workflows/candidate-bottles.yml"', + 'env -u GH_TOKEN -u GITHUB_TOKEN \\', + 'python3 authority/scripts/homebrew-bottle-candidate.py \\', + 'merge_method:"merge"', + 'prefix_campaign_layout_sha256:$campaign_layout', + ].each do |fragment| + check(candidate_capture.fetch("run").include?(fragment), + "publisher candidate evidence capture lacks #{fragment}") + end + candidate_probe = named_step( + candidate_steps, "Prove candidate refs are collision-free" + ) + candidate_prepare = named_step( + candidate_steps, "Prepare inert immutable candidate release" + ) + [candidate_probe, candidate_prepare].each do |step| + check((step.fetch("env", {}).keys & credential_names).empty? && + step.fetch("run").include?( + '[ -z "${GH_TOKEN:-}" ] && [ -z "${GITHUB_TOKEN:-}" ]' + ), "publisher #{step.fetch('name')} receives candidate credentials") + end + check(candidate_probe.fetch("run").include?( + 'python3 authority/scripts/homebrew-oci-layout.py merge-index \\' + ) && candidate_probe.fetch("run").include?( + 'homebrew_ref_status:"available"' + ), "publisher candidate collision proof no longer checks Homebrew selection") + check(candidate_prepare.fetch("run").include?( + 'python3 authority/scripts/homebrew-bottle-candidate.py prepare \\' + ) && candidate_prepare.fetch("run").include?( + '--package-input \\' + ), "publisher candidate release preparation changed") + candidate_write = named_step( + candidate_steps, "Publish and anonymously read back candidate release" + ) + [ + 'bash authority/scripts/publish-immutable-github-release.sh \\', + '--lock-root "$GITHUB_WORKSPACE/tap-authority"', + '--kandelo-main-contains-sha "$AUTHORITY_SHA"', + '--target-main-contains-sha "$TAP_AUTHORITY_SHA"', + 'This release is noncanonical until exact-head promotion.', + ].each do |fragment| + check(candidate_write.fetch("run").include?(fragment), + "publisher candidate release write lacks #{fragment}") + end + exact_main_helper_path = File.join( REPO_ROOT, ".github/scripts/require-exact-kandelo-main.sh" ) @@ -3161,6 +4699,9 @@ def check_publisher(workflow) "${{ needs.plan.outputs.artifact-name-prefix }}" \ "homebrew-oci-child-${{ matrix.formula }}-${{ matrix.arch }}-" \ "attempt-${{ github.run_attempt }}" + promotion_receipt_name = + "homebrew-candidate-promotion-${{ matrix.formula }}-${{ matrix.arch }}-" \ + "attempt-${{ github.run_attempt }}" index_publication_name = "homebrew-index-publication-${{ matrix.formula }}-attempt-${{ github.run_attempt }}" vfs_release_handoff_name = @@ -3181,6 +4722,21 @@ def check_publisher(workflow) "compression-level" => 0, "if-no-files-found" => "error", "retention-days" => 2, }, "publisher OCI child artifact contract changed") + candidate_package_upload = named_step( + build_steps, "Upload candidate package-input identity" + ) + check(candidate_package_upload["uses"] == UPLOAD_ACTION && + candidate_package_upload["if"] == + "${{ needs.plan.outputs.candidate-mode == 'true' }}" && + candidate_package_upload["with"] == { + "name" => + "homebrew-candidate-package-input-${{ matrix.formula }}-" \ + "${{ matrix.arch }}-attempt-${{ github.run_attempt }}", + "path" => + "${{ runner.temp }}/homebrew-candidate-packages/package-input.json", + "compression-level" => 0, + "if-no-files-found" => "error", "retention-days" => 2, + }, "publisher candidate package-input artifact contract changed") build_diagnostics = named_step( build_steps, "Upload unprivileged build diagnostics" ) @@ -3211,6 +4767,106 @@ def check_publisher(workflow) "path" => "${{ runner.temp }}/homebrew-oci-child", }, "publisher OCI child download contract changed") end + promotion_download = named_step( + upload_steps, "Download exact candidate promotion admission" + ) + check(promotion_download["uses"] == DOWNLOAD_ACTION && + promotion_download["id"] == "candidate-promotion" && + promotion_download["if"] == + "${{ needs.plan.outputs.candidate-promotion-mode == 'true' }}" && + promotion_download["continue-on-error"] == true && + promotion_download["with"] == { + "name" => promotion_receipt_name, + "path" => "${{ runner.temp }}/homebrew-candidate-promotion", + }, "publisher candidate promotion receipt download changed") + promotion_validation = named_step( + upload_steps, "Bind promotion artifacts to the exact merged candidate" + ) + check(promotion_validation["id"] == "validate-candidate-promotion" && + normalized_expression(promotion_validation["if"]) == + "${{ needs.plan.outputs.candidate-promotion-mode == 'true' && " \ + "steps.build-handoff.outcome == 'success' && " \ + "steps.oci-child.outcome == 'success' && " \ + "steps.candidate-promotion.outcome == 'success' }}" && + promotion_validation["env"] == { + "ABI" => "${{ needs.plan.outputs.abi }}", + "ARCH" => "${{ matrix.arch }}", + "CAMPAIGN_LAYOUT" => + "${{ needs.plan.outputs.prefix-campaign-layout-sha256 }}", + "CAMPAIGN_TAG" => + "${{ needs.plan.outputs.prefix-campaign-tag }}", + "CANDIDATE_TAG" => "${{ inputs.candidate-promotion-tag }}", + "FORMULA" => "${{ matrix.formula }}", + "MERGE_COMMIT" => "${{ needs.plan.outputs.kandelo-sha }}", + "PRODUCER_COMMIT" => + "${{ needs.plan.outputs.bottle-producer-sha }}", + "TAP_CHECKOUT_COMMIT" => + "${{ needs.plan.outputs.prefix-campaign-prepared-tap-commit }}", + "TAP_COMMIT" => "${{ needs.plan.outputs.tap-sha }}", + }, "publisher candidate promotion validation mapping changed") + promotion_validation_run = promotion_validation.fetch("run") + [ + '[ -z "${GH_TOKEN:-}" ] && [ -z "${GITHUB_TOKEN:-}" ]', + 'python3 kandelo/scripts/homebrew-candidate-campaign.py \\', + 'validate-admission', + 'candidate-campaign-admission.json', + '--candidate-tag "$CAMPAIGN_TAG"', + '--producer-commit "$PRODUCER_COMMIT"', + '--merge-commit "$MERGE_COMMIT"', + '--source-tap-commit "$TAP_COMMIT"', + '--abi "$ABI"', + '--guest-layout-sha256 "$CAMPAIGN_LAYOUT"', + 'python3 kandelo/scripts/homebrew-bottle-candidate.py \\', + 'validate-promotion', + '--receipt \\', + 'candidate-promotion.json', + '--candidate-tag "$CANDIDATE_TAG"', + '--producer-commit "$PRODUCER_COMMIT"', + '--merge-commit "$MERGE_COMMIT"', + '--tap-commit "$TAP_COMMIT"', + '--tap-checkout-commit "$TAP_CHECKOUT_COMMIT"', + '--campaign-tag "$CAMPAIGN_TAG"', + '--campaign-layout-sha256 "$CAMPAIGN_LAYOUT"', + '--formula "$FORMULA" --arch "$ARCH"', + '--build-handoff "$RUNNER_TEMP/homebrew-build-handoff"', + '--oci-child "$RUNNER_TEMP/homebrew-oci-child"', + '--package-input \\', + 'candidate-package-input.json', + ].each do |fragment| + check(promotion_validation_run.include?(fragment), + "publisher candidate promotion validation lacks #{fragment}") + end + campaign_admission_index = promotion_validation_run.index( + "python3 kandelo/scripts/homebrew-candidate-campaign.py" + ) + bottle_admission_index = promotion_validation_run.index( + "python3 kandelo/scripts/homebrew-bottle-candidate.py" + ) + check(campaign_admission_index && bottle_admission_index && + campaign_admission_index < bottle_admission_index, + "publisher validates bottle promotion before its campaign admission") + promotion_build_validation = named_step( + upload_steps, "Validate build data before exposing upload credentials" + ) + check(normalized_expression(promotion_build_validation["if"]) == + "${{ steps.build-handoff.outcome == 'success' && " \ + "steps.oci-child.outcome == 'success' && ( " \ + "needs.plan.outputs.candidate-promotion-mode != 'true' || " \ + "steps.validate-candidate-promotion.outcome == 'success' ) }}" && + upload_steps.index(promotion_download) < + upload_steps.index(promotion_validation) && + upload_steps.index(promotion_validation) < + upload_steps.index(promotion_build_validation), + "publisher exposes promotion bytes before exact receipt admission") + missing_promotion = named_step( + upload_steps, "Fail when the matching immutable handoff is absent" + ) + check(normalized_expression(missing_promotion["if"]) == + "${{ always() && ( steps.build-handoff.outcome != 'success' || " \ + "steps.oci-child.outcome != 'success' || ( " \ + "needs.plan.outputs.candidate-promotion-mode == 'true' && " \ + "steps.validate-candidate-promotion.outcome != 'success' ) ) }}", + "publisher does not fail closed on missing promotion admission") receipt_upload = named_step(upload_steps, "Upload strict upload receipt") check(receipt_upload["uses"] == UPLOAD_ACTION && receipt_upload["with"] == { "name" => upload_receipt_name, @@ -3224,6 +4880,50 @@ def check_publisher(workflow) "name" => upload_receipt_name, "path" => "${{ runner.temp }}/homebrew-upload-receipt", }, "publisher receipt download contract changed") + index_campaign_download = named_step( + index_steps, "Download exact candidate campaign admission for index" + ) + check(index_campaign_download["uses"] == DOWNLOAD_ACTION && + index_campaign_download["if"] == + "${{ needs.plan.outputs.candidate-promotion-mode == 'true' }}" && + index_campaign_download["with"] == { + "name" => + "homebrew-candidate-promotion-${{ matrix.formula }}-" \ + "wasm32-attempt-${{ github.run_attempt }}", + "path" => + "${{ runner.temp }}/homebrew-candidate-index-admission", + }, "publisher candidate index admission download changed") + index_campaign_validation = named_step( + index_steps, "Bind index write to the admitted candidate campaign" + ) + check(index_campaign_validation["if"] == + "${{ needs.plan.outputs.candidate-promotion-mode == 'true' }}" && + index_campaign_validation["env"] == { + "ABI" => "${{ needs.plan.outputs.abi }}", + "CAMPAIGN_LAYOUT" => + "${{ needs.plan.outputs.prefix-campaign-layout-sha256 }}", + "CAMPAIGN_TAG" => + "${{ needs.plan.outputs.prefix-campaign-tag }}", + "MERGE_COMMIT" => "${{ needs.plan.outputs.kandelo-sha }}", + "PRODUCER_COMMIT" => + "${{ needs.plan.outputs.bottle-producer-sha }}", + "TAP_COMMIT" => "${{ needs.plan.outputs.tap-sha }}", + }, "publisher candidate index admission mapping changed") + index_campaign_run = index_campaign_validation.fetch("run") + [ + '[ -z "${GH_TOKEN:-}" ] && [ -z "${GITHUB_TOKEN:-}" ]', + 'python3 kandelo/scripts/homebrew-candidate-campaign.py \\', + 'validate-admission', 'candidate-campaign-admission.json', + '--candidate-tag "$CAMPAIGN_TAG"', + '--producer-commit "$PRODUCER_COMMIT"', + '--merge-commit "$MERGE_COMMIT"', + '--source-tap-commit "$TAP_COMMIT"', + '--abi "$ABI"', + '--guest-layout-sha256 "$CAMPAIGN_LAYOUT"', + ].each do |fragment| + check(index_campaign_run.include?(fragment), + "publisher candidate index admission lacks #{fragment}") + end index_child_download = named_step(index_steps, "Download immutable OCI child layouts") check(index_child_download["uses"] == DOWNLOAD_ACTION && index_child_download["with"] == { "pattern" => "homebrew-oci-child-${{ matrix.formula }}-*-attempt-${{ github.run_attempt }}", @@ -3247,6 +4947,17 @@ def check_publisher(workflow) "compression-level" => 0, "if-no-files-found" => "error", "retention-days" => 2, }, "publisher version-index publication artifact contract changed") + index_write = named_step( + index_steps, + "Publish the complete Homebrew version index in isolated ORAS auth state" + ) + check(index_steps.index(index_campaign_download) < + index_steps.index(index_campaign_validation) && + index_steps.index(index_campaign_validation) < + index_steps.index(index_child_download) && + index_steps.index(index_campaign_validation) < + index_steps.index(index_write), + "publisher can write a promoted index before campaign admission") index_publication_download = named_step( verify_steps, "Download public Homebrew version-index evidence" ) @@ -3306,6 +5017,42 @@ def check_publisher(workflow) "path" => "${{ runner.temp }}/homebrew-vfs-release-receipt.json", "if-no-files-found" => "error", "retention-days" => 14, }, "publisher VFS release receipt artifact contract changed") + candidate_artifact_downloads = { + "Download candidate build handoff" => { + "name" => + "homebrew-build-handoff-${{ inputs.formulae }}-" \ + "${{ inputs.arches }}-attempt-${{ github.run_attempt }}", + "path" => "${{ runner.temp }}/candidate-build", + }, + "Download candidate OCI child" => { + "name" => + "homebrew-oci-child-${{ inputs.formulae }}-" \ + "${{ inputs.arches }}-attempt-${{ github.run_attempt }}", + "path" => "${{ runner.temp }}/candidate-oci", + }, + "Download candidate package-input identity" => { + "name" => + "homebrew-candidate-package-input-${{ inputs.formulae }}-" \ + "${{ inputs.arches }}-attempt-${{ github.run_attempt }}", + "path" => "${{ runner.temp }}/candidate-packages", + }, + } + candidate_artifact_downloads.each do |name, with| + step = named_step(candidate_steps, name) + check(step["uses"] == DOWNLOAD_ACTION && step["with"] == with, + "publisher #{name} artifact contract changed") + end + candidate_receipt = named_step( + candidate_steps, "Retain candidate publication receipt" + ) + check(candidate_receipt["uses"] == UPLOAD_ACTION && + candidate_receipt["with"] == { + "name" => + "homebrew-candidate-release-receipt-${{ inputs.formulae }}-" \ + "${{ inputs.arches }}-attempt-${{ github.run_attempt }}", + "path" => "${{ runner.temp }}/candidate-release-receipt.json", + "if-no-files-found" => "error", "retention-days" => 90, + }, "publisher candidate release receipt artifact contract changed") build_formula_step = named_step( build_steps, "Build and test Homebrew bottle without publisher credentials" @@ -7458,7 +9205,7 @@ def check_publisher(workflow) stripped if stripped.start_with?("--forbidden-root ") end end - check(forbidden_root_lines.length == 35, + check(forbidden_root_lines.length == 38, "publisher does not pass the exact trusted forbidden-root set at every archive boundary") check(forbidden_root_lines.none? { |line| line.include?("linuxbrew") || line.include?("/opt/") }, "publisher forbids canonical Homebrew prefix or opt metadata") @@ -7599,6 +9346,10 @@ def check_publisher(workflow) 'bash "$REPO_ROOT/scripts/test-homebrew-publisher-real-lifecycle.sh"', 'bash "$REPO_ROOT/scripts/test-homebrew-validate-host-dependency-plan.sh"', 'python3 "$REPO_ROOT/scripts/test-prepare-homebrew-recipe-host-runtime.py"', + 'python3 "$REPO_ROOT/scripts/test-homebrew-candidate-campaign.py"', + 'python3 "$REPO_ROOT/scripts/test-homebrew-bottle-candidate.py"', + 'python3 "$REPO_ROOT/scripts/test-homebrew-candidate-release-receipt.py"', + 'python3 "$REPO_ROOT/scripts/test-homebrew-candidate-caller-pins.py"', 'assert_atomic_publication_batch_closes_formula_metadata_wave', 'KANDELO_HOMEBREW_RESOLVED_TAPS_FILE="$(make_primary_resolved_tap_map "$tap_root")"', 'export KANDELO_HOMEBREW_RESOLVED_TAPS_FILE', @@ -7661,7 +9412,8 @@ def check_publisher(workflow) check(!values_for_key(workflow, "run").join("\n").include?("GITHUB_SHA"), "publisher reads an ambient workflow execution SHA") - check(!JSON.generate(plan).include?("${{ github.sha }}") && + check(JSON.generate(plan).scan("${{ github.sha }}").length == 2 && + values_for_key(plan, "CALLER_SHA") == ["${{ github.sha }}"] && values_for_key(plan, "REQUESTED_TAP_SHA") == ["${{ inputs.tap-ref }}"], "publisher substitutes workflow execution head for the requested tap source") check(contract_digest(plan_steps) == PUBLISHER_PLAN_DIGEST, @@ -7678,6 +9430,8 @@ def check_publisher(workflow) "publisher finalization step contract changed") check(contract_digest(vfs_release_steps) == PUBLISHER_VFS_RELEASE_DIGEST, "publisher VFS release step contract changed") + check(contract_digest(candidate_steps) == PUBLISHER_CANDIDATE_DIGEST, + "publisher candidate sealer step contract changed") end def check_maintenance(workflow) @@ -9406,6 +11160,8 @@ def self_test(publisher, native_compatibility, maintenance, maintenance = load_workflow(MAINTENANCE_PATH) first_publication = load_workflow(FIRST_PUBLICATION_PATH) prefix_first_child = load_workflow(PREFIX_FIRST_CHILD_PATH) + candidate_materializer = load_workflow(CANDIDATE_MATERIALIZER_PATH) + candidate_campaign = load_workflow(CANDIDATE_CAMPAIGN_PATH) self_test_privileged_recipe_host_runtime(all_workflows) self_test( publisher, native_compatibility, maintenance, first_publication, @@ -9420,6 +11176,8 @@ def self_test(publisher, native_compatibility, maintenance, check_maintenance(maintenance) check_first_publication(first_publication) check_prefix_first_child(prefix_first_child) + check_candidate_materializer(candidate_materializer) + check_candidate_campaign(candidate_campaign) check_tap_callers puts "check-homebrew-publish-workflow-trust.rb: ok" rescue KeyError, Psych::Exception, RuntimeError => e diff --git a/scripts/homebrew-bottle-candidate.py b/scripts/homebrew-bottle-candidate.py new file mode 100755 index 0000000000..c0ab2f73b4 --- /dev/null +++ b/scripts/homebrew-bottle-candidate.py @@ -0,0 +1,1913 @@ +#!/usr/bin/env python3 +"""Seal and re-materialize one pre-merge Homebrew bottle candidate. + +The command deliberately does not build Formulae, execute candidate code, or +write to GitHub. A reviewed workflow validates the ordinary build handoff and +OCI child before calling ``prepare``. A protected-main workflow calls +``materialize`` after proving the exact merge and package-generation facts. + +Candidate files are separate release assets. There is no candidate archive +to extract in a credentialed job. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +from typing import Any, NoReturn + + +SHA256 = re.compile(r"^[0-9a-f]{64}$") +COMMIT = re.compile(r"^[0-9a-f]{40}$") +FORMULA = re.compile(r"^[a-z0-9][a-z0-9._-]{0,254}$") +REPOSITORY = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +STAGING_TAG = re.compile( + r"^pr-([1-9][0-9]*)-staging-run-([1-9][0-9]*)-attempt-" + r"([1-9][0-9]*)$" +) +CANDIDATE_TAG = re.compile( + r"^homebrew-bottle-candidate-pr-([1-9][0-9]*)-run-" + r"([1-9][0-9]*)-attempt-([1-9][0-9]*)-sha256-" + r"([0-9a-f]{64})$" +) +HANDOFF_TAG = re.compile( + r"^homebrew-prefix-handoff-sha256-([0-9a-f]{64})$" +) +ARTIFACT_DIGEST = re.compile(r"^sha256:([0-9a-f]{64})$") +OCI_DIGEST = ARTIFACT_DIGEST +OCI_TAG = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$") +PKG_VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+,-]{0,255}$") + +MAX_JSON_BYTES = 16 * 1024 * 1024 +MAX_FILE_BYTES = 2 * 1024 * 1024 * 1024 +MAX_TOTAL_BYTES = 3 * 1024 * 1024 * 1024 +MAX_OCI_BLOBS = 8 +MAX_DEPENDENCIES = 256 +MAX_PACKAGE_ARCHIVES = 1024 + +BUILD_FILES = ( + ("manifest.json", "build-manifest.json"), + ("bottle.json", "build-bottle.json"), + ("dependency-provenance.json", "build-dependency-provenance.json"), + ("bottle.tar.gz", "build-bottle.tar.gz"), +) + + +def validate_expected_package_ledger(value: Any, abi: int) -> dict[str, Any]: + value = exact_keys( + value, + {"abi_version", "entries"}, + "expected package ledger", + ) + if value["abi_version"] != abi: + fail("expected package ledger has the wrong ABI") + entries = value["entries"] + if ( + not isinstance(entries, list) + or not entries + or len(entries) > MAX_PACKAGE_ARCHIVES + ): + fail("expected package ledger is empty or too large") + identities: list[tuple[str, str]] = [] + for position, entry in enumerate(entries): + entry = exact_keys( + entry, + { + "arch", + "cache_key_sha", + "git_inputs", + "kind", + "package", + "revision", + "version", + }, + f"expected package #{position}", + ) + package = require_string( + entry["package"], f"expected package #{position}", FORMULA + ) + arch = require_string(entry["arch"], f"expected arch #{position}") + if arch not in ("wasm32", "wasm64"): + fail("expected package ledger has an invalid architecture") + if entry["kind"] not in ("library", "program"): + fail("expected package ledger has an invalid package kind") + require_string(entry["version"], "expected package version", PKG_VERSION) + require_int(entry["revision"], "expected package revision") + require_string(entry["cache_key_sha"], "expected cache key", SHA256) + if not isinstance(entry["git_inputs"], list): + fail("expected package git inputs are not an array") + identities.append((package, arch)) + if identities != sorted(set(identities)): + fail("expected package ledger must be unique and sorted") + return value + + +def validate_package_snapshot(value: Any, expected: dict[str, Any]) -> dict[str, Any]: + value = exact_keys( + value, + {"abi_version", "complete_current", "entries", "release_tag"}, + "validated package snapshot", + ) + if ( + value["abi_version"] != expected["abi_version"] + or value["complete_current"] is not True + ): + fail("validated package snapshot is not complete and current") + entries = value["entries"] + if not isinstance(entries, list) or len(entries) != len(expected["entries"]): + fail("validated package snapshot does not cover the full ledger") + expected_by_identity = { + (entry["package"], entry["arch"]): entry for entry in expected["entries"] + } + seen: set[tuple[str, str]] = set() + for position, entry in enumerate(entries): + entry = exact_keys( + entry, + { + "arch", + "archive_sha256", + "asset", + "cache_key_sha", + "current", + "kind", + "package", + "revision", + "size", + "version", + }, + f"validated package snapshot entry #{position}", + ) + identity = (entry["package"], entry["arch"]) + wanted = expected_by_identity.get(identity) + if identity in seen or wanted is None: + fail("validated package snapshot has an unexpected package") + seen.add(identity) + if ( + entry["current"] is not True + or entry["kind"] != wanted["kind"] + or entry["version"] != wanted["version"] + or entry["revision"] != wanted["revision"] + or entry["cache_key_sha"] != wanted["cache_key_sha"] + ): + fail("validated package snapshot differs from the expected ledger") + name = require_string(entry["asset"], "validated package asset", maximum=255) + if "/" in name or "\\" in name or name in (".", ".."): + fail("validated package snapshot has an unsafe asset name") + require_string(entry["archive_sha256"], "package archive SHA-256", SHA256) + require_int(entry["size"], "package archive bytes", 1) + if seen != set(expected_by_identity): + fail("validated package snapshot omits an expected package") + return value + + +def create_package_input(arguments: argparse.Namespace) -> None: + abi = require_int(arguments.abi, "candidate package ABI", 1) + producer = require_string( + arguments.producer_commit, "candidate package producer", COMMIT + ) + expected, _ = load_json( + pathlib.Path(arguments.expected_ledger), "expected package ledger" + ) + expected = validate_expected_package_ledger(expected, abi) + snapshot, _ = load_json( + pathlib.Path(arguments.snapshot), "validated package snapshot" + ) + snapshot = validate_package_snapshot(snapshot, expected) + release, _ = load_json( + pathlib.Path(arguments.release_evidence), "package release evidence" + ) + release = exact_keys( + release, + { + "attempt", + "immutable", + "pr_number", + "release_id", + "repository", + "run_id", + "schema", + "tag", + "target_commit", + }, + "package release evidence", + ) + if release["schema"] != 1: + fail("package release evidence has an unsupported schema") + if normalized_repository(release["repository"]) != "automattic/kandelo": + fail("package release evidence has the wrong repository") + match = STAGING_TAG.fullmatch(require_string(release["tag"], "staging tag")) + if match is None: + fail("package release evidence has an invalid staging tag") + pr_number = require_int(release["pr_number"], "package PR number", 1) + run_id = require_int(release["run_id"], "package run ID", 1) + attempt = require_int(release["attempt"], "package run attempt", 1) + if tuple(map(int, match.groups())) != (pr_number, run_id, attempt): + fail("package release tag differs from its run identity") + require_int(release["release_id"], "package release ID", 1) + if release["immutable"] is not True or release["target_commit"] != producer: + fail("package release is not immutable at the candidate producer") + if snapshot["release_tag"] != release["tag"]: + fail("validated package snapshot names a different release") + index = regular_file( + pathlib.Path(arguments.index), "validated package index", MAX_JSON_BYTES + ) + archives = [] + expected_by_identity = { + (entry["package"], entry["arch"]): entry for entry in expected["entries"] + } + for entry in snapshot["entries"]: + wanted = expected_by_identity[(entry["package"], entry["arch"])] + archives.append( + { + "package": entry["package"], + "arch": entry["arch"], + "version": wanted["version"], + "revision": wanted["revision"], + "cache_key_sha": wanted["cache_key_sha"], + "name": entry["asset"], + "sha256": entry["archive_sha256"], + "bytes": entry["size"], + } + ) + archives.sort(key=lambda item: (item["package"], item["arch"])) + package_input = { + "schema": 1, + "kind": "kandelo-homebrew-candidate-package-input", + "repository": "Automattic/kandelo", + "producer_commit": producer, + "abi": abi, + "expected_ledger_sha256": sha256_bytes(canonical_json(expected)), + "index": {"sha256": sha256_file(index), "bytes": index.stat().st_size}, + "staging_release": { + "tag": release["tag"], + "release_id": release["release_id"], + "target_commit": producer, + "immutable": True, + "pr_number": pr_number, + "run_id": run_id, + "attempt": attempt, + }, + "archives": archives, + } + validate_package_input(package_input) + output = pathlib.Path(arguments.out) + if output.exists() or output.is_symlink(): + fail("candidate package input output must not already exist") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(pretty_json(package_input)) + + +def admit_package_input(arguments: argparse.Namespace) -> None: + candidate_path = pathlib.Path(arguments.candidate_package_input) + regenerated_path = pathlib.Path(arguments.regenerated_package_input) + candidate, candidate_payload = load_json( + candidate_path, "candidate package input" + ) + regenerated, regenerated_payload = load_json( + regenerated_path, "regenerated package input" + ) + candidate = validate_package_input(candidate) + regenerated = validate_package_input(regenerated) + if ( + candidate_payload != pretty_json(candidate) + or regenerated_payload != pretty_json(regenerated) + ): + fail("candidate package admission requires canonical JSON inputs") + if candidate_payload != regenerated_payload: + fail("regenerated package input differs from the sealed candidate") + producer = require_string( + arguments.producer_commit, "candidate package producer", COMMIT + ) + validated_main = require_string( + arguments.validated_main, "package validation main", COMMIT + ) + if candidate["producer_commit"] != producer: + fail("candidate package input names another producer") + main_root = exact_git_checkout( + pathlib.Path(arguments.validated_main_root), + validated_main, + "package validation main checkout", + ) + producer_tree = run_git(main_root, "rev-parse", f"{producer}^{{tree}}") + main_tree = run_git(main_root, "rev-parse", f"{validated_main}^{{tree}}") + if producer_tree != main_tree: + fail("package producer tree differs from validated main") + output = pathlib.Path(arguments.out) + if output.exists() or output.is_symlink(): + fail("admitted package input output must not already exist") + output.parent.mkdir(parents=True, exist_ok=True) + admission = { + "schema": 1, + "kind": "kandelo-homebrew-admitted-candidate-package-input", + "validated_against_main": validated_main, + "candidate_package_input_sha256": sha256_bytes(candidate_payload), + "package_input": candidate, + } + output.write_bytes(pretty_json(admission)) + + +class CandidateError(ValueError): + """A candidate did not satisfy its bounded data contract.""" + + +def fail(message: str) -> NoReturn: + raise CandidateError(message) + + +def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + fail(f"JSON object repeats key {key!r}") + result[key] = value + return result + + +def load_json(path: pathlib.Path, label: str) -> tuple[Any, bytes]: + path = regular_file(path, label, MAX_JSON_BYTES) + payload = path.read_bytes() + try: + value = json.loads(payload, object_pairs_hook=reject_duplicates) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + fail(f"{label} is not strict UTF-8 JSON: {error}") + return value, payload + + +def pretty_json(value: Any) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() + + +def canonical_json(value: Any) -> bytes: + return json.dumps(value, separators=(",", ":"), sort_keys=True).encode() + + +def exact_keys(value: Any, expected: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != expected: + fail(f"{label} has unexpected fields") + return value + + +def require_string( + value: Any, + label: str, + pattern: re.Pattern[str] | None = None, + maximum: int = 4096, +) -> str: + if ( + not isinstance(value, str) + or not value + or len(value.encode()) > maximum + or "\x00" in value + or "\n" in value + or "\r" in value + ): + fail(f"{label} is not a bounded string") + if pattern is not None and pattern.fullmatch(value) is None: + fail(f"{label} has an invalid format") + return value + + +def require_int(value: Any, label: str, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + fail(f"{label} is not an integer greater than or equal to {minimum}") + return value + + +def regular_file( + path: pathlib.Path, label: str, maximum: int = MAX_FILE_BYTES +) -> pathlib.Path: + try: + metadata = path.lstat() + except OSError as error: + fail(f"cannot inspect {label}: {error}") + if not path.is_file() or path.is_symlink(): + fail(f"{label} must be a regular non-symlink file") + if metadata.st_size < 1 or metadata.st_size > maximum: + fail(f"{label} is outside its byte bound") + return path + + +def real_directory(path: pathlib.Path, label: str) -> pathlib.Path: + try: + path.lstat() + except OSError as error: + fail(f"cannot inspect {label}: {error}") + if not path.is_dir() or path.is_symlink(): + fail(f"{label} must be a real directory") + return path.resolve() + + +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def sha256_file(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while block := stream.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def file_record(path: pathlib.Path, logical_path: str, asset_name: str) -> dict[str, Any]: + path = regular_file(path, logical_path) + if ( + "/" in asset_name + or "\\" in asset_name + or asset_name in ("", ".", "..", "candidate.json") + or len(asset_name.encode()) > 255 + ): + fail(f"asset name for {logical_path} is unsafe") + return { + "asset_name": asset_name, + "bytes": path.stat().st_size, + "path": logical_path, + "sha256": sha256_file(path), + } + + +def normalized_repository(value: str) -> str: + require_string(value, "repository", REPOSITORY) + return value.lower() + + +def run_git(root: pathlib.Path, *arguments: str) -> str: + result = subprocess.run( + ["git", "-C", str(root), *arguments], + check=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=120, + ) + if result.returncode != 0: + detail = result.stderr.strip()[:2048] + fail(f"git {' '.join(arguments)} failed: {detail}") + return result.stdout.strip() + + +def exact_git_checkout(root: pathlib.Path, expected: str, label: str) -> pathlib.Path: + root = real_directory(root, label) + require_string(expected, f"{label} commit", COMMIT) + if run_git(root, "rev-parse", "HEAD") != expected: + fail(f"{label} is not at its expected commit") + if run_git(root, "status", "--porcelain=v1", "--untracked-files=all"): + fail(f"{label} is not clean") + return root + + +def source_contract_file( + root: pathlib.Path, relative: str, label: str +) -> pathlib.Path: + current = root + parts = pathlib.PurePosixPath(relative).parts + for part in parts[:-1]: + current = current / part + if current.is_symlink() or not current.is_dir(): + fail(f"{label} parent must be a real directory") + result = regular_file(current / parts[-1], label, MAX_JSON_BYTES) + if result.resolve().parent != current.resolve(): + fail(f"{label} escaped its exact source root") + return result + + +def describe_source(arguments: argparse.Namespace) -> None: + root = exact_git_checkout( + pathlib.Path(arguments.root), + arguments.producer_commit, + "candidate source checkout", + ) + if pathlib.Path(run_git(root, "rev-parse", "--show-toplevel")) != root: + fail("candidate source checkout is not the Git worktree root") + snapshot = source_contract_file( + root, "abi/snapshot.json", "candidate ABI snapshot" + ) + layout = source_contract_file( + root, + "homebrew/kandelo-guest-layout.json", + "candidate guest layout", + ) + output = pathlib.Path(arguments.out) + if output.exists() or output.is_symlink(): + fail("candidate source description output must not already exist") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes( + pretty_json( + { + "schema": 1, + "producer_commit": arguments.producer_commit, + "producer_tree": run_git(root, "rev-parse", "HEAD^{tree}"), + "abi_snapshot_sha256": sha256_file(snapshot), + "guest_layout_sha256": sha256_file(layout), + } + ) + ) + + +def require_ancestor(root: pathlib.Path, ancestor: str, descendant: str, label: str) -> None: + result = subprocess.run( + ["git", "-C", str(root), "merge-base", "--is-ancestor", ancestor, descendant], + check=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + timeout=120, + ) + if result.returncode != 0: + fail(f"{label} is not on protected main history") + + +def validate_package_input(value: Any) -> dict[str, Any]: + value = exact_keys( + value, + { + "abi", + "archives", + "expected_ledger_sha256", + "index", + "kind", + "producer_commit", + "repository", + "schema", + "staging_release", + }, + "candidate package input", + ) + if value["schema"] != 1 or value["kind"] != "kandelo-homebrew-candidate-package-input": + fail("candidate package input has an unsupported contract") + if normalized_repository(value["repository"]) != "automattic/kandelo": + fail("candidate package input has the wrong repository") + require_int(value["abi"], "candidate package ABI", 1) + require_string(value["producer_commit"], "package producer commit", COMMIT) + require_string( + value["expected_ledger_sha256"], "expected package ledger SHA-256", SHA256 + ) + index = exact_keys( + value["index"], {"bytes", "sha256"}, "candidate package index" + ) + require_int(index["bytes"], "candidate package index bytes", 1) + require_string(index["sha256"], "candidate package index SHA-256", SHA256) + staging = exact_keys( + value["staging_release"], + { + "attempt", + "immutable", + "pr_number", + "release_id", + "run_id", + "tag", + "target_commit", + }, + "candidate staging release", + ) + match = STAGING_TAG.fullmatch( + require_string(staging["tag"], "candidate staging tag") + ) + if match is None: + fail("candidate staging release tag is invalid") + pr_number = require_int(staging["pr_number"], "candidate PR number", 1) + run_id = require_int(staging["run_id"], "candidate staging run", 1) + attempt = require_int(staging["attempt"], "candidate staging attempt", 1) + if tuple(map(int, match.groups())) != (pr_number, run_id, attempt): + fail("candidate staging tag differs from its run identity") + require_int(staging["release_id"], "candidate staging release ID", 1) + if staging["immutable"] is not True: + fail("candidate staging release is not immutable") + if staging["target_commit"] != value["producer_commit"]: + fail("candidate staging release targets a different producer") + archives = value["archives"] + if ( + not isinstance(archives, list) + or not archives + or len(archives) > MAX_PACKAGE_ARCHIVES + ): + fail("candidate package archive ledger is empty or too large") + identities: list[tuple[str, str]] = [] + names: set[str] = set() + for position, archive in enumerate(archives): + archive = exact_keys( + archive, + { + "arch", + "bytes", + "cache_key_sha", + "name", + "package", + "revision", + "sha256", + "version", + }, + f"candidate package archive #{position}", + ) + package = require_string( + archive["package"], f"candidate package #{position}", FORMULA + ) + arch = require_string(archive["arch"], f"candidate arch #{position}") + if arch not in ("wasm32", "wasm64"): + fail("candidate package archive has an invalid architecture") + require_string(archive["version"], "candidate package version", PKG_VERSION) + require_int(archive["revision"], "candidate package revision") + require_string(archive["cache_key_sha"], "candidate cache key", SHA256) + name = require_string(archive["name"], "candidate archive name", maximum=255) + if "/" in name or "\\" in name or name in (".", "..") or name in names: + fail("candidate package archive names are unsafe or duplicated") + names.add(name) + require_string(archive["sha256"], "candidate archive SHA-256", SHA256) + require_int(archive["bytes"], "candidate archive bytes", 1) + identities.append((package, arch)) + if identities != sorted(set(identities)): + fail("candidate package archive ledger must be unique and sorted") + return value + + +def validate_dependencies(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list) or len(value) > MAX_DEPENDENCIES: + fail("candidate dependencies are not a bounded array") + prior = "" + for position, dependency in enumerate(value): + dependency = exact_keys( + dependency, + {"formula", "manifest_sha256", "tag"}, + f"candidate dependency #{position}", + ) + name = require_string( + dependency["formula"], f"candidate dependency #{position} Formula", FORMULA + ) + digest = require_string( + dependency["manifest_sha256"], "candidate dependency SHA-256", SHA256 + ) + match = HANDOFF_TAG.fullmatch( + require_string(dependency["tag"], "candidate dependency tag") + ) + if match is None or match.group(1) != digest: + fail("candidate dependency tag differs from its manifest") + if name <= prior: + fail("candidate dependencies must be unique and sorted") + prior = name + return value + + +def validate_run_evidence(value: Any, formula: str, arch: str) -> dict[str, Any]: + value = exact_keys( + value, + { + "artifacts", + "caller_commit", + "conclusion", + "event", + "repository", + "run_attempt", + "run_id", + "schema", + "status", + "workflow_path", + }, + "candidate run evidence", + ) + if value["schema"] != 1: + fail("candidate run evidence has an unsupported schema") + normalized_repository(value["repository"]) + require_string(value["caller_commit"], "candidate caller commit", COMMIT) + if value["event"] != "repository_dispatch": + fail("candidate run did not use repository_dispatch") + if value["workflow_path"] != ".github/workflows/candidate-bottles.yml": + fail("candidate run did not use its reviewed caller") + run_id = require_int(value["run_id"], "candidate run ID", 1) + attempt = require_int(value["run_attempt"], "candidate run attempt", 1) + if value["status"] not in ("in_progress", "completed"): + fail("candidate run status is invalid") + if value["conclusion"] not in (None, "success"): + fail("candidate run conclusion is not successful") + if value["status"] == "completed" and value["conclusion"] != "success": + fail("completed candidate run is not successful") + expected_names = { + f"homebrew-build-handoff-{formula}-{arch}-attempt-{attempt}", + f"homebrew-oci-child-{formula}-{arch}-attempt-{attempt}", + f"homebrew-candidate-package-input-{formula}-{arch}-attempt-{attempt}", + } + artifacts = value["artifacts"] + if not isinstance(artifacts, list) or len(artifacts) != 3: + fail("candidate run must bind exactly three candidate artifacts") + names: set[str] = set() + ids: set[int] = set() + for position, artifact in enumerate(artifacts): + artifact = exact_keys( + artifact, + {"bytes", "digest", "id", "name", "run_attempt", "run_id"}, + f"candidate artifact #{position}", + ) + artifact_id = require_int(artifact["id"], "candidate artifact ID", 1) + name = require_string(artifact["name"], "candidate artifact name", maximum=255) + require_int(artifact["bytes"], "candidate artifact bytes", 1) + require_string(artifact["digest"], "candidate artifact digest", ARTIFACT_DIGEST) + if artifact["run_id"] != run_id or artifact["run_attempt"] != attempt: + fail("candidate artifact belongs to a different run") + if artifact_id in ids or name in names: + fail("candidate artifact identity is duplicated") + ids.add(artifact_id) + names.add(name) + if names != expected_names: + fail("candidate artifacts differ from the exact Formula run") + return value + + +def validate_source_evidence(value: Any) -> dict[str, Any]: + value = exact_keys( + value, + { + "abi", + "abi_snapshot_sha256", + "base_commit", + "guest_layout", + "kandelo_repository", + "merge_method", + "pr_number", + "producer_commit", + "producer_tree", + "prefix_campaign_layout_sha256", + "prefix_campaign_tag", + "release_tag", + "tap_commit", + "tap_checkout_commit", + "tap_checkout_tree", + "tap_name", + "tap_repository", + "workflow_authority_commit", + }, + "candidate source evidence", + ) + if normalized_repository(value["kandelo_repository"]) != "automattic/kandelo": + fail("candidate source has the wrong Kandelo repository") + normalized_repository(value["tap_repository"]) + require_string(value["tap_name"], "candidate tap name", REPOSITORY) + pr_number = require_int(value["pr_number"], "candidate PR number", 1) + del pr_number + require_string(value["base_commit"], "candidate base commit", COMMIT) + require_string(value["producer_commit"], "candidate producer commit", COMMIT) + require_string(value["producer_tree"], "candidate producer tree", COMMIT) + require_string( + value["workflow_authority_commit"], "candidate workflow authority", COMMIT + ) + if value["workflow_authority_commit"] != value["base_commit"]: + fail("candidate validator authority must equal its protected base") + require_string(value["tap_commit"], "candidate tap commit", COMMIT) + require_string( + value["tap_checkout_commit"], "candidate prepared tap commit", COMMIT + ) + require_string( + value["tap_checkout_tree"], "candidate prepared tap tree", COMMIT + ) + campaign = require_string( + value["prefix_campaign_tag"], "candidate prefix campaign tag" + ) + if re.fullmatch( + r"homebrew-prefix-campaign-candidate-pr-[1-9][0-9]*-run-" + r"[1-9][0-9]*-attempt-[1-9][0-9]*-sha256-[0-9a-f]{64}", + campaign, + ) is None: + fail("candidate prefix campaign tag is invalid") + require_string( + value["prefix_campaign_layout_sha256"], + "candidate prefix campaign layout SHA-256", + SHA256, + ) + abi = require_int(value["abi"], "candidate ABI", 1) + if value["merge_method"] != "merge": + fail("candidate promotion requires an exact-head merge commit") + if value["release_tag"] != f"bottles-abi-v{abi}": + fail("candidate bottle release tag differs from its ABI") + require_string(value["abi_snapshot_sha256"], "ABI snapshot SHA-256", SHA256) + layout = exact_keys( + value["guest_layout"], {"path", "sha256"}, "candidate guest layout" + ) + if layout["path"] != "homebrew/kandelo-guest-layout.json": + fail("candidate guest layout path is not canonical") + require_string(layout["sha256"], "candidate guest layout SHA-256", SHA256) + return value + + +def validate_destination(value: Any, receipt: dict[str, Any]) -> dict[str, Any]: + value = exact_keys( + value, + { + "child_digest", + "child_ref", + "child_status", + "formula", + "homebrew_ref", + "homebrew_ref_status", + "observed_at", + "remote", + "top_ref", + "top_digest", + "top_status", + }, + "candidate destination evidence", + ) + if value["formula"] != receipt["formula"]: + fail("candidate destination names a different Formula") + require_string(value["observed_at"], "destination observation time", maximum=128) + expected_remote = ( + f"ghcr.io/{receipt['tap_repository'].lower()}/{receipt['formula']}" + ) + if value["remote"].lower() != expected_remote: + fail("candidate destination remote is not canonical") + if value["child_ref"] != receipt["oci"]["transport_tag"]: + fail("candidate destination child ref differs from the OCI child") + if value["homebrew_ref"] != receipt["oci"]["homebrew_ref"]: + fail("candidate destination Homebrew ref differs from the OCI child") + if value["top_ref"] != receipt["top_ref"]: + fail("candidate destination top ref differs from the OCI child") + if value["child_status"] == "missing": + if value["child_digest"] is not None: + fail("missing candidate child unexpectedly has a digest") + elif value["child_status"] == "present": + digest = require_string( + value["child_digest"], "candidate child digest", OCI_DIGEST + ) + if digest != receipt["oci"]["manifest"]["digest"]: + fail("candidate transport ref contains different OCI bytes") + else: + fail("candidate child ref has an ambiguous registry status") + # WHY: the content-addressed transport tag is not Homebrew's selection + # key. The sealer separately proves the live top index can accept this + # version/rebuild ref without replacing another ABI's bytes. + if value["homebrew_ref_status"] != "available": + fail("candidate Formula version/rebuild destination is not collision-free") + if value["top_status"] == "missing": + if value["top_digest"] is not None: + fail("missing candidate top ref unexpectedly has a digest") + elif value["top_status"] == "present": + require_string(value["top_digest"], "candidate top digest", OCI_DIGEST) + else: + fail("candidate top ref has an ambiguous registry status") + return value + + +def validate_build_and_oci( + build_root: pathlib.Path, + oci_root: pathlib.Path, + source: dict[str, Any], +) -> tuple[dict[str, Any], dict[str, Any], pathlib.Path]: + build_root = real_directory(build_root, "candidate build handoff") + oci_root = real_directory(oci_root, "candidate OCI child") + actual_build = sorted(path.name for path in build_root.iterdir()) + if actual_build != sorted(item[0] for item in BUILD_FILES): + fail("candidate build handoff has an unexpected file inventory") + for path in build_root.iterdir(): + regular_file(path, f"candidate build file {path.name}") + manifest, _ = load_json(build_root / "manifest.json", "build manifest") + manifest = exact_keys( + manifest, + { + "arch", + "bottle", + "bottle_root_url", + "dependency_provenance", + "formula", + "kandelo_commit", + "release_tag", + "schema", + "tap_checkout_commit", + "tap_commit", + "tap_name", + "tap_repository", + }, + "build manifest", + ) + if manifest["schema"] != 4: + fail("build manifest has an unsupported schema") + formula = require_string(manifest["formula"], "candidate Formula", FORMULA) + arch = require_string(manifest["arch"], "candidate architecture") + if arch not in ("wasm32", "wasm64"): + fail("candidate architecture is invalid") + if ( + manifest["release_tag"] != source["release_tag"] + or manifest["kandelo_commit"] != source["producer_commit"] + or manifest["tap_commit"] != source["tap_commit"] + or manifest["tap_checkout_commit"] != source["tap_checkout_commit"] + or manifest["tap_repository"].lower() != source["tap_repository"].lower() + or manifest["tap_name"].lower() != source["tap_name"].lower() + ): + fail("build manifest differs from candidate source evidence") + bottle = exact_keys( + manifest["bottle"], + {"archive", "bytes", "cellar", "json", "sha256", "tag"}, + "build bottle", + ) + if bottle["archive"] != "bottle.tar.gz" or bottle["json"] != "bottle.json": + fail("candidate build bottle paths are not canonical") + bottle_path = regular_file(build_root / "bottle.tar.gz", "candidate bottle") + if ( + require_int(bottle["bytes"], "candidate bottle bytes", 1) + != bottle_path.stat().st_size + or require_string(bottle["sha256"], "candidate bottle SHA-256", SHA256) + != sha256_file(bottle_path) + ): + fail("candidate bottle differs from its build manifest") + dependency = exact_keys( + manifest["dependency_provenance"], + {"bytes", "json", "sha256"}, + "build dependency provenance", + ) + if dependency["json"] != "dependency-provenance.json": + fail("candidate dependency provenance path is not canonical") + dependency_path = regular_file( + build_root / "dependency-provenance.json", "candidate dependency provenance" + ) + if ( + dependency["bytes"] != dependency_path.stat().st_size + or dependency["sha256"] != sha256_file(dependency_path) + ): + fail("candidate dependency provenance differs from its manifest") + + receipt, _ = load_json(oci_root / "receipt.json", "OCI child receipt") + receipt = exact_keys( + receipt, + { + "abi", + "arch", + "bottle", + "bottle_rebuild", + "formula", + "formula_revision", + "formula_source_identity_sha256", + "formula_source_sha256", + "kandelo_commit", + "kind", + "oci", + "pkg_version", + "schema", + "source_closure_sha256", + "tap_commit", + "tap_name", + "tap_repository", + "top_ref", + }, + "OCI child receipt", + ) + if receipt["schema"] != 2 or receipt["kind"] != "child": + fail("OCI child receipt has an unsupported contract") + if ( + receipt["formula"] != formula + or receipt["arch"] != arch + or receipt["abi"] != source["abi"] + or receipt["kandelo_commit"] != source["producer_commit"] + or receipt["tap_commit"] != source["tap_commit"] + or receipt["tap_repository"].lower() != source["tap_repository"].lower() + or receipt["tap_name"].lower() != source["tap_name"].lower() + ): + fail("OCI child receipt differs from the candidate source") + receipt_bottle = exact_keys( + receipt["bottle"], {"bytes", "sha256", "url"}, "OCI receipt bottle" + ) + if ( + receipt_bottle["bytes"] != bottle["bytes"] + or receipt_bottle["sha256"] != bottle["sha256"] + ): + fail("OCI child receipt names different bottle bytes") + require_string(receipt["pkg_version"], "candidate package version", PKG_VERSION) + require_int(receipt["formula_revision"], "candidate Formula revision") + require_int(receipt["bottle_rebuild"], "candidate bottle rebuild") + require_string( + receipt["formula_source_identity_sha256"], + "candidate Formula identity SHA-256", + SHA256, + ) + require_string( + receipt["formula_source_sha256"], "candidate Formula SHA-256", SHA256 + ) + require_string( + receipt["source_closure_sha256"], "candidate source closure SHA-256", SHA256 + ) + oci = exact_keys( + receipt["oci"], + {"config", "diff_id", "homebrew_ref", "manifest", "platform", "transport_tag"}, + "OCI child identity", + ) + require_string(oci["homebrew_ref"], "OCI child ref", OCI_TAG) + require_string(oci["transport_tag"], "OCI transport tag", OCI_TAG) + manifest_descriptor = exact_keys( + oci["manifest"], {"digest", "size"}, "OCI manifest descriptor" + ) + manifest_match = OCI_DIGEST.fullmatch( + require_string(manifest_descriptor["digest"], "OCI manifest digest") + ) + assert manifest_match is not None + require_int(manifest_descriptor["size"], "OCI manifest bytes", 1) + if oci["transport_tag"] != f"sha256-{manifest_match.group(1)}": + fail("OCI transport tag is not content-derived") + require_string(receipt["top_ref"], "OCI top ref", OCI_TAG) + + layout = real_directory(oci_root / "layout", "candidate OCI layout") + expected_static = {"index.json", "oci-layout", "blobs"} + if {path.name for path in layout.iterdir()} != expected_static: + fail("candidate OCI layout has an unexpected top-level inventory") + regular_file(layout / "index.json", "candidate OCI index", MAX_JSON_BYTES) + regular_file(layout / "oci-layout", "candidate OCI marker", MAX_JSON_BYTES) + blob_root = real_directory(layout / "blobs", "candidate OCI blob root") + if {path.name for path in blob_root.iterdir()} != {"sha256"}: + fail("candidate OCI blob root is not canonical") + sha_root = real_directory(blob_root / "sha256", "candidate OCI SHA-256 root") + blobs = sorted(sha_root.iterdir(), key=lambda path: path.name) + if not blobs or len(blobs) > MAX_OCI_BLOBS: + fail("candidate OCI blob inventory is empty or too large") + for blob in blobs: + require_string(blob.name, "candidate OCI blob name", SHA256) + regular_file(blob, f"candidate OCI blob {blob.name}") + if sha256_file(blob) != blob.name: + fail("candidate OCI blob name differs from its bytes") + required_blobs = { + bottle["sha256"], + manifest_match.group(1), + } + config = exact_keys( + oci["config"], + {"digest", "mediaType", "size"}, + "OCI config descriptor", + ) + if config["mediaType"] != "application/vnd.oci.image.config.v1+json": + fail("OCI config descriptor has the wrong media type") + config_match = OCI_DIGEST.fullmatch( + require_string(config["digest"], "OCI config digest") + ) + assert config_match is not None + require_int(config["size"], "OCI config bytes", 1) + required_blobs.add(config_match.group(1)) + if not required_blobs.issubset({blob.name for blob in blobs}): + fail("candidate OCI layout lacks a receipt-bound blob") + return manifest, receipt, layout + + +def copy_assets( + build_root: pathlib.Path, + oci_root: pathlib.Path, + package_input_path: pathlib.Path, + assets: pathlib.Path, +) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for source_name, asset_name in BUILD_FILES: + source = build_root / source_name + destination = assets / asset_name + shutil.copyfile(source, destination) + records.append(file_record(destination, f"build/{source_name}", asset_name)) + for source, logical, asset_name in ( + (oci_root / "receipt.json", "oci/receipt.json", "oci-receipt.json"), + (oci_root / "layout/oci-layout", "oci/layout/oci-layout", "oci-layout.json"), + (oci_root / "layout/index.json", "oci/layout/index.json", "oci-index.json"), + (package_input_path, "package-input.json", "package-input.json"), + ): + destination = assets / asset_name + shutil.copyfile(source, destination) + records.append(file_record(destination, logical, asset_name)) + sha_root = oci_root / "layout/blobs/sha256" + for source in sorted(sha_root.iterdir(), key=lambda path: path.name): + asset_name = f"oci-blob-{source.name}" + destination = assets / asset_name + shutil.copyfile(source, destination) + records.append( + file_record( + destination, + f"oci/layout/blobs/sha256/{source.name}", + asset_name, + ) + ) + return sorted(records, key=lambda record: record["path"]) + + +def candidate_tag(manifest_payload: bytes, manifest: dict[str, Any]) -> str: + return ( + "homebrew-bottle-candidate-pr-" + f"{manifest['source']['pr_number']}-run-" + f"{manifest['run']['run_id']}-attempt-" + f"{manifest['run']['run_attempt']}-sha256-" + f"{sha256_bytes(manifest_payload)}" + ) + + +def validate_candidate_manifest( + value: Any, payload: bytes, expected_tag: str | None = None +) -> dict[str, Any]: + value = exact_keys( + value, + { + "dependencies", + "destination", + "files", + "formula", + "kind", + "package_input", + "run", + "schema", + "source", + }, + "Homebrew bottle candidate", + ) + if value["schema"] != 1 or value["kind"] != "kandelo-homebrew-bottle-candidate": + fail("Homebrew bottle candidate has an unsupported contract") + source = validate_source_evidence(value["source"]) + formula = exact_keys( + value["formula"], + { + "arch", + "bottle_rebuild", + "formula_revision", + "formula_source_identity_sha256", + "formula_source_sha256", + "name", + "pkg_version", + "source_closure_sha256", + }, + "candidate Formula", + ) + name = require_string(formula["name"], "candidate Formula name", FORMULA) + if formula["arch"] not in ("wasm32", "wasm64"): + fail("candidate Formula architecture is invalid") + require_string(formula["pkg_version"], "candidate Formula version", PKG_VERSION) + require_int(formula["formula_revision"], "candidate Formula revision") + require_int(formula["bottle_rebuild"], "candidate bottle rebuild") + for key in ( + "formula_source_identity_sha256", + "formula_source_sha256", + "source_closure_sha256", + ): + require_string(formula[key], f"candidate {key}", SHA256) + dependencies = validate_dependencies(value["dependencies"]) + if dependencies: + fail("candidate schema 1 is restricted to leaf Formulae") + run = validate_run_evidence(value["run"], name, formula["arch"]) + if normalized_repository(run["repository"]) != normalized_repository( + source["tap_repository"] + ): + fail("candidate run repository differs from the tap source") + package_record = exact_keys( + value["package_input"], {"bytes", "sha256"}, "candidate package input record" + ) + require_int(package_record["bytes"], "candidate package input bytes", 1) + require_string(package_record["sha256"], "candidate package input SHA-256", SHA256) + files = value["files"] + if not isinstance(files, list) or not files: + fail("candidate file inventory is empty") + paths: list[str] = [] + assets: set[str] = set() + total = len(payload) + for position, record in enumerate(files): + record = exact_keys( + record, + {"asset_name", "bytes", "path", "sha256"}, + f"candidate file #{position}", + ) + path = require_string(record["path"], "candidate logical path", maximum=512) + if ( + path.startswith("/") + or "\\" in path + or any(part in ("", ".", "..") for part in path.split("/")) + ): + fail("candidate logical path is unsafe") + asset = require_string(record["asset_name"], "candidate asset name", maximum=255) + if "/" in asset or "\\" in asset or asset in assets: + fail("candidate release asset is unsafe or duplicated") + assets.add(asset) + byte_count = require_int(record["bytes"], "candidate asset bytes", 1) + require_string(record["sha256"], "candidate asset SHA-256", SHA256) + total += byte_count + if total > MAX_TOTAL_BYTES: + fail("candidate release exceeds its aggregate byte bound") + paths.append(path) + if paths != sorted(set(paths)): + fail("candidate file inventory must be unique and sorted") + fixed_paths = { + *(f"build/{name}" for name, _asset in BUILD_FILES), + "oci/receipt.json", + "oci/layout/oci-layout", + "oci/layout/index.json", + "package-input.json", + } + path_set = set(paths) + if not fixed_paths.issubset(path_set): + fail("candidate file inventory omits a required handoff file") + blob_paths = path_set - fixed_paths + if not blob_paths or len(blob_paths) > MAX_OCI_BLOBS or any( + re.fullmatch(r"oci/layout/blobs/sha256/[0-9a-f]{64}", path) is None + for path in blob_paths + ): + fail("candidate file inventory has an invalid OCI blob set") + if expected_tag is not None: + match = CANDIDATE_TAG.fullmatch(expected_tag) + if match is None: + fail("candidate release tag is invalid") + if ( + int(match.group(1)) != source["pr_number"] + or int(match.group(2)) != run["run_id"] + or int(match.group(3)) != run["run_attempt"] + or match.group(4) != sha256_bytes(payload) + ): + fail("candidate release tag differs from candidate.json") + return value + + +def describe_release(arguments: argparse.Namespace) -> None: + candidate_path = pathlib.Path(arguments.candidate) + value, payload = load_json(candidate_path, "candidate.json") + manifest = validate_candidate_manifest(value, payload, arguments.candidate_tag) + output = pathlib.Path(arguments.out) + if output.exists() or output.is_symlink(): + fail("candidate release description output must not already exist") + output.parent.mkdir(parents=True, exist_ok=True) + description = { + "schema": 1, + "kind": "kandelo-homebrew-bottle-candidate-release-description", + "candidate_tag": arguments.candidate_tag, + "candidate_sha256": sha256_bytes(payload), + "assets": [ + { + "asset_name": record["asset_name"], + "bytes": record["bytes"], + "sha256": record["sha256"], + } + for record in manifest["files"] + ], + "manifest": manifest, + } + output.write_bytes(pretty_json(description)) + + +def prepare(arguments: argparse.Namespace) -> None: + source, _ = load_json(pathlib.Path(arguments.source), "candidate source evidence") + source = validate_source_evidence(source) + package_input_path = regular_file( + pathlib.Path(arguments.package_input), "candidate package input", MAX_JSON_BYTES + ) + package_input, package_payload = load_json( + package_input_path, "candidate package input" + ) + package_input = validate_package_input(package_input) + # WHY: admission later compares the exact released package ledger bytes. + # One canonical encoding prevents semantically equal JSON from acquiring + # several different identities across pre-merge and post-merge jobs. + if package_payload != pretty_json(package_input): + fail("candidate package input must use canonical pretty JSON") + if ( + package_input["producer_commit"] != source["producer_commit"] + or package_input["abi"] != source["abi"] + or package_input["staging_release"]["pr_number"] != source["pr_number"] + ): + fail("candidate package input differs from candidate source") + dependencies, _ = load_json( + pathlib.Path(arguments.dependencies), "candidate dependencies" + ) + dependencies = validate_dependencies(dependencies) + if dependencies: + fail("candidate schema 1 is restricted to leaf Formulae") + build_root = real_directory( + pathlib.Path(arguments.build_handoff), "candidate build handoff" + ) + oci_root = real_directory(pathlib.Path(arguments.oci_child), "candidate OCI child") + build, receipt, _layout = validate_build_and_oci(build_root, oci_root, source) + run, _ = load_json(pathlib.Path(arguments.run_evidence), "candidate run evidence") + run = validate_run_evidence(run, receipt["formula"], receipt["arch"]) + destination, _ = load_json( + pathlib.Path(arguments.destination), "candidate destination evidence" + ) + destination = validate_destination(destination, receipt) + + output = pathlib.Path(arguments.out) + if output.exists() or output.is_symlink(): + fail("candidate output must not already exist") + output.parent.mkdir(parents=True, exist_ok=True) + temporary = pathlib.Path( + tempfile.mkdtemp(prefix=f".{output.name}.", dir=output.parent) + ) + try: + assets = temporary / "assets" + assets.mkdir() + file_records = copy_assets(build_root, oci_root, package_input_path, assets) + package_record = next( + record for record in file_records if record["path"] == "package-input.json" + ) + if ( + package_record["bytes"] != len(package_payload) + or package_record["sha256"] != sha256_bytes(package_payload) + ): + fail("copied candidate package input changed") + manifest = { + "schema": 1, + "kind": "kandelo-homebrew-bottle-candidate", + "source": source, + "run": run, + "formula": { + "name": receipt["formula"], + "arch": receipt["arch"], + "pkg_version": receipt["pkg_version"], + "formula_revision": receipt["formula_revision"], + "bottle_rebuild": receipt["bottle_rebuild"], + "formula_source_identity_sha256": receipt[ + "formula_source_identity_sha256" + ], + "formula_source_sha256": receipt["formula_source_sha256"], + "source_closure_sha256": receipt["source_closure_sha256"], + }, + "destination": destination, + "dependencies": dependencies, + "package_input": { + "bytes": package_record["bytes"], + "sha256": package_record["sha256"], + }, + "files": file_records, + } + payload = pretty_json(manifest) + validate_candidate_manifest(manifest, payload) + (assets / "candidate.json").write_bytes(payload) + tag = candidate_tag(payload, manifest) + release_assets = [ + { + "name": path.name, + "bytes": path.stat().st_size, + "sha256": sha256_file(path), + } + for path in sorted(assets.iterdir(), key=lambda path: path.name) + ] + release_names = [asset["name"] for asset in release_assets] + release = { + "schema": 1, + "repository": source["tap_repository"], + "tag": tag, + "target_commitish": run["caller_commit"], + "title": ( + f"Kandelo Homebrew candidate: {receipt['formula']}/" + f"{receipt['arch']}" + ), + "body": ( + f"Run-bound, noncanonical bottle candidate for Kandelo PR " + f"#{source['pr_number']}." + ), + "assets": release_assets, + "preferred_asset_names": release_names, + "accepted_existing_asset_sets": [], + } + (temporary / "release-manifest.json").write_bytes(pretty_json(release)) + (temporary / "tag.txt").write_text(f"{tag}\n") + os.replace(temporary, output) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +def verify_release_assets(root: pathlib.Path, manifest: dict[str, Any]) -> None: + expected = {"candidate.json"} + total = 0 + for record in manifest["files"]: + expected.add(record["asset_name"]) + path = regular_file(root / record["asset_name"], record["path"]) + if path.stat().st_size != record["bytes"] or sha256_file(path) != record["sha256"]: + fail(f"candidate asset {record['asset_name']} differs from candidate.json") + total += path.stat().st_size + if total > MAX_TOTAL_BYTES: + fail("candidate assets exceed their aggregate byte bound") + actual = {path.name for path in root.iterdir()} + if actual != expected: + fail("candidate release contains a missing or unexpected asset") + + +def validate_completed_run( + value: Any, candidate_run: dict[str, Any], formula: str, arch: str +) -> None: + current = validate_run_evidence(value, formula, arch) + if current["status"] != "completed" or current["conclusion"] != "success": + fail("candidate workflow run has not completed successfully") + # Artifact IDs and digests are immutable identities. Require the whole + # record so a same-named artifact from a rerun cannot replace the input. + if current != {**candidate_run, "status": "completed", "conclusion": "success"}: + fail("candidate workflow evidence changed after release preparation") + + +def validate_merge( + root: pathlib.Path, + source: dict[str, Any], + merge_commit: str, + current_main: str, +) -> None: + require_string(merge_commit, "candidate merge commit", COMMIT) + require_string(current_main, "current Kandelo main", COMMIT) + root = exact_git_checkout(root, merge_commit, "Kandelo activation checkout") + # WHY: after merge, a branch ref and GitHub's PR head are mutable metadata. + # The merge object is immutable and already names both exact commits, so it + # is the stronger authority for deciding whether these bytes may survive. + parents = run_git(root, "show", "-s", "--format=%P", merge_commit) + expected_parents = f"{source['base_commit']} {source['producer_commit']}" + if parents != expected_parents: + fail("candidate merge does not preserve the prepared base and exact head") + producer_tree = run_git(root, "rev-parse", f"{source['producer_commit']}^{{tree}}") + merge_tree = run_git(root, "rev-parse", f"{merge_commit}^{{tree}}") + if producer_tree != source["producer_tree"] or merge_tree != producer_tree: + fail("candidate producer and merged trees are not identical") + require_ancestor(root, merge_commit, current_main, "candidate merge") + require_ancestor(root, source["producer_commit"], current_main, "candidate producer") + require_ancestor( + root, + source["workflow_authority_commit"], + current_main, + "candidate validator authority", + ) + + +def validate_tap_history( + root: pathlib.Path, + source: dict[str, Any], + run: dict[str, Any], + current_main: str, +) -> None: + root = exact_git_checkout( + root, + source["tap_checkout_commit"], + "prepared tap activation checkout", + ) + require_string(current_main, "current tap main", COMMIT) + if ( + run_git(root, "rev-parse", "HEAD^{tree}") + != source["tap_checkout_tree"] + ): + fail("prepared tap activation tree differs from the candidate") + require_ancestor( + root, + source["tap_commit"], + source["tap_checkout_commit"], + "prepared candidate tap source", + ) + require_ancestor(root, source["tap_commit"], current_main, "candidate tap source") + require_ancestor( + root, + run["caller_commit"], + current_main, + "candidate tap workflow authority", + ) + + +def copy_materialized_assets( + root: pathlib.Path, + manifest: dict[str, Any], + build_output: pathlib.Path, + oci_output: pathlib.Path, + package_output: pathlib.Path, +) -> None: + if any( + path.exists() or path.is_symlink() + for path in (build_output, oci_output, package_output) + ): + fail("materialized candidate outputs must not already exist") + build_output.parent.mkdir(parents=True, exist_ok=True) + oci_output.parent.mkdir(parents=True, exist_ok=True) + build_tmp = pathlib.Path( + tempfile.mkdtemp(prefix=f".{build_output.name}.", dir=build_output.parent) + ) + oci_tmp = pathlib.Path( + tempfile.mkdtemp(prefix=f".{oci_output.name}.", dir=oci_output.parent) + ) + try: + (oci_tmp / "layout/blobs/sha256").mkdir(parents=True) + by_path = {record["path"]: record for record in manifest["files"]} + for source_name, _asset_name in BUILD_FILES: + record = by_path[f"build/{source_name}"] + shutil.copyfile(root / record["asset_name"], build_tmp / source_name) + for logical, destination in ( + ("oci/receipt.json", oci_tmp / "receipt.json"), + ("oci/layout/oci-layout", oci_tmp / "layout/oci-layout"), + ("oci/layout/index.json", oci_tmp / "layout/index.json"), + ): + record = by_path[logical] + shutil.copyfile(root / record["asset_name"], destination) + blob_prefix = "oci/layout/blobs/sha256/" + for logical, record in by_path.items(): + if logical.startswith(blob_prefix): + digest = logical.removeprefix(blob_prefix) + shutil.copyfile( + root / record["asset_name"], + oci_tmp / "layout/blobs/sha256" / digest, + ) + package_record = by_path["package-input.json"] + package_output.parent.mkdir(parents=True, exist_ok=True) + package_tmp = package_output.with_name(f".{package_output.name}.tmp") + if package_tmp.exists() or package_tmp.is_symlink(): + fail("candidate package temporary output is occupied") + shutil.copyfile(root / package_record["asset_name"], package_tmp) + os.replace(build_tmp, build_output) + os.replace(oci_tmp, oci_output) + os.replace(package_tmp, package_output) + except Exception: + shutil.rmtree(build_tmp, ignore_errors=True) + shutil.rmtree(oci_tmp, ignore_errors=True) + package_tmp = package_output.with_name(f".{package_output.name}.tmp") + package_tmp.unlink(missing_ok=True) + raise + + +def materialize(arguments: argparse.Namespace) -> None: + root = real_directory(pathlib.Path(arguments.candidate_root), "candidate release") + candidate_path = regular_file(root / "candidate.json", "candidate.json", MAX_JSON_BYTES) + manifest, payload = load_json(candidate_path, "candidate.json") + manifest = validate_candidate_manifest(manifest, payload, arguments.candidate_tag) + verify_release_assets(root, manifest) + package_record = manifest["package_input"] + package_asset = next( + record for record in manifest["files"] if record["path"] == "package-input.json" + ) + if ( + package_record["bytes"] != package_asset["bytes"] + or package_record["sha256"] != package_asset["sha256"] + ): + fail("candidate package input record differs from its release asset") + package_input, _ = load_json( + root / package_asset["asset_name"], "released candidate package input" + ) + package_input = validate_package_input(package_input) + admitted, admitted_payload = load_json( + pathlib.Path(arguments.admitted_package_input), "admitted package input" + ) + admitted = exact_keys( + admitted, + { + "candidate_package_input_sha256", + "kind", + "package_input", + "schema", + "validated_against_main", + }, + "admitted package input", + ) + if ( + admitted["schema"] != 1 + or admitted["kind"] != "kandelo-homebrew-admitted-candidate-package-input" + ): + fail("admitted package input has an unsupported contract") + require_string( + admitted["validated_against_main"], "package validation main commit", COMMIT + ) + require_string( + admitted["candidate_package_input_sha256"], + "admitted candidate package input SHA-256", + SHA256, + ) + if ( + admitted["candidate_package_input_sha256"] != sha256_bytes(pretty_json(package_input)) + or admitted["package_input"] != package_input + ): + fail("admitted package generation does not contain the candidate archives") + completed_run, _ = load_json( + pathlib.Path(arguments.completed_run_evidence), "completed candidate run" + ) + validate_completed_run( + completed_run, + manifest["run"], + manifest["formula"]["name"], + manifest["formula"]["arch"], + ) + validate_merge( + pathlib.Path(arguments.kandelo_root), + manifest["source"], + arguments.merge_commit, + arguments.current_kandelo_main, + ) + if admitted["validated_against_main"] != arguments.merge_commit: + fail("package input was not admitted by the exact candidate merge") + validate_tap_history( + pathlib.Path(arguments.tap_root), + manifest["source"], + manifest["run"], + arguments.current_tap_main, + ) + if arguments.dependencies: + dependencies, _ = load_json( + pathlib.Path(arguments.dependencies), "activation dependencies" + ) + if validate_dependencies(dependencies) != manifest["dependencies"]: + fail("activation dependencies differ from the bottle candidate") + elif manifest["dependencies"]: + fail("activation omitted candidate dependencies") + copy_materialized_assets( + root, + manifest, + pathlib.Path(arguments.out_build_handoff), + pathlib.Path(arguments.out_oci_child), + pathlib.Path(arguments.out_package_input), + ) + # Re-run the same structural cross-check after reconstruction. The + # protected workflow separately runs the complete handoff and Wasm + # validators before exposing GHCR credentials. + _build, reconstructed_receipt, _layout = validate_build_and_oci( + pathlib.Path(arguments.out_build_handoff), + pathlib.Path(arguments.out_oci_child), + manifest["source"], + ) + expected_formula = { + "name": reconstructed_receipt["formula"], + "arch": reconstructed_receipt["arch"], + "pkg_version": reconstructed_receipt["pkg_version"], + "formula_revision": reconstructed_receipt["formula_revision"], + "bottle_rebuild": reconstructed_receipt["bottle_rebuild"], + "formula_source_identity_sha256": reconstructed_receipt[ + "formula_source_identity_sha256" + ], + "formula_source_sha256": reconstructed_receipt["formula_source_sha256"], + "source_closure_sha256": reconstructed_receipt["source_closure_sha256"], + } + if manifest["formula"] != expected_formula: + fail("candidate Formula identity differs from its OCI receipt") + validate_destination(manifest["destination"], reconstructed_receipt) + receipt = { + "schema": 1, + "kind": "kandelo-homebrew-bottle-candidate-promotion", + "candidate_tag": arguments.candidate_tag, + "candidate_sha256": sha256_bytes(payload), + "source": manifest["source"], + "merge_commit": arguments.merge_commit, + "validated_against_main": admitted["validated_against_main"], + "run": manifest["run"], + "formula": manifest["formula"], + "package_input_sha256": package_record["sha256"], + "admission_sha256": sha256_bytes(admitted_payload), + "dependencies": manifest["dependencies"], + "files": manifest["files"], + } + output = pathlib.Path(arguments.out_receipt) + if output.exists() or output.is_symlink(): + fail("promotion receipt output must not already exist") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(pretty_json(receipt)) + + +def validate_promotion(arguments: argparse.Namespace) -> None: + value, _ = load_json( + pathlib.Path(arguments.receipt), "candidate promotion receipt" + ) + value = exact_keys( + value, + { + "admission_sha256", + "candidate_sha256", + "candidate_tag", + "dependencies", + "files", + "formula", + "kind", + "merge_commit", + "package_input_sha256", + "run", + "schema", + "source", + "validated_against_main", + }, + "candidate promotion receipt", + ) + if ( + value["schema"] != 1 + or value["kind"] != "kandelo-homebrew-bottle-candidate-promotion" + ): + fail("candidate promotion receipt has an unsupported contract") + candidate_tag_value = require_string( + value["candidate_tag"], "promoted candidate tag" + ) + candidate_digest = require_string( + value["candidate_sha256"], "promoted candidate SHA-256", SHA256 + ) + match = CANDIDATE_TAG.fullmatch(candidate_tag_value) + if ( + match is None + or candidate_tag_value != arguments.candidate_tag + or match.group(4) != candidate_digest + ): + fail("candidate promotion receipt names another candidate") + source = validate_source_evidence(value["source"]) + if ( + source["producer_commit"] != arguments.producer_commit + or source["tap_commit"] != arguments.tap_commit + or source["tap_checkout_commit"] != arguments.tap_checkout_commit + or source["prefix_campaign_tag"] != arguments.campaign_tag + or source["prefix_campaign_layout_sha256"] + != arguments.campaign_layout_sha256 + ): + fail("candidate promotion source differs from the publication plan") + merge_commit = require_string( + value["merge_commit"], "candidate promotion merge", COMMIT + ) + if ( + merge_commit != arguments.merge_commit + or value["validated_against_main"] != merge_commit + ): + fail("candidate promotion was not admitted by this exact merge") + if validate_dependencies(value["dependencies"]): + fail("candidate promotion v1 is restricted to leaf Formulae") + formula = value["formula"] + if ( + not isinstance(formula, dict) + or formula.get("name") != arguments.formula + or formula.get("arch") != arguments.arch + ): + fail("candidate promotion Formula differs from the publication plan") + run = validate_run_evidence( + value["run"], arguments.formula, arguments.arch + ) + if ( + int(match.group(2)) != run["run_id"] + or int(match.group(3)) != run["run_attempt"] + ): + fail("candidate promotion receipt names another workflow run") + require_string( + value["package_input_sha256"], + "promoted package input SHA-256", + SHA256, + ) + package_path = regular_file( + pathlib.Path(arguments.package_input), + "promoted candidate package input", + MAX_JSON_BYTES, + ) + package_input, package_payload = load_json( + package_path, "promoted candidate package input" + ) + package_input = validate_package_input(package_input) + if package_payload != pretty_json(package_input): + fail("promoted candidate package input is not canonical JSON") + if value["package_input_sha256"] != sha256_bytes(package_payload): + fail("promoted candidate package input differs from its receipt") + require_string( + value["admission_sha256"], + "promoted package admission SHA-256", + SHA256, + ) + _build, child, _layout = validate_build_and_oci( + pathlib.Path(arguments.build_handoff), + pathlib.Path(arguments.oci_child), + source, + ) + expected_formula = { + "name": child["formula"], + "arch": child["arch"], + "pkg_version": child["pkg_version"], + "formula_revision": child["formula_revision"], + "bottle_rebuild": child["bottle_rebuild"], + "formula_source_identity_sha256": child[ + "formula_source_identity_sha256" + ], + "formula_source_sha256": child["formula_source_sha256"], + "source_closure_sha256": child["source_closure_sha256"], + } + if formula != expected_formula: + fail("candidate promotion Formula differs from its exact OCI child") + records = value["files"] + if not isinstance(records, list): + fail("candidate promotion file inventory is invalid") + by_path: dict[str, dict[str, Any]] = {} + for position, record in enumerate(records): + record = exact_keys( + record, + {"asset_name", "bytes", "path", "sha256"}, + f"candidate promotion file #{position}", + ) + logical = require_string( + record["path"], "candidate promotion logical path", maximum=512 + ) + if logical in by_path: + fail("candidate promotion file inventory repeats a path") + require_int(record["bytes"], "candidate promotion file bytes", 1) + require_string( + record["sha256"], "candidate promotion file SHA-256", SHA256 + ) + by_path[logical] = record + build_root = pathlib.Path(arguments.build_handoff) + oci_root = pathlib.Path(arguments.oci_child) + current_files = { + **{ + f"build/{name}": build_root / name + for name, _asset in BUILD_FILES + }, + "oci/receipt.json": oci_root / "receipt.json", + "oci/layout/oci-layout": oci_root / "layout/oci-layout", + "oci/layout/index.json": oci_root / "layout/index.json", + "package-input.json": package_path, + **{ + f"oci/layout/blobs/sha256/{path.name}": path + for path in (oci_root / "layout/blobs/sha256").iterdir() + }, + } + if set(current_files) != set(by_path): + fail("candidate promotion receipt differs from the exact artifact files") + for logical, current in current_files.items(): + record = by_path[logical] + if ( + record["bytes"] != current.stat().st_size + or record["sha256"] != sha256_file(current) + ): + fail(f"candidate promotion artifact {logical} changed") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + + describe_parser = commands.add_parser("describe-release") + describe_parser.add_argument("--candidate", required=True) + describe_parser.add_argument("--candidate-tag", required=True) + describe_parser.add_argument("--out", required=True) + + source_parser = commands.add_parser("describe-source") + source_parser.add_argument("--root", required=True) + source_parser.add_argument("--producer-commit", required=True) + source_parser.add_argument("--out", required=True) + + package_parser = commands.add_parser("package-input") + package_parser.add_argument("--expected-ledger", required=True) + package_parser.add_argument("--snapshot", required=True) + package_parser.add_argument("--release-evidence", required=True) + package_parser.add_argument("--index", required=True) + package_parser.add_argument("--producer-commit", required=True) + package_parser.add_argument("--abi", required=True, type=int) + package_parser.add_argument("--out", required=True) + + admission_parser = commands.add_parser("admit-package-input") + admission_parser.add_argument("--candidate-package-input", required=True) + admission_parser.add_argument("--regenerated-package-input", required=True) + admission_parser.add_argument("--validated-main-root", required=True) + admission_parser.add_argument("--validated-main", required=True) + admission_parser.add_argument("--producer-commit", required=True) + admission_parser.add_argument("--out", required=True) + + prepare_parser = commands.add_parser("prepare") + prepare_parser.add_argument("--source", required=True) + prepare_parser.add_argument("--run-evidence", required=True) + prepare_parser.add_argument("--destination", required=True) + prepare_parser.add_argument("--dependencies", required=True) + prepare_parser.add_argument("--package-input", required=True) + prepare_parser.add_argument("--build-handoff", required=True) + prepare_parser.add_argument("--oci-child", required=True) + prepare_parser.add_argument("--out", required=True) + + materialize_parser = commands.add_parser("materialize") + materialize_parser.add_argument("--candidate-root", required=True) + materialize_parser.add_argument("--candidate-tag", required=True) + materialize_parser.add_argument("--completed-run-evidence", required=True) + materialize_parser.add_argument("--kandelo-root", required=True) + materialize_parser.add_argument("--tap-root", required=True) + materialize_parser.add_argument("--merge-commit", required=True) + materialize_parser.add_argument("--current-kandelo-main", required=True) + materialize_parser.add_argument("--current-tap-main", required=True) + materialize_parser.add_argument("--admitted-package-input", required=True) + materialize_parser.add_argument("--dependencies") + materialize_parser.add_argument("--out-build-handoff", required=True) + materialize_parser.add_argument("--out-oci-child", required=True) + materialize_parser.add_argument("--out-package-input", required=True) + materialize_parser.add_argument("--out-receipt", required=True) + + promotion_parser = commands.add_parser("validate-promotion") + promotion_parser.add_argument("--receipt", required=True) + promotion_parser.add_argument("--candidate-tag", required=True) + promotion_parser.add_argument("--producer-commit", required=True) + promotion_parser.add_argument("--merge-commit", required=True) + promotion_parser.add_argument("--tap-commit", required=True) + promotion_parser.add_argument("--tap-checkout-commit", required=True) + promotion_parser.add_argument("--campaign-tag", required=True) + promotion_parser.add_argument("--campaign-layout-sha256", required=True) + promotion_parser.add_argument("--formula", required=True) + promotion_parser.add_argument( + "--arch", choices=("wasm32", "wasm64"), required=True + ) + promotion_parser.add_argument("--build-handoff", required=True) + promotion_parser.add_argument("--oci-child", required=True) + promotion_parser.add_argument("--package-input", required=True) + return parser.parse_args() + + +def main() -> int: + arguments = parse_args() + try: + if arguments.command == "describe-release": + describe_release(arguments) + elif arguments.command == "describe-source": + describe_source(arguments) + elif arguments.command == "package-input": + create_package_input(arguments) + elif arguments.command == "admit-package-input": + admit_package_input(arguments) + elif arguments.command == "prepare": + prepare(arguments) + elif arguments.command == "materialize": + materialize(arguments) + else: + validate_promotion(arguments) + except (CandidateError, OSError, subprocess.SubprocessError) as error: + print(f"homebrew-bottle-candidate: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/homebrew-candidate-caller-pins.py b/scripts/homebrew-candidate-caller-pins.py new file mode 100755 index 0000000000..f838287be1 --- /dev/null +++ b/scripts/homebrew-candidate-caller-pins.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Render and validate immutable Kandelo candidate workflow callers.""" + +from __future__ import annotations + +import argparse +import pathlib +import re +import shutil +import sys +import tempfile +from typing import NoReturn + + +COMMIT = re.compile(r"^[0-9a-f]{40}$") +BASE_TOKEN = "__KANDELO_CANDIDATE_BASE_SHA__" +MERGE_TOKEN = "__KANDELO_CANDIDATE_MERGE_SHA__" +WORKFLOW_ROOT = pathlib.Path(".github/workflows") +CALLERS = { + "campaign": ( + "candidate-campaign.yml", + ["reusable-homebrew-candidate-campaign.yml"], + ), + "bottle": ( + "candidate-bottles.yml", + ["reusable-homebrew-bottle-publish.yml"], + ), + "promotion": ( + "promote-candidate-bottle.yml", + [ + "reusable-homebrew-bottle-candidate-materialize.yml", + "reusable-homebrew-bottle-publish.yml", + ], + ), +} +USES = re.compile( + r"^[ ]+uses: Automattic/kandelo/\.github/workflows/" + r"([A-Za-z0-9._-]+)@([^\s]+)[ ]*$", + re.MULTILINE, +) + + +class CallerPinError(ValueError): + """A candidate caller did not satisfy the immutable pin contract.""" + + +def fail(message: str) -> NoReturn: + raise CallerPinError(message) + + +def require_commit(value: str, label: str) -> str: + if COMMIT.fullmatch(value) is None: + fail(f"{label} must be an exact lowercase commit SHA") + return value + + +def real_root(path: pathlib.Path, label: str) -> pathlib.Path: + if path.is_symlink() or not path.is_dir(): + fail(f"{label} must be a real directory") + return path.resolve() + + +def read_caller(root: pathlib.Path, mode: str) -> tuple[pathlib.Path, str]: + filename, _expected = CALLERS[mode] + path = root / WORKFLOW_ROOT / filename + if path.is_symlink() or not path.is_file(): + fail(f"{mode} caller must be a regular file") + if path.stat().st_size > 256 * 1024: + fail(f"{mode} caller exceeds its byte bound") + return path, path.read_text(encoding="utf-8") + + +def validate_text(text: str, mode: str, expected_sha: str) -> None: + expected_sha = require_commit(expected_sha, "candidate caller authority") + _filename, expected_workflows = CALLERS[mode] + found = USES.findall(text) + wanted = [(workflow, expected_sha) for workflow in expected_workflows] + if found != wanted: + fail( + f"{mode} caller must pin exactly {expected_workflows} to " + f"{expected_sha}" + ) + if "@main" in text or BASE_TOKEN in text or MERGE_TOKEN in text: + fail(f"{mode} caller still contains a mutable or unresolved authority") + + +def validate(arguments: argparse.Namespace) -> None: + root = real_root(pathlib.Path(arguments.tap_root), "candidate caller root") + _path, text = read_caller(root, arguments.mode) + validate_text(text, arguments.mode, arguments.kandelo_sha) + + +def render(arguments: argparse.Namespace) -> None: + template = real_root( + pathlib.Path(arguments.template_root), "candidate caller template root" + ) + output = pathlib.Path(arguments.out) + if output.exists() or output.is_symlink(): + fail("rendered candidate caller output already exists") + base = require_commit(arguments.base_sha, "candidate base") + merge = require_commit(arguments.merge_sha, "candidate merge") + output.parent.mkdir(parents=True, exist_ok=True) + temporary = pathlib.Path( + tempfile.mkdtemp(prefix=f".{output.name}.", dir=output.parent) + ) + try: + destination = temporary / WORKFLOW_ROOT + destination.mkdir(parents=True) + for mode, (filename, _workflows) in CALLERS.items(): + source, text = read_caller(template, mode) + expected_token = MERGE_TOKEN if mode == "promotion" else BASE_TOKEN + unwanted_token = BASE_TOKEN if mode == "promotion" else MERGE_TOKEN + expected_count = 2 if mode == "promotion" else 1 + if text.count(expected_token) != expected_count or unwanted_token in text: + fail(f"{mode} caller template has an invalid placeholder contract") + rendered = text.replace(BASE_TOKEN, base).replace(MERGE_TOKEN, merge) + target = destination / filename + target.write_text(rendered, encoding="utf-8") + validate_text(rendered, mode, merge if mode == "promotion" else base) + if source.stat().st_mode & 0o111: + target.chmod(target.stat().st_mode | 0o111) + shutil.move(str(temporary), str(output)) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + validate_parser = commands.add_parser("validate") + validate_parser.add_argument("--tap-root", required=True) + validate_parser.add_argument( + "--mode", choices=tuple(CALLERS), required=True + ) + validate_parser.add_argument("--kandelo-sha", required=True) + render_parser = commands.add_parser("render") + render_parser.add_argument("--template-root", required=True) + render_parser.add_argument("--base-sha", required=True) + render_parser.add_argument("--merge-sha", required=True) + render_parser.add_argument("--out", required=True) + return parser.parse_args() + + +def main() -> int: + arguments = parse_args() + try: + if arguments.command == "validate": + validate(arguments) + else: + render(arguments) + except (CallerPinError, OSError, UnicodeError) as error: + print(f"homebrew-candidate-caller-pins: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/homebrew-candidate-campaign.py b/scripts/homebrew-candidate-campaign.py new file mode 100755 index 0000000000..33c5bab15e --- /dev/null +++ b/scripts/homebrew-candidate-campaign.py @@ -0,0 +1,1044 @@ +#!/usr/bin/env python3 +"""Seal and admit a noncanonical Homebrew campaign for an unmerged PR. + +Candidate code may derive the campaign only in a credential-free job. Code +from the pull request never publishes it. Protected-main code validates the +result, seals an immutable release, and later admits it only when one exact +merge commit preserves the candidate tree unchanged. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +from typing import Any, NoReturn + + +ROOT = pathlib.Path(__file__).resolve().parent.parent +EXECUTOR_PATH = ROOT / "scripts/homebrew-prefix-campaign-executor.py" +CAMPAIGN_PATH = ROOT / "scripts/homebrew-prefix-campaign.py" + +COMMIT = re.compile(r"^[0-9a-f]{40}$") +SHA256 = re.compile(r"^[0-9a-f]{64}$") +REPOSITORY = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +CANDIDATE_TAG = re.compile( + r"^homebrew-prefix-campaign-candidate-pr-([1-9][0-9]*)-run-" + r"([1-9][0-9]*)-attempt-([1-9][0-9]*)-sha256-([0-9a-f]{64})$" +) +ARTIFACT_DIGEST = re.compile(r"^sha256:([0-9a-f]{64})$") + +MAX_JSON_BYTES = 64 * 1024 * 1024 + + +class CandidateCampaignError(ValueError): + """Candidate campaign evidence did not satisfy its closed contract.""" + + +def fail(message: str) -> NoReturn: + raise CandidateCampaignError(message) + + +def load_tool(name: str, path: pathlib.Path) -> Any: + specification = importlib.util.spec_from_file_location(name, path) + if specification is None or specification.loader is None: + fail(f"cannot load reviewed tool {path}") + module = importlib.util.module_from_spec(specification) + sys.modules[name] = module + specification.loader.exec_module(module) + return module + + +EXECUTOR = load_tool("homebrew_candidate_campaign_executor", EXECUTOR_PATH) + + +def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + fail(f"JSON repeats key {key!r}") + result[key] = value + return result + + +def pretty_json(value: Any) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() + + +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def sha256_file(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while block := stream.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def exact_keys(value: Any, expected: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != expected: + fail(f"{label} must contain exactly {sorted(expected)}") + return value + + +def require_string( + value: Any, + label: str, + pattern: re.Pattern[str] | None = None, + maximum: int = 4096, +) -> str: + if ( + not isinstance(value, str) + or not value + or len(value.encode()) > maximum + or "\0" in value + or "\n" in value + or "\r" in value + or (pattern is not None and pattern.fullmatch(value) is None) + ): + fail(f"{label} is invalid") + return value + + +def require_int(value: Any, label: str, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + fail(f"{label} is invalid") + return value + + +def normalized_repository(value: Any, label: str) -> str: + return require_string(value, label, REPOSITORY).lower() + + +def regular_file( + path: pathlib.Path, label: str, maximum: int = MAX_JSON_BYTES +) -> pathlib.Path: + try: + metadata = path.lstat() + except OSError as error: + fail(f"cannot inspect {label}: {error}") + if not path.is_file() or path.is_symlink(): + fail(f"{label} must be a regular non-symlink file") + if metadata.st_size < 1 or metadata.st_size > maximum: + fail(f"{label} is outside its byte bound") + return path + + +def load_json( + path: pathlib.Path, label: str, *, canonical: bool = True +) -> tuple[Any, bytes]: + payload = regular_file(path, label).read_bytes() + try: + value = json.loads( + payload.decode(), + object_pairs_hook=reject_duplicates, + parse_constant=lambda item: fail( + f"{label} contains invalid constant {item}" + ), + ) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + fail(f"{label} is not strict UTF-8 JSON: {error}") + if canonical and payload != pretty_json(value): + fail(f"{label} is not canonical pretty JSON") + return value, payload + + +def real_directory(path: pathlib.Path, label: str) -> pathlib.Path: + try: + path.lstat() + except OSError as error: + fail(f"cannot inspect {label}: {error}") + if not path.is_dir() or path.is_symlink(): + fail(f"{label} must be one real directory") + return path.resolve() + + +def run_git(root: pathlib.Path, *arguments: str) -> str: + result = subprocess.run( + ["git", "-C", str(root), *arguments], + check=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=120, + ) + if result.returncode != 0: + fail( + f"git {' '.join(arguments)} failed: " + f"{result.stderr.strip()[:4096]}" + ) + return result.stdout.strip() + + +def exact_git_checkout( + root: pathlib.Path, expected: str, label: str +) -> pathlib.Path: + root = real_directory(root, label) + require_string(expected, f"{label} commit", COMMIT) + if pathlib.Path(run_git(root, "rev-parse", "--show-toplevel")) != root: + fail(f"{label} is not its Git worktree root") + if run_git(root, "rev-parse", "HEAD") != expected: + fail(f"{label} is not at its expected commit") + if run_git(root, "status", "--porcelain=v1", "--untracked-files=all"): + fail(f"{label} is not clean") + return root + + +def require_ancestor( + root: pathlib.Path, ancestor: str, descendant: str, label: str +) -> None: + result = subprocess.run( + ["git", "-C", str(root), "merge-base", "--is-ancestor", ancestor, + descendant], + check=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + timeout=120, + ) + if result.returncode != 0: + fail(f"{label} is not on protected main history") + + +def source_file(root: pathlib.Path, relative: str, label: str) -> pathlib.Path: + current = root + parts = pathlib.PurePosixPath(relative).parts + for part in parts[:-1]: + current = current / part + if current.is_symlink() or not current.is_dir(): + fail(f"{label} parent must be a real directory") + path = regular_file(current / parts[-1], label) + if path.resolve().parent != current.resolve(): + fail(f"{label} escaped its source root") + return path + + +def validate_source(value: Any) -> dict[str, Any]: + value = exact_keys( + value, + { + "abi", + "abi_snapshot", + "base_commit", + "guest_layout", + "kandelo_repository", + "kind", + "native_homebrew_commit", + "old_metadata", + "pr_number", + "producer_commit", + "producer_tree", + "schema", + "source_tap_commit", + "source_tap_tree", + "tap_name", + "tap_repository", + "tap_workflow_authority_commit", + "workflow_authority_commit", + }, + "candidate campaign source", + ) + if ( + value["schema"] != 1 + or value["kind"] + != "kandelo-homebrew-prefix-campaign-candidate-source" + ): + fail("candidate campaign source has an unsupported contract") + if normalized_repository( + value["kandelo_repository"], "Kandelo repository" + ) != "automattic/kandelo": + fail("candidate campaign names another Kandelo repository") + if normalized_repository( + value["tap_repository"], "tap repository" + ) != "kandelo-dev/homebrew-tap-core": + fail("candidate campaign v1 names another tap repository") + if normalized_repository( + value["tap_name"], "tap name" + ) != "kandelo-dev/tap-core": + fail("candidate campaign v1 names another tap") + require_int(value["pr_number"], "candidate campaign PR", 1) + require_int(value["abi"], "candidate campaign ABI", 1) + for field in ( + "base_commit", + "producer_commit", + "producer_tree", + "source_tap_commit", + "source_tap_tree", + "tap_workflow_authority_commit", + "workflow_authority_commit", + "native_homebrew_commit", + ): + require_string(value[field], field.replace("_", " "), COMMIT) + for field, expected_path in ( + ("abi_snapshot", "abi/snapshot.json"), + ("guest_layout", "homebrew/kandelo-guest-layout.json"), + ("old_metadata", "Kandelo/metadata.json"), + ): + record = exact_keys( + value[field], {"path", "sha256"}, field.replace("_", " ") + ) + if record["path"] != expected_path: + fail(f"candidate campaign {field} path is not canonical") + require_string(record["sha256"], f"{field} SHA-256", SHA256) + if value["workflow_authority_commit"] != value["base_commit"]: + fail("candidate campaign v1 validator authority must be its base") + return value + + +def describe_source(arguments: argparse.Namespace) -> None: + producer = exact_git_checkout( + pathlib.Path(arguments.kandelo_root), + arguments.producer_commit, + "candidate Kandelo source", + ) + tap = exact_git_checkout( + pathlib.Path(arguments.tap_root), + arguments.source_tap_commit, + "candidate tap source", + ) + snapshot_path = source_file(producer, "abi/snapshot.json", "ABI snapshot") + snapshot, snapshot_payload = load_json(snapshot_path, "ABI snapshot") + if not isinstance(snapshot, dict): + fail("ABI snapshot must be an object") + abi = require_int(snapshot.get("abi_version"), "ABI snapshot version", 1) + layout_path = source_file( + producer, + "homebrew/kandelo-guest-layout.json", + "guest layout", + ) + _layout, layout_payload = load_json(layout_path, "guest layout") + roots_path = source_file( + producer, + "homebrew/homebrew-native-compatibility-roots.json", + "native Homebrew roots", + ) + roots, _roots_payload = load_json(roots_path, "native Homebrew roots") + if not isinstance(roots, dict): + fail("native Homebrew roots must be an object") + native_commit = require_string( + roots.get("homebrew_commit"), "native Homebrew commit", COMMIT + ) + metadata_path = source_file( + tap, "Kandelo/metadata.json", "old tap metadata" + ) + _metadata, metadata_payload = load_json( + metadata_path, "old tap metadata" + ) + source = validate_source( + { + "schema": 1, + "kind": "kandelo-homebrew-prefix-campaign-candidate-source", + "kandelo_repository": arguments.kandelo_repository, + "pr_number": arguments.pr_number, + "base_commit": arguments.base_commit, + "producer_commit": arguments.producer_commit, + "producer_tree": run_git(producer, "rev-parse", "HEAD^{tree}"), + "workflow_authority_commit": arguments.workflow_authority_commit, + "abi": abi, + "abi_snapshot": { + "path": "abi/snapshot.json", + "sha256": sha256_bytes(snapshot_payload), + }, + "guest_layout": { + "path": "homebrew/kandelo-guest-layout.json", + "sha256": sha256_bytes(layout_payload), + }, + "tap_repository": arguments.tap_repository, + "tap_name": arguments.tap_name, + "source_tap_commit": arguments.source_tap_commit, + "source_tap_tree": run_git(tap, "rev-parse", "HEAD^{tree}"), + "tap_workflow_authority_commit": ( + arguments.tap_workflow_authority_commit + ), + "old_metadata": { + "path": "Kandelo/metadata.json", + "sha256": sha256_bytes(metadata_payload), + }, + "native_homebrew_commit": native_commit, + } + ) + output = pathlib.Path(arguments.out) + if output.exists() or output.is_symlink(): + fail("candidate campaign source output already exists") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(pretty_json(source)) + + +def validate_run(value: Any, source: dict[str, Any]) -> dict[str, Any]: + value = exact_keys( + value, + { + "artifacts", + "caller_commit", + "conclusion", + "event", + "repository", + "run_attempt", + "run_id", + "schema", + "status", + "workflow_path", + }, + "candidate campaign run", + ) + if value["schema"] != 1: + fail("candidate campaign run has an unsupported contract") + if normalized_repository( + value["repository"], "candidate campaign run repository" + ) != normalized_repository(source["tap_repository"], "tap repository"): + fail("candidate campaign run belongs to another repository") + if ( + value["workflow_path"] != ".github/workflows/candidate-campaign.yml" + or value["event"] != "repository_dispatch" + or value["caller_commit"] + != source["tap_workflow_authority_commit"] + ): + fail("candidate campaign run did not use its reviewed caller") + run_id = require_int(value["run_id"], "candidate campaign run ID", 1) + attempt = require_int( + value["run_attempt"], "candidate campaign run attempt", 1 + ) + if value["status"] not in ("in_progress", "completed"): + fail("candidate campaign run status is invalid") + if value["conclusion"] not in (None, "success"): + fail("candidate campaign run did not succeed") + if value["status"] == "completed" and value["conclusion"] != "success": + fail("completed candidate campaign run is not successful") + artifacts = value["artifacts"] + if not isinstance(artifacts, list) or len(artifacts) != 1: + fail("candidate campaign run must bind exactly one artifact") + artifact = exact_keys( + artifacts[0], + {"bytes", "digest", "id", "name", "run_attempt", "run_id"}, + "candidate campaign artifact", + ) + expected_name = ( + f"homebrew-candidate-campaign-derivation-attempt-{attempt}" + ) + if artifact["name"] != expected_name: + fail("candidate campaign run names another artifact") + require_int(artifact["id"], "candidate campaign artifact ID", 1) + require_int(artifact["bytes"], "candidate campaign artifact bytes", 1) + require_string( + artifact["digest"], "candidate campaign artifact digest", + ARTIFACT_DIGEST, + ) + if artifact["run_id"] != run_id or artifact["run_attempt"] != attempt: + fail("candidate campaign artifact belongs to another run") + return value + + +def validate_campaign_authority( + campaign: dict[str, Any], source: dict[str, Any] +) -> None: + authority = campaign["authority"] + if ( + authority.get("kandelo_commit") != source["producer_commit"] + or authority.get("current_kandelo_abi") != source["abi"] + or authority.get("old_tap_commit") != source["source_tap_commit"] + or authority.get("source_tap_commit") + != source["source_tap_commit"] + or str(authority.get("tap_repository", "")).lower() + != source["tap_repository"].lower() + or str(authority.get("tap_name", "")).lower() + != source["tap_name"].lower() + or authority.get("native_homebrew_commit") + != source["native_homebrew_commit"] + or authority.get("abi_snapshot") != source["abi_snapshot"] + or authority.get("guest_layout") != source["guest_layout"] + or authority.get("old_metadata") != source["old_metadata"] + ): + fail("candidate campaign authority differs from its exact sources") + + +def candidate_tag( + manifest_payload: bytes, source: dict[str, Any], run: dict[str, Any] +) -> str: + return ( + "homebrew-prefix-campaign-candidate-pr-" + f"{source['pr_number']}-run-{run['run_id']}-attempt-" + f"{run['run_attempt']}-sha256-{sha256_bytes(manifest_payload)}" + ) + + +def validate_manifest( + value: Any, + payload: bytes, + campaign_payload: bytes, + expected_tag: str | None = None, +) -> dict[str, Any]: + value = exact_keys( + value, + {"campaign", "kind", "run", "schema", "source"}, + "candidate campaign manifest", + ) + if ( + value["schema"] != 1 + or value["kind"] != "kandelo-homebrew-prefix-campaign-candidate" + ): + fail("candidate campaign manifest has an unsupported contract") + source = validate_source(value["source"]) + run = validate_run(value["run"], source) + campaign_record = exact_keys( + value["campaign"], {"bytes", "sha256"}, "candidate campaign asset" + ) + if ( + campaign_record["bytes"] != len(campaign_payload) + or campaign_record["sha256"] != sha256_bytes(campaign_payload) + ): + fail("candidate campaign asset differs from its manifest") + if expected_tag is not None: + match = CANDIDATE_TAG.fullmatch(expected_tag) + if ( + match is None + or int(match.group(1)) != source["pr_number"] + or int(match.group(2)) != run["run_id"] + or int(match.group(3)) != run["run_attempt"] + or match.group(4) != sha256_bytes(payload) + ): + fail("candidate campaign tag differs from its manifest") + return value + + +def prepare(arguments: argparse.Namespace) -> None: + source, _source_payload = load_json( + pathlib.Path(arguments.source), "candidate campaign source" + ) + source = validate_source(source) + run, _run_payload = load_json( + pathlib.Path(arguments.run_evidence), "candidate campaign run" + ) + run = validate_run(run, source) + campaign_path = pathlib.Path(arguments.campaign) + campaign, campaign_payload, _index = EXECUTOR.load_campaign(campaign_path) + validate_campaign_authority(campaign, source) + output = pathlib.Path(arguments.out) + if output.exists() or output.is_symlink(): + fail("candidate campaign output already exists") + output.parent.mkdir(parents=True, exist_ok=True) + temporary = pathlib.Path( + tempfile.mkdtemp(prefix=f".{output.name}.", dir=output.parent) + ) + try: + assets = temporary / "assets" + assets.mkdir() + campaign_asset = assets / "campaign.json" + shutil.copyfile(campaign_path, campaign_asset) + if campaign_asset.read_bytes() != campaign_payload: + fail("candidate campaign changed while copied") + manifest = { + "schema": 1, + "kind": "kandelo-homebrew-prefix-campaign-candidate", + "source": source, + "run": run, + "campaign": { + "bytes": len(campaign_payload), + "sha256": sha256_bytes(campaign_payload), + }, + } + manifest_payload = pretty_json(manifest) + validate_manifest(manifest, manifest_payload, campaign_payload) + manifest_asset = assets / "candidate-campaign.json" + manifest_asset.write_bytes(manifest_payload) + tag = candidate_tag(manifest_payload, source, run) + release_assets = [ + { + "name": path.name, + "bytes": path.stat().st_size, + "sha256": sha256_file(path), + } + for path in sorted(assets.iterdir(), key=lambda path: path.name) + ] + release = { + "schema": 1, + "repository": source["tap_repository"], + "tag": tag, + "target_commitish": run["caller_commit"], + "title": f"Kandelo candidate campaign for PR #{source['pr_number']}", + "body": ( + "Run-bound, noncanonical campaign evidence. It cannot " + "publish bottles until an exact-head merge is admitted." + ), + "assets": release_assets, + "preferred_asset_names": [ + asset["name"] for asset in release_assets + ], + "accepted_existing_asset_sets": [], + } + (temporary / "release-manifest.json").write_bytes( + pretty_json(release) + ) + (temporary / "tag.txt").write_text(f"{tag}\n") + os.replace(temporary, output) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +def describe_release(arguments: argparse.Namespace) -> None: + manifest, manifest_payload = load_json( + pathlib.Path(arguments.candidate), "candidate campaign manifest" + ) + campaign_path = pathlib.Path(arguments.campaign) + _campaign, campaign_payload, _index = EXECUTOR.load_campaign(campaign_path) + manifest = validate_manifest( + manifest, manifest_payload, campaign_payload, arguments.candidate_tag + ) + validate_campaign_authority(_campaign, manifest["source"]) + output = pathlib.Path(arguments.out) + if output.exists() or output.is_symlink(): + fail("candidate campaign description already exists") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes( + pretty_json( + { + "schema": 1, + "kind": ( + "kandelo-homebrew-prefix-campaign-candidate-description" + ), + "candidate_tag": arguments.candidate_tag, + "candidate_sha256": sha256_bytes(manifest_payload), + "campaign_sha256": sha256_bytes(campaign_payload), + "manifest": manifest, + } + ) + ) + + +def fetch_release(arguments: argparse.Namespace) -> None: + repository = normalized_repository(arguments.repository, "release repository") + tag = require_string(arguments.tag, "candidate campaign tag") + match = CANDIDATE_TAG.fullmatch(tag) + if match is None: + fail("candidate campaign release tag is invalid") + output = pathlib.Path(arguments.out) + candidate_output = pathlib.Path(arguments.candidate_out) + receipt_output = pathlib.Path(arguments.receipt_out) + for path, label in ( + (output, "campaign output"), + (candidate_output, "candidate manifest output"), + (receipt_output, "candidate readback receipt"), + ): + if path.exists() or path.is_symlink(): + fail(f"{label} already exists") + path.parent.mkdir(parents=True, exist_ok=True) + assets, release = EXECUTOR.release_assets(repository, tag) + if set(assets) != {"campaign.json", "candidate-campaign.json"}: + fail("candidate campaign release has an unexpected asset inventory") + if assets["candidate-campaign.json"]["sha256"] != match.group(4): + fail("candidate campaign release differs from its tag") + temporary = pathlib.Path( + tempfile.mkdtemp(prefix=".candidate-campaign-readback-", dir=output.parent) + ) + try: + staged_campaign = temporary / "campaign.json" + staged_candidate = temporary / "candidate-campaign.json" + EXECUTOR.fetch_one_asset(assets, "campaign.json", staged_campaign) + EXECUTOR.fetch_one_asset( + assets, "candidate-campaign.json", staged_candidate + ) + candidate, candidate_payload = load_json( + staged_candidate, "candidate campaign manifest" + ) + campaign, campaign_payload, _index = EXECUTOR.load_campaign( + staged_campaign + ) + candidate = validate_manifest( + candidate, candidate_payload, campaign_payload, tag + ) + validate_campaign_authority(campaign, candidate["source"]) + if release.get("target_commitish") != candidate["run"]["caller_commit"]: + fail("candidate campaign release targets another caller") + receipt = { + "schema": 1, + "kind": "kandelo-homebrew-prefix-campaign-candidate-readback", + "repository": repository, + "tag": tag, + "release_id": require_int( + release.get("id"), "candidate campaign release ID", 1 + ), + "target_commitish": release["target_commitish"], + "candidate_sha256": sha256_bytes(candidate_payload), + "campaign_sha256": sha256_bytes(campaign_payload), + } + staged_receipt = temporary / "receipt.json" + staged_receipt.write_bytes(pretty_json(receipt)) + os.link(staged_campaign, output) + try: + os.link(staged_candidate, candidate_output) + os.link(staged_receipt, receipt_output) + except OSError: + output.unlink(missing_ok=True) + candidate_output.unlink(missing_ok=True) + receipt_output.unlink(missing_ok=True) + raise + finally: + shutil.rmtree(temporary, ignore_errors=True) + + +def validate_completed_run( + completed: Any, recorded: dict[str, Any], source: dict[str, Any] +) -> None: + completed = validate_run(completed, source) + expected = dict(recorded) + expected["status"] = "completed" + expected["conclusion"] = "success" + if completed != expected: + fail("completed candidate campaign run differs from sealed evidence") + + +def validate_exact_merge( + main_root: pathlib.Path, + producer_root: pathlib.Path, + source: dict[str, Any], + merge_commit: str, + current_main: str, +) -> None: + main_root = exact_git_checkout(main_root, merge_commit, "merged Kandelo") + producer_root = exact_git_checkout( + producer_root, source["producer_commit"], "candidate producer" + ) + # WHY: Git branch and PR-head metadata can move after merge. The immutable + # merge object below proves the exact base and producer more directly. + parents = run_git(main_root, "show", "-s", "--format=%P", merge_commit) + if parents != f"{source['base_commit']} {source['producer_commit']}": + fail("candidate campaign merge did not preserve [base, exact head]") + producer_tree = run_git( + producer_root, "rev-parse", "HEAD^{tree}" + ) + merge_tree = run_git(main_root, "rev-parse", "HEAD^{tree}") + if producer_tree != source["producer_tree"] or merge_tree != producer_tree: + fail("candidate campaign merge tree differs from the candidate") + require_string(current_main, "current Kandelo main", COMMIT) + require_ancestor(main_root, merge_commit, current_main, "candidate merge") + require_ancestor( + main_root, + source["workflow_authority_commit"], + current_main, + "candidate campaign validator authority", + ) + + +def recorded_probe_dependencies( + campaign_module: Any, campaign: dict[str, Any] +) -> Any: + probes: dict[tuple[str, str], dict[str, Any]] = {} + for formula in campaign["formulae"]: + destination = formula.get("destination") + if not isinstance(destination, dict): + fail("candidate campaign Formula lacks destination evidence") + admission = destination.get("admission") + if not isinstance(admission, dict) or not isinstance( + admission.get("probe"), dict + ): + fail("candidate campaign Formula lacks a bounded destination probe") + key = (destination.get("remote"), destination.get("reference")) + if ( + any(not isinstance(item, str) or not item for item in key) + or key in probes + ): + fail("candidate campaign repeats a destination identity") + probes[key] = admission["probe"] + + def probe( + remote: str, reference: str, _kandelo_root: pathlib.Path + ) -> dict[str, Any]: + key = (remote, reference) + if key not in probes: + fail("candidate campaign derivation requested an unsealed destination") + return json.loads(json.dumps(probes[key])) + + # WHY: registry absence is time-sensitive. A sibling candidate may be + # promoted after this campaign was sealed. Reuse only the recorded probe + # while rederiving every source-controlled decision. Each selected bottle + # still performs its own live collision probe immediately before upload. + return campaign_module.CampaignDependencies(probe_destination=probe) + + +def admit(arguments: argparse.Namespace) -> None: + candidate, candidate_payload = load_json( + pathlib.Path(arguments.candidate), "candidate campaign manifest" + ) + campaign_path = pathlib.Path(arguments.campaign) + campaign, campaign_payload, _index = EXECUTOR.load_campaign(campaign_path) + candidate = validate_manifest( + candidate, + candidate_payload, + campaign_payload, + arguments.candidate_tag, + ) + source = candidate["source"] + validate_campaign_authority(campaign, source) + completed, _completed_payload = load_json( + pathlib.Path(arguments.completed_run_evidence), + "completed candidate campaign run", + ) + validate_completed_run(completed, candidate["run"], source) + main_root = pathlib.Path(arguments.kandelo_main_root) + producer_root = pathlib.Path(arguments.producer_root) + validate_exact_merge( + main_root, + producer_root, + source, + arguments.merge_commit, + arguments.current_kandelo_main, + ) + tap_root = exact_git_checkout( + pathlib.Path(arguments.tap_root), + source["source_tap_commit"], + "candidate campaign tap source", + ) + if run_git(tap_root, "rev-parse", "HEAD^{tree}") != source["source_tap_tree"]: + fail("candidate campaign tap tree differs from its source evidence") + require_ancestor( + tap_root, + source["source_tap_commit"], + arguments.current_tap_main, + "candidate campaign tap source", + ) + require_ancestor( + tap_root, + source["tap_workflow_authority_commit"], + arguments.current_tap_main, + "candidate campaign tap workflow authority", + ) + native_root = exact_git_checkout( + pathlib.Path(arguments.native_brew_root), + source["native_homebrew_commit"], + "candidate native Homebrew", + ) + main_root = exact_git_checkout( + main_root, arguments.merge_commit, "merged Kandelo" + ) + campaign_tool = main_root / "scripts/homebrew-prefix-campaign.py" + campaign_module = load_tool( + "homebrew_candidate_campaign_recheck", campaign_tool + ) + options = campaign_module.CampaignOptions( + kandelo_root=producer_root, + kandelo_commit=source["producer_commit"], + old_tap_root=tap_root, + old_tap_commit=source["source_tap_commit"], + source_tap_root=tap_root, + source_tap_commit=source["source_tap_commit"], + native_brew_root=native_root, + native_brew_commit=source["native_homebrew_commit"], + metadata_sha256=source["old_metadata"]["sha256"], + guest_layout_sha256=source["guest_layout"]["sha256"], + jobs=campaign_module.MAX_JOBS, + ) + regenerated = campaign_module.derive_campaign( + options, recorded_probe_dependencies(campaign_module, campaign) + ) + if campaign_module.pretty_json(regenerated) != campaign_payload: + fail("protected main regenerated a different candidate campaign") + receipt = { + "schema": 1, + "kind": "kandelo-homebrew-prefix-campaign-candidate-admission", + "candidate_tag": arguments.candidate_tag, + "candidate_sha256": sha256_bytes(candidate_payload), + "campaign_sha256": sha256_bytes(campaign_payload), + "producer_commit": source["producer_commit"], + "merge_commit": arguments.merge_commit, + "validated_against_main": arguments.merge_commit, + "source_tap_commit": source["source_tap_commit"], + "tap_workflow_authority_commit": source[ + "tap_workflow_authority_commit" + ], + "abi": source["abi"], + "abi_snapshot_sha256": source["abi_snapshot"]["sha256"], + "guest_layout_sha256": source["guest_layout"]["sha256"], + "run_id": candidate["run"]["run_id"], + "run_attempt": candidate["run"]["run_attempt"], + } + output = pathlib.Path(arguments.out) + if output.exists() or output.is_symlink(): + fail("candidate campaign admission output already exists") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(pretty_json(receipt)) + + +def validate_admission_value(value: Any) -> dict[str, Any]: + value = exact_keys( + value, + { + "abi", + "abi_snapshot_sha256", + "campaign_sha256", + "candidate_sha256", + "candidate_tag", + "guest_layout_sha256", + "kind", + "merge_commit", + "producer_commit", + "run_attempt", + "run_id", + "schema", + "source_tap_commit", + "tap_workflow_authority_commit", + "validated_against_main", + }, + "candidate campaign admission", + ) + if ( + value["schema"] != 1 + or value["kind"] + != "kandelo-homebrew-prefix-campaign-candidate-admission" + ): + fail("candidate campaign admission has an unsupported contract") + for field in ( + "abi_snapshot_sha256", + "campaign_sha256", + "candidate_sha256", + "guest_layout_sha256", + ): + require_string(value[field], field.replace("_", " "), SHA256) + for field in ( + "merge_commit", + "producer_commit", + "source_tap_commit", + "tap_workflow_authority_commit", + "validated_against_main", + ): + require_string(value[field], field.replace("_", " "), COMMIT) + require_int(value["abi"], "candidate campaign admission ABI", 1) + require_int(value["run_id"], "candidate campaign admission run", 1) + require_int( + value["run_attempt"], "candidate campaign admission attempt", 1 + ) + match = CANDIDATE_TAG.fullmatch( + require_string(value["candidate_tag"], "candidate campaign tag") + ) + if ( + match is None + or match.group(4) != value["candidate_sha256"] + or int(match.group(2)) != value["run_id"] + or int(match.group(3)) != value["run_attempt"] + or value["validated_against_main"] != value["merge_commit"] + ): + fail("candidate campaign admission is internally inconsistent") + return value + + +def validate_admission(arguments: argparse.Namespace) -> None: + value, _payload = load_json( + pathlib.Path(arguments.receipt), "candidate campaign admission" + ) + value = validate_admission_value(value) + expected = { + "candidate_tag": arguments.candidate_tag, + "producer_commit": arguments.producer_commit, + "merge_commit": arguments.merge_commit, + "source_tap_commit": arguments.source_tap_commit, + "abi": arguments.abi, + "guest_layout_sha256": arguments.guest_layout_sha256, + } + for field, wanted in expected.items(): + if value[field] != wanted: + fail( + "candidate campaign admission differs from publication " + f"field {field}" + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + + source = commands.add_parser("describe-source") + source.add_argument("--kandelo-root", required=True) + source.add_argument("--kandelo-repository", required=True) + source.add_argument("--base-commit", required=True) + source.add_argument("--producer-commit", required=True) + source.add_argument("--workflow-authority-commit", required=True) + source.add_argument("--pr-number", required=True, type=int) + source.add_argument("--tap-root", required=True) + source.add_argument("--tap-repository", required=True) + source.add_argument("--tap-name", required=True) + source.add_argument("--source-tap-commit", required=True) + source.add_argument("--tap-workflow-authority-commit", required=True) + source.add_argument("--out", required=True) + + prepare_command = commands.add_parser("prepare") + prepare_command.add_argument("--source", required=True) + prepare_command.add_argument("--run-evidence", required=True) + prepare_command.add_argument("--campaign", required=True) + prepare_command.add_argument("--out", required=True) + + describe = commands.add_parser("describe-release") + describe.add_argument("--candidate", required=True) + describe.add_argument("--campaign", required=True) + describe.add_argument("--candidate-tag", required=True) + describe.add_argument("--out", required=True) + + fetch = commands.add_parser("fetch-release") + fetch.add_argument("--repository", required=True) + fetch.add_argument("--tag", required=True) + fetch.add_argument("--out", required=True) + fetch.add_argument("--candidate-out", required=True) + fetch.add_argument("--receipt-out", required=True) + + admission = commands.add_parser("admit") + admission.add_argument("--candidate", required=True) + admission.add_argument("--campaign", required=True) + admission.add_argument("--candidate-tag", required=True) + admission.add_argument("--completed-run-evidence", required=True) + admission.add_argument("--kandelo-main-root", required=True) + admission.add_argument("--producer-root", required=True) + admission.add_argument("--tap-root", required=True) + admission.add_argument("--native-brew-root", required=True) + admission.add_argument("--merge-commit", required=True) + admission.add_argument("--current-kandelo-main", required=True) + admission.add_argument("--current-tap-main", required=True) + admission.add_argument("--out", required=True) + + validate = commands.add_parser("validate-admission") + validate.add_argument("--receipt", required=True) + validate.add_argument("--candidate-tag", required=True) + validate.add_argument("--producer-commit", required=True) + validate.add_argument("--merge-commit", required=True) + validate.add_argument("--source-tap-commit", required=True) + validate.add_argument("--abi", required=True, type=int) + validate.add_argument("--guest-layout-sha256", required=True) + return parser.parse_args() + + +def main() -> int: + arguments = parse_args() + try: + if arguments.command == "describe-source": + describe_source(arguments) + elif arguments.command == "prepare": + prepare(arguments) + elif arguments.command == "describe-release": + describe_release(arguments) + elif arguments.command == "fetch-release": + fetch_release(arguments) + elif arguments.command == "admit": + admit(arguments) + else: + validate_admission(arguments) + except ( + CandidateCampaignError, + EXECUTOR.ExecutorError, + OSError, + subprocess.SubprocessError, + ) as error: + print(f"homebrew-candidate-campaign: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/homebrew-candidate-release-receipt.py b/scripts/homebrew-candidate-release-receipt.py new file mode 100755 index 0000000000..50f14f563b --- /dev/null +++ b/scripts/homebrew-candidate-release-receipt.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +"""Validate durable sealer evidence for an immutable candidate release.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import re +import sys +from typing import Any, NoReturn + + +SHA256 = re.compile(r"^[0-9a-f]{64}$") +COMMIT = re.compile(r"^[0-9a-f]{40}$") +REPOSITORY = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +ASSET_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,254}$") +MAX_JSON_BYTES = 16 * 1024 * 1024 +MAX_ASSETS = 256 +MAX_TOTAL_BYTES = 4 * 1024 * 1024 * 1024 + + +class ReceiptError(ValueError): + """The release receipt did not satisfy its closed evidence contract.""" + + +def fail(message: str) -> NoReturn: + raise ReceiptError(message) + + +def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + fail(f"JSON repeats key {key!r}") + result[key] = value + return result + + +def load_json(path: pathlib.Path, label: str) -> Any: + if path.is_symlink() or not path.is_file(): + fail(f"{label} must be a regular file") + if path.stat().st_size > MAX_JSON_BYTES: + fail(f"{label} exceeds its byte bound") + with path.open("r", encoding="utf-8") as stream: + return json.load(stream, object_pairs_hook=reject_duplicates) + + +def exact_keys(value: Any, expected: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != expected: + fail(f"{label} must contain exactly {sorted(expected)}") + return value + + +def require_string( + value: Any, + label: str, + pattern: re.Pattern[str] | None = None, + maximum: int = 1024, +) -> str: + if not isinstance(value, str) or not value or len(value) > maximum: + fail(f"{label} must be a bounded nonempty string") + if pattern is not None and pattern.fullmatch(value) is None: + fail(f"{label} has an invalid format") + return value + + +def require_int(value: Any, label: str, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + fail(f"{label} must be an integer >= {minimum}") + return value + + +def normalized_repository(value: Any, label: str) -> str: + return require_string(value, label, REPOSITORY).lower() + + +def sha256_file(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while block := stream.read(1024 * 1024): + digest.update(block) + return digest.hexdigest() + + +def validate_receipt(value: Any) -> dict[str, Any]: + value = exact_keys( + value, + { + "assets", + "immutable", + "release_id", + "repository", + "schema", + "status", + "tag", + "target_commitish", + "visibility", + }, + "candidate sealer receipt", + ) + if ( + value["schema"] != 1 + or value["status"] != "success" + or value["visibility"] != "public-anonymous-readback" + or value["immutable"] is not True + ): + fail("candidate sealer receipt does not record a successful public release") + normalized_repository(value["repository"], "receipt repository") + require_string(value["tag"], "receipt tag", maximum=512) + require_string(value["target_commitish"], "receipt target", COMMIT) + require_int(value["release_id"], "receipt release ID", 1) + assets = value["assets"] + if not isinstance(assets, list) or not assets or len(assets) > MAX_ASSETS: + fail("candidate sealer receipt has an invalid asset count") + names: list[str] = [] + ids: set[int] = set() + total = 0 + for position, asset in enumerate(assets): + asset = exact_keys( + asset, + {"asset_id", "bytes", "name", "sha256", "url"}, + f"candidate sealer receipt asset #{position}", + ) + name = require_string(asset["name"], "receipt asset name", ASSET_NAME) + asset_id = require_int(asset["asset_id"], "receipt asset ID", 1) + byte_count = require_int(asset["bytes"], "receipt asset bytes", 1) + require_string(asset["sha256"], "receipt asset SHA-256", SHA256) + url = require_string(asset["url"], "receipt asset URL", maximum=4096) + if not url.startswith("https://github.com/"): + fail("receipt asset URL is not a public GitHub download") + if asset_id in ids: + fail("candidate sealer receipt repeats an asset ID") + ids.add(asset_id) + names.append(name) + total += byte_count + if total > MAX_TOTAL_BYTES: + fail("candidate sealer receipt exceeds its aggregate byte bound") + if names != sorted(set(names)): + fail("candidate sealer receipt assets must be unique and sorted") + return value + + +def validate_live_release( + release: Any, + live_assets: Any, + receipt: dict[str, Any], + expected_repository: str, + expected_tag: str, + expected_target: str, +) -> list[dict[str, Any]]: + if not isinstance(release, dict): + fail("live candidate release must be an object") + repository = normalized_repository( + expected_repository, "expected release repository" + ) + require_string(expected_target, "expected release target", COMMIT) + if ( + normalized_repository(receipt["repository"], "receipt repository") + != repository + or receipt["tag"] != expected_tag + or receipt["target_commitish"] != expected_target + ): + fail("candidate sealer receipt names another release") + if ( + release.get("id") != receipt["release_id"] + or release.get("tag_name") != expected_tag + or release.get("target_commitish") != expected_target + or release.get("immutable") is not True + or release.get("draft") is not False + or release.get("prerelease") is not False + ): + fail("live candidate release differs from the protected receipt") + if not isinstance(live_assets, list) or len(live_assets) > MAX_ASSETS: + fail("live candidate release has an invalid asset inventory") + by_name: dict[str, dict[str, Any]] = {} + ids: set[int] = set() + for position, asset in enumerate(live_assets): + if not isinstance(asset, dict): + fail(f"live release asset #{position} must be an object") + name = require_string(asset.get("name"), "live asset name", ASSET_NAME) + asset_id = require_int(asset.get("id"), "live asset ID", 1) + require_int(asset.get("size"), "live asset bytes", 1) + require_string(asset.get("digest"), "live asset digest", maximum=71) + url = require_string( + asset.get("browser_download_url"), + "live asset download URL", + maximum=4096, + ) + if ( + asset.get("state") != "uploaded" + or not url.startswith("https://github.com/") + or name in by_name + or asset_id in ids + ): + fail("live candidate release has ambiguous asset metadata") + by_name[name] = asset + ids.add(asset_id) + receipt_by_name = {asset["name"]: asset for asset in receipt["assets"]} + if set(by_name) != set(receipt_by_name): + fail("live candidate release inventory differs from the protected receipt") + plan_assets: list[dict[str, Any]] = [] + for name in sorted(receipt_by_name): + recorded = receipt_by_name[name] + live = by_name[name] + if ( + live["id"] != recorded["asset_id"] + or live["size"] != recorded["bytes"] + or live["digest"] != f"sha256:{recorded['sha256']}" + or live["browser_download_url"] != recorded["url"] + ): + fail(f"live candidate asset {name} differs from the protected receipt") + plan_assets.append(dict(recorded)) + return plan_assets + + +def plan(arguments: argparse.Namespace) -> None: + receipt = validate_receipt( + load_json(pathlib.Path(arguments.receipt), "candidate sealer receipt") + ) + release = load_json(pathlib.Path(arguments.release), "live candidate release") + assets = load_json( + pathlib.Path(arguments.release_assets), "live candidate release assets" + ) + plan_assets = validate_live_release( + release, + assets, + receipt, + arguments.repository, + arguments.tag, + arguments.target_commit, + ) + output = pathlib.Path(arguments.out) + if output.exists() or output.is_symlink(): + fail("candidate release readback plan already exists") + output.parent.mkdir(parents=True, exist_ok=True) + value = { + "schema": 1, + "kind": "kandelo-homebrew-candidate-release-readback-plan", + "repository": receipt["repository"].lower(), + "tag": receipt["tag"], + "target_commitish": receipt["target_commitish"], + "release_id": receipt["release_id"], + "assets": plan_assets, + } + output.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def validate_plan(value: Any) -> dict[str, Any]: + value = exact_keys( + value, + { + "assets", + "kind", + "release_id", + "repository", + "schema", + "tag", + "target_commitish", + }, + "candidate release readback plan", + ) + if ( + value["schema"] != 1 + or value["kind"] + != "kandelo-homebrew-candidate-release-readback-plan" + ): + fail("candidate release readback plan has an unsupported contract") + receipt_shape = { + "schema": 1, + "status": "success", + "visibility": "public-anonymous-readback", + "repository": value["repository"], + "tag": value["tag"], + "target_commitish": value["target_commitish"], + "release_id": value["release_id"], + "immutable": True, + "assets": value["assets"], + } + validate_receipt(receipt_shape) + return value + + +def verify_readback(arguments: argparse.Namespace) -> None: + plan_value = validate_plan( + load_json(pathlib.Path(arguments.plan), "candidate release readback plan") + ) + root = pathlib.Path(arguments.asset_root) + if root.is_symlink() or not root.is_dir(): + fail("candidate release readback root must be a real directory") + expected = {asset["name"] for asset in plan_value["assets"]} + actual = {path.name for path in root.iterdir()} + if actual != expected: + fail("anonymous candidate release readback has an unexpected inventory") + for asset in plan_value["assets"]: + path = root / asset["name"] + if path.is_symlink() or not path.is_file(): + fail(f"anonymous candidate asset {asset['name']} is not regular") + if ( + path.stat().st_size != asset["bytes"] + or sha256_file(path) != asset["sha256"] + ): + fail(f"anonymous candidate asset {asset['name']} changed") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + plan_parser = commands.add_parser("plan") + plan_parser.add_argument("--receipt", required=True) + plan_parser.add_argument("--release", required=True) + plan_parser.add_argument("--release-assets", required=True) + plan_parser.add_argument("--repository", required=True) + plan_parser.add_argument("--tag", required=True) + plan_parser.add_argument("--target-commit", required=True) + plan_parser.add_argument("--out", required=True) + verify_parser = commands.add_parser("verify-readback") + verify_parser.add_argument("--plan", required=True) + verify_parser.add_argument("--asset-root", required=True) + return parser.parse_args() + + +def main() -> int: + arguments = parse_args() + try: + if arguments.command == "plan": + plan(arguments) + else: + verify_readback(arguments) + except (ReceiptError, OSError, json.JSONDecodeError) as error: + print(f"homebrew-candidate-release-receipt: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/homebrew-prefix-campaign-publisher.py b/scripts/homebrew-prefix-campaign-publisher.py index f2028ab862..939f9777a5 100755 --- a/scripts/homebrew-prefix-campaign-publisher.py +++ b/scripts/homebrew-prefix-campaign-publisher.py @@ -23,11 +23,16 @@ ROOT = pathlib.Path(__file__).resolve().parent.parent CAMPAIGN_TOOL = ROOT / "scripts/homebrew-prefix-campaign.py" EXECUTOR_TOOL = ROOT / "scripts/homebrew-prefix-campaign-executor.py" +CANDIDATE_CAMPAIGN_TOOL = ROOT / "scripts/homebrew-candidate-campaign.py" COMMIT = re.compile(r"^[0-9a-f]{40}$") FORMULA = re.compile(r"^[a-z0-9][a-z0-9._-]{0,254}$") CAMPAIGN_TAG = re.compile( r"^homebrew-prefix-campaign-sha256-([0-9a-f]{64})$" ) +CANDIDATE_CAMPAIGN_TAG = re.compile( + r"^homebrew-prefix-campaign-candidate-pr-[1-9][0-9]*-run-" + r"[1-9][0-9]*-attempt-[1-9][0-9]*-sha256-([0-9a-f]{64})$" +) HANDOFF_TAG = re.compile( r"^homebrew-prefix-handoff-sha256-([0-9a-f]{64})$" ) @@ -57,6 +62,10 @@ def load_tool(name: str, path: pathlib.Path) -> Any: CAMPAIGN = load_tool("homebrew_prefix_campaign_publisher_campaign", CAMPAIGN_TOOL) EXECUTOR = load_tool("homebrew_prefix_campaign_publisher_executor", EXECUTOR_TOOL) +CANDIDATE_CAMPAIGN = load_tool( + "homebrew_prefix_campaign_publisher_candidate", + CANDIDATE_CAMPAIGN_TOOL, +) def pretty_json(value: Any) -> bytes: @@ -146,6 +155,32 @@ def real_git_checkout( return root +def exact_kandelo_data_root( + value: pathlib.Path, + expected_commit: str, +) -> pathlib.Path: + # WHY: a promoted producer checkout is inert data read by code from + # protected main. Resolving a symlink first would erase the evidence that + # the caller redirected that data boundary. + lexical_root = pathlib.Path(os.path.abspath(value)) + if lexical_root.is_symlink() or not lexical_root.is_dir(): + fail("Kandelo data source must be one real directory") + root = lexical_root.resolve() + if root != lexical_root: + fail("Kandelo data source must not traverse symlink ancestors") + if pathlib.Path(run_git(root, "rev-parse", "--show-toplevel")) != root: + fail("Kandelo data source must be the exact Git worktree root") + git_directory = root / ".git" + if git_directory.is_symlink() or not git_directory.is_dir(): + fail("Kandelo data source must own one real .git directory") + if ( + run_git(root, "rev-parse", "HEAD") != expected_commit + or run_git(root, "status", "--short", "--untracked-files=all") + ): + fail("Kandelo data source must be the exact clean candidate commit") + return root + + def parse_dependency_request( raw: str, ) -> tuple[tuple[str, str], ...]: @@ -268,7 +303,9 @@ def validate_campaign_authority( ) -def campaign_guest_layout(campaign: dict[str, Any]) -> dict[str, str]: +def campaign_guest_layout( + campaign: dict[str, Any], kandelo_root: pathlib.Path = ROOT +) -> dict[str, str]: guest_layout = EXECUTOR.exact_keys( campaign["authority"].get("guest_layout"), {"path", "sha256"}, @@ -281,11 +318,16 @@ def campaign_guest_layout(campaign: dict[str, Any]) -> dict[str, str]: "campaign guest layout SHA-256", EXECUTOR.SHA256, ) + layout_parent = kandelo_root / pathlib.Path(GUEST_LAYOUT_PATH).parent + if layout_parent.is_symlink() or not layout_parent.is_dir(): + fail("Kandelo guest layout parent must be one real directory") contract = EXECUTOR.regular_file( - ROOT / GUEST_LAYOUT_PATH, + layout_parent / pathlib.Path(GUEST_LAYOUT_PATH).name, "Kandelo guest layout contract", EXECUTOR.MAX_JSON_BYTES, ) + if contract.resolve().parent != layout_parent.resolve(): + fail("Kandelo guest layout escaped its exact data source") if EXECUTOR.sha256_file(contract) != digest: fail("Kandelo guest layout differs from campaign authority") return {"path": GUEST_LAYOUT_PATH, "sha256": digest} @@ -392,12 +434,27 @@ def default_fetch_campaign( output: pathlib.Path, receipt: pathlib.Path, ) -> None: - EXECUTOR.fetch_campaign_release( - repository=repository, - tag=tag, - output=output, - receipt_output=receipt, - ) + if CANDIDATE_CAMPAIGN_TAG.fullmatch(tag) is not None: + candidate_output = output.with_name( + f".{output.name}.candidate-campaign.json" + ) + CANDIDATE_CAMPAIGN.fetch_release( + argparse.Namespace( + repository=repository, + tag=tag, + out=str(output), + candidate_out=str(candidate_output), + receipt_out=str(receipt), + ) + ) + candidate_output.unlink() + else: + EXECUTOR.fetch_campaign_release( + repository=repository, + tag=tag, + output=output, + receipt_output=receipt, + ) def default_fetch_handoff( @@ -573,6 +630,7 @@ def prepare( receipt_output: pathlib.Path, github_env: pathlib.Path | None = None, github_output: pathlib.Path | None = None, + kandelo_root: pathlib.Path | None = None, dependencies: PreparationDependencies = PreparationDependencies(), ) -> dict[str, Any]: for value, label in ( @@ -581,9 +639,24 @@ def prepare( ): if COMMIT.fullmatch(value) is None: fail(f"{label} is invalid") + if kandelo_root is None: + layout_root = ROOT + else: + layout_root = exact_kandelo_data_root( + pathlib.Path(kandelo_root), kandelo_commit + ) formula = EXECUTOR.require_string(formula, "campaign Formula", FORMULA) campaign_match = CAMPAIGN_TAG.fullmatch(campaign_tag) - if campaign_match is None or set(campaign_match.group(1)) == {"0"}: + candidate_campaign_match = CANDIDATE_CAMPAIGN_TAG.fullmatch( + campaign_tag + ) + if ( + campaign_match is None + and candidate_campaign_match is None + ) or ( + campaign_match is not None + and set(campaign_match.group(1)) == {"0"} + ): fail("campaign tag is invalid or inert") if arch not in (None, "wasm32", "wasm64"): fail("campaign publisher architecture is invalid") @@ -631,9 +704,42 @@ def prepare( campaign, campaign_payload, index = EXECUTOR.load_campaign( campaign_path ) - guest_layout = campaign_guest_layout(campaign) - if sha256_bytes(campaign_payload) != campaign_match.group(1): - fail("campaign tag differs from the fetched campaign") + guest_layout = campaign_guest_layout(campaign, layout_root) + campaign_sha256 = sha256_bytes(campaign_payload) + if campaign_match is not None: + if campaign_sha256 != campaign_match.group(1): + fail("campaign tag differs from the fetched campaign") + else: + receipt, _receipt_payload = EXECUTOR.load_json_bytes( + campaign_receipt, "candidate campaign readback receipt" + ) + receipt = EXECUTOR.exact_keys( + receipt, + { + "campaign_sha256", + "candidate_sha256", + "kind", + "release_id", + "repository", + "schema", + "tag", + "target_commitish", + }, + "candidate campaign readback receipt", + ) + assert candidate_campaign_match is not None + if ( + receipt["schema"] != 1 + or receipt["kind"] + != "kandelo-homebrew-prefix-campaign-candidate-readback" + or receipt["tag"] != campaign_tag + or receipt["campaign_sha256"] != campaign_sha256 + or receipt["candidate_sha256"] + != candidate_campaign_match.group(1) + or str(receipt["repository"]).lower() + != tap_repository.lower() + ): + fail("candidate campaign readback receipt is not exact") if formula not in index: fail(f"Formula {formula} is outside the campaign") admission_kind = index[formula]["destination"]["admission"][ @@ -826,6 +932,14 @@ def prepare( "prefix-campaign-layout-sha256=" f"{guest_layout['sha256']}\n" ) + output.write( + "prefix-campaign-prepared-tap-commit=" + f"{prepared_commit}\n" + ) + output.write( + "prefix-campaign-prepared-tap-tree=" + f"{prepared_tree}\n" + ) return receipt finally: if transaction.exists(): @@ -868,11 +982,18 @@ def verify(*, tap_root: pathlib.Path, receipt_path: pathlib.Path) -> None: "campaign publisher campaign SHA-256", EXECUTOR.SHA256, ) - EXECUTOR.require_string( - campaign["tag"], - "campaign publisher campaign tag", - CAMPAIGN_TAG, + campaign_tag = EXECUTOR.require_string( + campaign["tag"], "campaign publisher campaign tag" ) + canonical_match = CAMPAIGN_TAG.fullmatch(campaign_tag) + candidate_match = CANDIDATE_CAMPAIGN_TAG.fullmatch(campaign_tag) + if canonical_match is None and candidate_match is None: + fail("campaign publisher campaign tag is invalid") + if ( + canonical_match is not None + and canonical_match.group(1) != campaign["sha256"] + ): + fail("campaign publisher canonical tag differs from its campaign") guest_layout = EXECUTOR.exact_keys( campaign["guest_layout"], {"path", "sha256"}, @@ -937,6 +1058,7 @@ def parse_args() -> argparse.Namespace: prepare_parser.add_argument("--receipt-out", required=True) prepare_parser.add_argument("--github-env") prepare_parser.add_argument("--github-output") + prepare_parser.add_argument("--kandelo-root") verify_parser = commands.add_parser("verify") verify_parser.add_argument("--tap-root", required=True) verify_parser.add_argument("--receipt", required=True) @@ -969,6 +1091,11 @@ def main() -> int: if arguments.github_output else None ), + kandelo_root=( + pathlib.Path(arguments.kandelo_root) + if arguments.kandelo_root + else None + ), ) elif arguments.command == "verify": verify( diff --git a/scripts/homebrew-prefix-campaign.py b/scripts/homebrew-prefix-campaign.py index cd1f946d8f..78e713eef2 100755 --- a/scripts/homebrew-prefix-campaign.py +++ b/scripts/homebrew-prefix-campaign.py @@ -2638,8 +2638,15 @@ def _derive_campaign_from_snapshots( require_timestamp(metadata["generated_at"], "old tap metadata generated_at") require_string(metadata["generator"], "old tap metadata generator") metadata_abi = require_int(metadata["kandelo_abi"], "old tap metadata ABI", 1) - if metadata_abi != current_abi: - fail("old tap metadata ABI differs from the exact current Kandelo ABI") + # WHY: an ABI bump starts with a catalog published for the preceding ABI. + # That catalog is still the collision and provenance authority for its + # immutable bottles. The variant planner below marks every bottle whose + # ABI differs from current_abi as a required rebuild. Rejecting the old + # catalog here would make it impossible to plan the first honest campaign + # for a new ABI; accepting a catalog from a newer ABI would instead plan a + # downlevel candidate from evidence this kernel cannot consume. + if metadata_abi > current_abi: + fail("old tap metadata ABI is newer than the exact Kandelo ABI") if metadata["release_tag"] != f"bottles-abi-v{metadata_abi}": fail("old tap metadata release tag does not match its ABI") require_commit( diff --git a/scripts/test-homebrew-bottle-candidate.py b/scripts/test-homebrew-bottle-candidate.py new file mode 100755 index 0000000000..c9d0523094 --- /dev/null +++ b/scripts/test-homebrew-bottle-candidate.py @@ -0,0 +1,978 @@ +#!/usr/bin/env python3 +"""Regression tests for pre-merge Homebrew bottle candidates.""" + +from __future__ import annotations + +import hashlib +import json +import os +import pathlib +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +TOOL = ROOT / "scripts/homebrew-bottle-candidate.py" +RELEASE_VALIDATOR = ( + ROOT / "scripts/validate-immutable-github-release-manifest.py" +) + + +def sha256(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def write_json(path: pathlib.Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def git(root: pathlib.Path, *arguments: str, input_text: str | None = None) -> str: + result = subprocess.run( + ["git", "-C", str(root), *arguments], + input=input_text, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + return result.stdout.strip() + + +class CandidateFixture: + def __init__(self, root: pathlib.Path, arch: str) -> None: + self.root = root + self.arch = arch + self.kandelo = root / "kandelo" + self.tap = root / "tap" + self.inputs = root / "inputs" + self.inputs.mkdir(parents=True) + self._git_repositories() + self._package_input() + self._build_and_oci() + self._evidence() + + def _git_repositories(self) -> None: + self.kandelo.mkdir() + git(self.kandelo, "init", "--quiet", "--initial-branch=main") + git(self.kandelo, "config", "user.name", "test") + git(self.kandelo, "config", "user.email", "test@example.invalid") + (self.kandelo / "base.txt").write_text("base\n") + git(self.kandelo, "add", "base.txt") + git(self.kandelo, "commit", "--quiet", "-m", "base") + self.base = git(self.kandelo, "rev-parse", "HEAD") + git(self.kandelo, "switch", "--quiet", "-c", "candidate") + (self.kandelo / "abi.txt").write_text("43\n") + (self.kandelo / "abi").mkdir() + (self.kandelo / "abi/snapshot.json").write_text('{"abi":43}\n') + (self.kandelo / "homebrew").mkdir() + (self.kandelo / "homebrew/kandelo-guest-layout.json").write_text( + '{"prefix":"/opt/kandelo/homebrew"}\n' + ) + git(self.kandelo, "add", "abi.txt", "abi", "homebrew") + git(self.kandelo, "commit", "--quiet", "-m", "abi candidate") + self.producer = git(self.kandelo, "rev-parse", "HEAD") + self.producer_tree = git(self.kandelo, "rev-parse", "HEAD^{tree}") + merge = git( + self.kandelo, + "commit-tree", + self.producer_tree, + "-p", + self.base, + "-p", + self.producer, + input_text="merge candidate\n", + ) + git(self.kandelo, "branch", "-f", "main", merge) + git(self.kandelo, "switch", "--quiet", "main") + self.merge = merge + + self.tap.mkdir() + git(self.tap, "init", "--quiet", "--initial-branch=main") + git(self.tap, "config", "user.name", "test") + git(self.tap, "config", "user.email", "test@example.invalid") + (self.tap / "Formula").mkdir() + (self.tap / "Formula/zlib.rb").write_text("class Zlib < Formula\nend\n") + git(self.tap, "add", "Formula/zlib.rb") + git(self.tap, "commit", "--quiet", "-m", "formula source") + self.tap_source = git(self.tap, "rev-parse", "HEAD") + self.tap_prepared = git( + self.tap, + "commit-tree", + git(self.tap, "rev-parse", "HEAD^{tree}"), + "-p", + self.tap_source, + input_text="prepared candidate Formula\n", + ) + (self.tap / ".github/workflows").mkdir(parents=True) + (self.tap / ".github/workflows/candidate-bottles.yml").write_text( + "name: candidate\n" + ) + git(self.tap, "add", ".github/workflows/candidate-bottles.yml") + git(self.tap, "commit", "--quiet", "-m", "candidate caller") + self.tap_caller = git(self.tap, "rev-parse", "HEAD") + + def _package_input(self) -> None: + archives = [] + for package, arch in (("rootfs", "wasm32"), ("rootfs", "wasm64")): + payload = f"{package}-{arch}-archive".encode() + archives.append( + { + "package": package, + "arch": arch, + "version": "1.0.0", + "revision": 0, + "cache_key_sha": sha256(f"{package}-{arch}".encode()), + "name": f"{package}-1.0.0-abi43-{arch}-test.tar.zst", + "sha256": sha256(payload), + "bytes": len(payload), + } + ) + self.package_input = { + "schema": 1, + "kind": "kandelo-homebrew-candidate-package-input", + "repository": "Automattic/kandelo", + "producer_commit": self.producer, + "abi": 43, + "expected_ledger_sha256": sha256(b"complete-ledger"), + "index": {"sha256": sha256(b"index"), "bytes": 5}, + "staging_release": { + "tag": "pr-42-staging-run-700-attempt-1", + "release_id": 900, + "target_commit": self.producer, + "immutable": True, + "pr_number": 42, + "run_id": 700, + "attempt": 1, + }, + "archives": archives, + } + self.package_path = self.inputs / "package-input.json" + write_json(self.package_path, self.package_input) + + def _build_and_oci(self) -> None: + self.build = self.inputs / "build" + self.oci = self.inputs / "oci" + self.build.mkdir() + (self.oci / "layout/blobs/sha256").mkdir(parents=True) + bottle = f"exact-{self.arch}-bottle".encode() + bottle_sha = sha256(bottle) + (self.build / "bottle.tar.gz").write_bytes(bottle) + write_json(self.build / "bottle.json", {"fixture": True}) + write_json(self.build / "dependency-provenance.json", {"dependencies": []}) + dependency = (self.build / "dependency-provenance.json").read_bytes() + manifest = { + "schema": 4, + "formula": "zlib", + "arch": self.arch, + "release_tag": "bottles-abi-v43", + "tap_repository": "Kandelo-dev/homebrew-tap-core", + "tap_name": "kandelo-dev/tap-core", + "tap_commit": self.tap_source, + "tap_checkout_commit": self.tap_prepared, + "kandelo_commit": self.producer, + "bottle_root_url": ( + "https://ghcr.io/v2/kandelo-dev/homebrew-tap-core" + ), + "bottle": { + "archive": "bottle.tar.gz", + "json": "bottle.json", + "tag": f"{self.arch}_kandelo", + "cellar": "/opt/kandelo/homebrew/Cellar", + "sha256": bottle_sha, + "bytes": len(bottle), + }, + "dependency_provenance": { + "json": "dependency-provenance.json", + "sha256": sha256(dependency), + "bytes": len(dependency), + }, + } + write_json(self.build / "manifest.json", manifest) + + config = b'{"architecture":"wasm"}' + oci_manifest = b'{"schemaVersion":2}' + config_sha = sha256(config) + manifest_sha = sha256(oci_manifest) + for digest, payload in ( + (bottle_sha, bottle), + (config_sha, config), + (manifest_sha, oci_manifest), + ): + (self.oci / "layout/blobs/sha256" / digest).write_bytes(payload) + write_json(self.oci / "layout/oci-layout", {"imageLayoutVersion": "1.0.0"}) + write_json( + self.oci / "layout/index.json", + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [], + }, + ) + self.oci_receipt = { + "schema": 2, + "kind": "child", + "formula": "zlib", + "arch": self.arch, + "abi": 43, + "pkg_version": "1.3.1", + "formula_revision": 0, + "bottle_rebuild": 7, + "formula_source_identity_sha256": sha256(b"formula identity"), + "formula_source_sha256": sha256(b"formula source"), + "source_closure_sha256": sha256(b"source closure"), + "kandelo_commit": self.producer, + "tap_commit": self.tap_source, + "tap_repository": "Kandelo-dev/homebrew-tap-core", + "tap_name": "kandelo-dev/tap-core", + "top_ref": "1.3.1-7", + "bottle": { + "bytes": len(bottle), + "sha256": bottle_sha, + "url": ( + "https://ghcr.io/v2/kandelo-dev/homebrew-tap-core/" + f"zlib/blobs/sha256:{bottle_sha}" + ), + }, + "oci": { + "config": { + "digest": f"sha256:{config_sha}", + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": len(config), + }, + "diff_id": f"sha256:{bottle_sha}", + "homebrew_ref": f"1.3.1.{self.arch}_kandelo.7", + "manifest": { + "digest": f"sha256:{manifest_sha}", + "size": len(oci_manifest), + }, + "platform": { + "architecture": "wasm", + "os": "kandelo", + "variant": self.arch, + }, + "transport_tag": f"sha256-{manifest_sha}", + }, + } + write_json(self.oci / "receipt.json", self.oci_receipt) + + def _evidence(self) -> None: + self.source = { + "kandelo_repository": "Automattic/kandelo", + "workflow_authority_commit": self.base, + "base_commit": self.base, + "producer_commit": self.producer, + "producer_tree": self.producer_tree, + "merge_method": "merge", + "pr_number": 42, + "abi": 43, + "abi_snapshot_sha256": sha256( + (self.kandelo / "abi/snapshot.json").read_bytes() + ), + "guest_layout": { + "path": "homebrew/kandelo-guest-layout.json", + "sha256": sha256( + ( + self.kandelo + / "homebrew/kandelo-guest-layout.json" + ).read_bytes() + ), + }, + "release_tag": "bottles-abi-v43", + "tap_repository": "Kandelo-dev/homebrew-tap-core", + "tap_name": "kandelo-dev/tap-core", + "tap_commit": self.tap_source, + "tap_checkout_commit": self.tap_prepared, + "tap_checkout_tree": git( + self.tap, "rev-parse", f"{self.tap_prepared}^{{tree}}" + ), + "prefix_campaign_tag": ( + "homebrew-prefix-campaign-candidate-pr-77-run-900-" + "attempt-2-sha256-" + "3" * 64 + ), + "prefix_campaign_layout_sha256": sha256(b"campaign layout"), + } + self.source_path = self.inputs / "source.json" + write_json(self.source_path, self.source) + attempt = 1 + self.run = { + "schema": 1, + "repository": "Kandelo-dev/homebrew-tap-core", + "workflow_path": ".github/workflows/candidate-bottles.yml", + "caller_commit": self.tap_caller, + "event": "repository_dispatch", + "run_id": 800, + "run_attempt": attempt, + "status": "in_progress", + "conclusion": None, + "artifacts": [ + { + "id": 1001, + "name": f"homebrew-build-handoff-zlib-{self.arch}-attempt-1", + "bytes": 100, + "digest": f"sha256:{sha256(b'build artifact')}", + "run_id": 800, + "run_attempt": attempt, + }, + { + "id": 1002, + "name": f"homebrew-oci-child-zlib-{self.arch}-attempt-1", + "bytes": 200, + "digest": f"sha256:{sha256(b'oci artifact')}", + "run_id": 800, + "run_attempt": attempt, + }, + { + "id": 1003, + "name": ( + "homebrew-candidate-package-input-zlib-" + f"{self.arch}-attempt-1" + ), + "bytes": 300, + "digest": f"sha256:{sha256(b'package input artifact')}", + "run_id": 800, + "run_attempt": attempt, + }, + ], + } + self.run_path = self.inputs / "run.json" + write_json(self.run_path, self.run) + self.destination = { + "formula": "zlib", + "remote": "ghcr.io/kandelo-dev/homebrew-tap-core/zlib", + "child_ref": self.oci_receipt["oci"]["transport_tag"], + "child_digest": None, + "homebrew_ref": self.oci_receipt["oci"]["homebrew_ref"], + "homebrew_ref_status": "available", + "top_ref": self.oci_receipt["top_ref"], + "child_status": "missing", + "top_status": "missing", + "top_digest": None, + "observed_at": "2026-08-01T20:00:00Z", + } + self.destination_path = self.inputs / "destination.json" + write_json(self.destination_path, self.destination) + self.dependencies: list[dict[str, object]] = [] + self.dependencies_path = self.inputs / "dependencies.json" + write_json(self.dependencies_path, self.dependencies) + self.pr = { + "number": 42, + "state": "MERGED", + "baseRefName": "main", + "headRefOid": self.producer, + "mergeCommit": {"oid": self.merge}, + } + self.pr_path = self.inputs / "pr.json" + write_json(self.pr_path, self.pr) + self.completed_run = {**self.run, "status": "completed", "conclusion": "success"} + self.completed_run_path = self.inputs / "completed-run.json" + write_json(self.completed_run_path, self.completed_run) + package_payload = json.dumps( + self.package_input, indent=2, sort_keys=True + ).encode() + b"\n" + self.admitted = { + "schema": 1, + "kind": "kandelo-homebrew-admitted-candidate-package-input", + "validated_against_main": self.merge, + "candidate_package_input_sha256": sha256(package_payload), + "package_input": self.package_input, + } + self.admitted_path = self.inputs / "admitted.json" + write_json(self.admitted_path, self.admitted) + + def prepare(self, expect_success: bool = True) -> subprocess.CompletedProcess[str]: + self.prepared = self.root / "prepared" + result = subprocess.run( + [ + "python3", + str(TOOL), + "prepare", + "--source", + str(self.source_path), + "--run-evidence", + str(self.run_path), + "--destination", + str(self.destination_path), + "--dependencies", + str(self.dependencies_path), + "--package-input", + str(self.package_path), + "--build-handoff", + str(self.build), + "--oci-child", + str(self.oci), + "--out", + str(self.prepared), + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if expect_success and result.returncode != 0: + raise AssertionError(result.stderr) + return result + + def materialize(self, expect_success: bool = True) -> subprocess.CompletedProcess[str]: + git(self.tap, "checkout", "--quiet", "--detach", self.tap_prepared) + tag = (self.prepared / "tag.txt").read_text().strip() + self.out_build = self.root / "materialized/build" + self.out_oci = self.root / "materialized/oci" + self.out_package = self.root / "materialized/package-input.json" + self.out_receipt = self.root / "materialized/promotion.json" + result = subprocess.run( + [ + "python3", + str(TOOL), + "materialize", + "--candidate-root", + str(self.prepared / "assets"), + "--candidate-tag", + tag, + "--completed-run-evidence", + str(self.completed_run_path), + "--kandelo-root", + str(self.kandelo), + "--tap-root", + str(self.tap), + "--merge-commit", + self.merge, + "--current-kandelo-main", + self.merge, + "--current-tap-main", + self.tap_caller, + "--admitted-package-input", + str(self.admitted_path), + "--dependencies", + str(self.dependencies_path), + "--out-build-handoff", + str(self.out_build), + "--out-oci-child", + str(self.out_oci), + "--out-package-input", + str(self.out_package), + "--out-receipt", + str(self.out_receipt), + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if expect_success and result.returncode != 0: + raise AssertionError(result.stderr) + return result + + +class BottleCandidateTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = pathlib.Path(self.temporary.name) + + def tearDown(self) -> None: + self.temporary.cleanup() + + def fixture(self, arch: str = "wasm32") -> CandidateFixture: + return CandidateFixture(self.root / arch, arch) + + def test_two_architectures_round_trip_exact_bytes(self) -> None: + for arch in ("wasm32", "wasm64"): + fixture = self.fixture(arch) + fixture.prepare() + fixture.materialize() + self.assertEqual( + (fixture.build / "bottle.tar.gz").read_bytes(), + (fixture.out_build / "bottle.tar.gz").read_bytes(), + ) + source_blobs = sorted( + (fixture.oci / "layout/blobs/sha256").iterdir() + ) + promoted_blobs = sorted( + (fixture.out_oci / "layout/blobs/sha256").iterdir() + ) + self.assertEqual( + [(path.name, path.read_bytes()) for path in source_blobs], + [(path.name, path.read_bytes()) for path in promoted_blobs], + ) + receipt = json.loads(fixture.out_receipt.read_text()) + self.assertEqual( + receipt["source"]["producer_commit"], fixture.producer + ) + self.assertEqual(receipt["merge_commit"], fixture.merge) + + def test_prepared_release_uses_the_shared_immutable_contract(self) -> None: + fixture = self.fixture() + fixture.prepare() + stage = fixture.root / "validated-release-assets" + normalized = fixture.root / "validated-release.json" + result = subprocess.run( + [ + "python3", + str(RELEASE_VALIDATOR), + "--manifest", + str(fixture.prepared / "release-manifest.json"), + "--asset-root", + str(fixture.prepared / "assets"), + "--stage-dir", + str(stage), + "--out-manifest", + str(normalized), + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + sorted(path.name for path in stage.iterdir()), + sorted(path.name for path in (fixture.prepared / "assets").iterdir()), + ) + + def test_package_input_binds_the_complete_validated_ledger(self) -> None: + fixture = self.fixture() + expected_entries = [] + snapshot_entries = [] + for archive in fixture.package_input["archives"]: + expected_entries.append( + { + "package": archive["package"], + "kind": "program", + "arch": archive["arch"], + "version": archive["version"], + "revision": archive["revision"], + "cache_key_sha": archive["cache_key_sha"], + "git_inputs": [], + } + ) + snapshot_entries.append( + { + "package": archive["package"], + "kind": "program", + "arch": archive["arch"], + "version": archive["version"], + "revision": archive["revision"], + "cache_key_sha": archive["cache_key_sha"], + "current": True, + "asset": archive["name"], + "archive_sha256": archive["sha256"], + "size": archive["bytes"], + } + ) + expected = fixture.root / "expected.json" + snapshot = fixture.root / "snapshot.json" + release = fixture.root / "release.json" + index = fixture.root / "index.toml" + output = fixture.root / "created-package-input.json" + write_json( + expected, + {"abi_version": 43, "entries": expected_entries}, + ) + write_json( + snapshot, + { + "abi_version": 43, + "release_tag": fixture.package_input["staging_release"]["tag"], + "complete_current": True, + "entries": snapshot_entries, + }, + ) + write_json( + release, + { + "schema": 1, + "repository": "Automattic/kandelo", + **fixture.package_input["staging_release"], + }, + ) + index.write_text("abi_version = 43\n") + result = subprocess.run( + [ + "python3", + str(TOOL), + "package-input", + "--expected-ledger", + str(expected), + "--snapshot", + str(snapshot), + "--release-evidence", + str(release), + "--index", + str(index), + "--producer-commit", + fixture.producer, + "--abi", + "43", + "--out", + str(output), + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + created = json.loads(output.read_text()) + self.assertEqual(created["archives"], fixture.package_input["archives"]) + self.assertEqual( + created["staging_release"], fixture.package_input["staging_release"] + ) + + def test_package_input_admission_requires_the_exact_merge_tree(self) -> None: + fixture = self.fixture() + output = fixture.root / "admitted-by-tool.json" + result = subprocess.run( + [ + "python3", + str(TOOL), + "admit-package-input", + "--candidate-package-input", + str(fixture.package_path), + "--regenerated-package-input", + str(fixture.package_path), + "--producer-commit", + fixture.producer, + "--validated-main", + fixture.merge, + "--validated-main-root", + str(fixture.kandelo), + "--out", + str(output), + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + admitted = json.loads(output.read_text()) + self.assertEqual(admitted["validated_against_main"], fixture.merge) + + changed = fixture.root / "changed-package-input.json" + value = dict(fixture.package_input) + value["expected_ledger_sha256"] = "4" * 64 + write_json(changed, value) + rejected = subprocess.run( + [ + "python3", + str(TOOL), + "admit-package-input", + "--candidate-package-input", + str(fixture.package_path), + "--regenerated-package-input", + str(changed), + "--producer-commit", + fixture.producer, + "--validated-main", + fixture.merge, + "--validated-main-root", + str(fixture.kandelo), + "--out", + str(fixture.root / "rejected.json"), + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("differs from the sealed candidate", rejected.stderr) + + def test_promotion_receipt_binds_reconstructed_artifacts(self) -> None: + fixture = self.fixture() + fixture.prepare() + fixture.materialize() + command = [ + "python3", + str(TOOL), + "validate-promotion", + "--receipt", + str(fixture.out_receipt), + "--candidate-tag", + (fixture.prepared / "tag.txt").read_text().strip(), + "--producer-commit", + fixture.producer, + "--merge-commit", + fixture.merge, + "--tap-commit", + fixture.tap_source, + "--tap-checkout-commit", + fixture.tap_prepared, + "--campaign-tag", + fixture.source["prefix_campaign_tag"], + "--campaign-layout-sha256", + fixture.source["prefix_campaign_layout_sha256"], + "--formula", + "zlib", + "--arch", + fixture.arch, + "--build-handoff", + str(fixture.out_build), + "--oci-child", + str(fixture.out_oci), + "--package-input", + str(fixture.out_package), + ] + result = subprocess.run( + command, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + (fixture.out_build / "bottle.tar.gz").write_bytes(b"changed") + rejected = subprocess.run( + command, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("differs from its build manifest", rejected.stderr) + + def test_promotion_requires_the_exact_package_input(self) -> None: + fixture = self.fixture() + fixture.prepare() + fixture.materialize() + value = json.loads(fixture.out_package.read_text()) + value["archives"][0]["revision"] += 1 + write_json(fixture.out_package, value) + command = [ + "python3", + str(TOOL), + "validate-promotion", + "--receipt", + str(fixture.out_receipt), + "--candidate-tag", + (fixture.prepared / "tag.txt").read_text().strip(), + "--producer-commit", + fixture.producer, + "--merge-commit", + fixture.merge, + "--tap-commit", + fixture.tap_source, + "--tap-checkout-commit", + fixture.tap_prepared, + "--campaign-tag", + fixture.source["prefix_campaign_tag"], + "--campaign-layout-sha256", + fixture.source["prefix_campaign_layout_sha256"], + "--formula", + "zlib", + "--arch", + fixture.arch, + "--build-handoff", + str(fixture.out_build), + "--oci-child", + str(fixture.out_oci), + "--package-input", + str(fixture.out_package), + ] + rejected = subprocess.run( + command, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("differs from its receipt", rejected.stderr) + + def test_promotion_rejects_an_extra_recorded_file(self) -> None: + fixture = self.fixture() + fixture.prepare() + fixture.materialize() + receipt = json.loads(fixture.out_receipt.read_text()) + receipt["files"].append( + { + "asset_name": "unexpected.bin", + "bytes": 1, + "path": "unexpected.bin", + "sha256": sha256(b"x"), + } + ) + write_json(fixture.out_receipt, receipt) + command = [ + "python3", + str(TOOL), + "validate-promotion", + "--receipt", + str(fixture.out_receipt), + "--candidate-tag", + (fixture.prepared / "tag.txt").read_text().strip(), + "--producer-commit", + fixture.producer, + "--merge-commit", + fixture.merge, + "--tap-commit", + fixture.tap_source, + "--tap-checkout-commit", + fixture.tap_prepared, + "--campaign-tag", + fixture.source["prefix_campaign_tag"], + "--campaign-layout-sha256", + fixture.source["prefix_campaign_layout_sha256"], + "--formula", + "zlib", + "--arch", + fixture.arch, + "--build-handoff", + str(fixture.out_build), + "--oci-child", + str(fixture.out_oci), + "--package-input", + str(fixture.out_package), + ] + rejected = subprocess.run( + command, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("exact artifact files", rejected.stderr) + + def test_candidate_authority_must_equal_the_base(self) -> None: + fixture = self.fixture() + fixture.source["workflow_authority_commit"] = fixture.producer + write_json(fixture.source_path, fixture.source) + result = fixture.prepare(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("must equal its protected base", result.stderr) + + def test_substituted_release_asset_is_rejected(self) -> None: + fixture = self.fixture() + fixture.prepare() + asset = fixture.prepared / "assets/build-bottle.tar.gz" + asset.write_bytes(b"substituted") + result = fixture.materialize(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("differs from candidate.json", result.stderr) + + def test_partial_or_changed_package_generation_is_rejected(self) -> None: + fixture = self.fixture() + fixture.prepare() + admitted = json.loads(fixture.admitted_path.read_text()) + admitted["package_input"]["archives"].pop() + write_json(fixture.admitted_path, admitted) + result = fixture.materialize(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("does not contain the candidate archives", result.stderr) + + def test_wrong_merge_parent_is_rejected(self) -> None: + fixture = self.fixture() + fixture.prepare() + wrong = git( + fixture.kandelo, + "commit-tree", + fixture.producer_tree, + "-p", + fixture.producer, + input_text="wrong merge\n", + ) + git(fixture.kandelo, "reset", "--hard", wrong) + fixture.merge = wrong + fixture.pr["mergeCommit"]["oid"] = wrong + write_json(fixture.pr_path, fixture.pr) + fixture.admitted["validated_against_main"] = wrong + write_json(fixture.admitted_path, fixture.admitted) + result = fixture.materialize(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("does not preserve the prepared base", result.stderr) + + def test_failed_workflow_run_is_rejected(self) -> None: + fixture = self.fixture() + fixture.prepare() + failed = {**fixture.completed_run, "conclusion": "failure"} + write_json(fixture.completed_run_path, failed) + result = fixture.materialize(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("conclusion is not successful", result.stderr) + + def test_dependency_substitution_is_rejected(self) -> None: + fixture = self.fixture() + fixture.prepare() + dependencies = [ + { + "formula": "dependency", + "manifest_sha256": "3" * 64, + "tag": "homebrew-prefix-handoff-sha256-" + "3" * 64, + } + ] + write_json(fixture.dependencies_path, dependencies) + result = fixture.materialize(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("activation dependencies differ", result.stderr) + + def test_unsorted_dependencies_are_rejected(self) -> None: + fixture = self.fixture() + fixture.dependencies = [ + { + "formula": "z-last", + "manifest_sha256": "1" * 64, + "tag": "homebrew-prefix-handoff-sha256-" + "1" * 64, + }, + { + "formula": "a-first", + "manifest_sha256": "2" * 64, + "tag": "homebrew-prefix-handoff-sha256-" + "2" * 64, + }, + ] + write_json(fixture.dependencies_path, fixture.dependencies) + result = fixture.prepare(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("unique and sorted", result.stderr) + + def test_existing_homebrew_ref_requires_new_rebuild(self) -> None: + fixture = self.fixture() + fixture.destination["child_status"] = "present" + fixture.destination["child_digest"] = \ + fixture.oci_receipt["oci"]["manifest"]["digest"] + fixture.destination["homebrew_ref_status"] = "occupied" + fixture.destination["top_status"] = "present" + fixture.destination["top_digest"] = "sha256:" + "4" * 64 + write_json(fixture.destination_path, fixture.destination) + result = fixture.prepare(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("not collision-free", result.stderr) + self.assertFalse((fixture.root / "prepared").exists()) + + fixture.destination["child_status"] = "missing" + fixture.destination["child_digest"] = None + fixture.destination["homebrew_ref_status"] = "available" + fixture.destination["top_status"] = "present" + fixture.destination["top_digest"] = "sha256:" + "4" * 64 + write_json(fixture.destination_path, fixture.destination) + fixture.prepare() + self.assertTrue((fixture.prepared / "assets/candidate.json").is_file()) + + def test_ambiguous_artifact_set_is_rejected(self) -> None: + fixture = self.fixture() + extra = dict(fixture.run["artifacts"][0]) + extra["id"] = 1004 + extra["name"] = "unexpected" + fixture.run["artifacts"].append(extra) + write_json(fixture.run_path, fixture.run) + result = fixture.prepare(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("exactly three candidate artifacts", result.stderr) + + def test_output_retry_does_not_overwrite_completed_result(self) -> None: + fixture = self.fixture() + fixture.prepare() + original = (fixture.prepared / "assets/candidate.json").read_bytes() + result = fixture.prepare(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("must not already exist", result.stderr) + self.assertEqual( + original, (fixture.prepared / "assets/candidate.json").read_bytes() + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test-homebrew-candidate-caller-pins.py b/scripts/test-homebrew-candidate-caller-pins.py new file mode 100755 index 0000000000..8ceb5d0be2 --- /dev/null +++ b/scripts/test-homebrew-candidate-caller-pins.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Regression tests for candidate caller rendering and pins.""" + +from __future__ import annotations + +import pathlib +import subprocess +import tempfile +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +TOOL = ROOT / "scripts/homebrew-candidate-caller-pins.py" +TEMPLATE = ROOT / "homebrew/homebrew-tap-core" + + +class CallerPinTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = pathlib.Path(self.temporary.name) + self.base = "a" * 40 + self.merge = "b" * 40 + self.output = self.root / "rendered" + + def tearDown(self) -> None: + self.temporary.cleanup() + + def render(self) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "python3", + str(TOOL), + "render", + "--template-root", + str(TEMPLATE), + "--base-sha", + self.base, + "--merge-sha", + self.merge, + "--out", + str(self.output), + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + def validate(self, mode: str, sha: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "python3", + str(TOOL), + "validate", + "--tap-root", + str(self.output), + "--mode", + mode, + "--kandelo-sha", + sha, + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + def test_rendered_callers_pin_base_and_merge_exactly(self) -> None: + result = self.render() + self.assertEqual(result.returncode, 0, result.stderr) + for mode, sha in ( + ("campaign", self.base), + ("bottle", self.base), + ("promotion", self.merge), + ): + validated = self.validate(mode, sha) + self.assertEqual(validated.returncode, 0, validated.stderr) + + def test_wrong_expected_commit_is_rejected(self) -> None: + result = self.render() + self.assertEqual(result.returncode, 0, result.stderr) + rejected = self.validate("bottle", self.merge) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("must pin exactly", rejected.stderr) + + def test_unrendered_template_is_not_deployable(self) -> None: + self.output = TEMPLATE + rejected = self.validate("campaign", self.base) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("must pin exactly", rejected.stderr) + + def test_mutable_ref_is_rejected_even_with_the_exact_pin(self) -> None: + result = self.render() + self.assertEqual(result.returncode, 0, result.stderr) + caller = self.output / ".github/workflows/candidate-bottles.yml" + caller.write_text(caller.read_text() + "# forbidden @main ref\n") + rejected = self.validate("bottle", self.base) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("mutable or unresolved", rejected.stderr) + + def test_extra_reusable_workflow_call_is_rejected(self) -> None: + result = self.render() + self.assertEqual(result.returncode, 0, result.stderr) + caller = self.output / ".github/workflows/candidate-bottles.yml" + caller.write_text( + caller.read_text() + + " uses: Automattic/kandelo/.github/workflows/extra.yml@" + + self.base + + "\n" + ) + rejected = self.validate("bottle", self.base) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("must pin exactly", rejected.stderr) + + def test_render_requires_exact_lowercase_commit_shas(self) -> None: + self.base = "A" * 40 + rejected = self.render() + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("exact lowercase commit SHA", rejected.stderr) + + def test_render_does_not_replace_an_existing_output(self) -> None: + result = self.render() + self.assertEqual(result.returncode, 0, result.stderr) + rejected = self.render() + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("already exists", rejected.stderr) + + def test_validate_rejects_a_symlinked_caller(self) -> None: + result = self.render() + self.assertEqual(result.returncode, 0, result.stderr) + caller = self.output / ".github/workflows/candidate-campaign.yml" + target = self.root / "caller.yml" + caller.rename(target) + caller.symlink_to(target) + rejected = self.validate("campaign", self.base) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("must be a regular file", rejected.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test-homebrew-candidate-campaign.py b/scripts/test-homebrew-candidate-campaign.py new file mode 100755 index 0000000000..134f422d4d --- /dev/null +++ b/scripts/test-homebrew-candidate-campaign.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +"""Adversarial tests for pre-merge Homebrew campaign evidence.""" + +from __future__ import annotations + +import importlib.util +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile +import types +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parent.parent +TOOL_PATH = ROOT / "scripts/homebrew-candidate-campaign.py" +sys.dont_write_bytecode = True +SPEC = importlib.util.spec_from_file_location( + "homebrew_candidate_campaign_test_tool", TOOL_PATH +) +assert SPEC is not None and SPEC.loader is not None +TOOL = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = TOOL +SPEC.loader.exec_module(TOOL) + +KANDELO_REPOSITORY = "Automattic/kandelo" +TAP_REPOSITORY = "kandelo-dev/homebrew-tap-core" +TAP_NAME = "kandelo-dev/tap-core" + + +def write_json(path: pathlib.Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(TOOL.pretty_json(value)) + + +def run(root: pathlib.Path, *arguments: str) -> str: + result = subprocess.run( + list(arguments), + cwd=root, + check=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + return result.stdout.strip() + + +def git(root: pathlib.Path, *arguments: str) -> str: + return run(root, "git", *arguments) + + +def commit(root: pathlib.Path, message: str) -> str: + git(root, "add", "-A") + git( + root, + "-c", + "user.name=Candidate Campaign Test", + "-c", + "user.email=candidate@example.invalid", + "commit", + "-m", + message, + ) + return git(root, "rev-parse", "HEAD") + + +def formula_record() -> dict[str, object]: + return { + "dependencies": [], + "destination": { + "admission": { + "kind": "anonymous-absence", + "method": "anonymous-oras-manifest-probe", + "probe": { + "digest": None, + "kind": "manifest", + "schema": 1, + "status": "missing", + }, + "schema": 1, + }, + "bottle_rebuild": 0, + "reference": "1.0", + "remote": f"ghcr.io/{TAP_REPOSITORY}/alpha", + }, + "formula_source": { + "identity_excluding_bottle_sha256": "8" * 64, + "path": "Formula/alpha.rb", + "sha256": "9" * 64, + }, + "name": "alpha", + "source_kind": "reviewed-new-entrant", + "variants": [ + { + "arch": "wasm32", + "build_input": {"kind": "formula-source"}, + "disposition": { + "kind": "required-build", + "reasons": ["new-campaign-entrant"], + }, + "selected_by": "reviewed-campaign-input", + } + ], + "version": "1.0", + } + + +class CandidateCampaignFixture: + def __init__(self) -> None: + self.temporary = tempfile.TemporaryDirectory( + prefix="homebrew-candidate-campaign-test-" + ) + self.root = pathlib.Path(self.temporary.name) + self.base = "1" * 40 + self.producer = "2" * 40 + self.producer_tree = "3" * 40 + self.tap = "4" * 40 + self.tap_tree = "5" * 40 + self.tap_authority = "6" * 40 + self.native = "7" * 40 + self.source = { + "schema": 1, + "kind": "kandelo-homebrew-prefix-campaign-candidate-source", + "kandelo_repository": KANDELO_REPOSITORY, + "pr_number": 77, + "base_commit": self.base, + "producer_commit": self.producer, + "producer_tree": self.producer_tree, + "workflow_authority_commit": self.base, + "abi": 43, + "abi_snapshot": { + "path": "abi/snapshot.json", + "sha256": "a" * 64, + }, + "guest_layout": { + "path": "homebrew/kandelo-guest-layout.json", + "sha256": "b" * 64, + }, + "tap_repository": TAP_REPOSITORY, + "tap_name": TAP_NAME, + "source_tap_commit": self.tap, + "source_tap_tree": self.tap_tree, + "tap_workflow_authority_commit": self.tap_authority, + "old_metadata": { + "path": "Kandelo/metadata.json", + "sha256": "c" * 64, + }, + "native_homebrew_commit": self.native, + } + self.run = { + "schema": 1, + "repository": TAP_REPOSITORY, + "workflow_path": ".github/workflows/candidate-campaign.yml", + "caller_commit": self.tap_authority, + "event": "repository_dispatch", + "run_id": 900, + "run_attempt": 2, + "status": "in_progress", + "conclusion": None, + "artifacts": [ + { + "id": 901, + "name": ( + "homebrew-candidate-campaign-derivation-attempt-2" + ), + "bytes": 1234, + "digest": "sha256:" + "d" * 64, + "run_id": 900, + "run_attempt": 2, + } + ], + } + self.campaign = { + "schema": 2, + "kind": "kandelo-homebrew-guest-prefix-campaign", + "authority": { + "abi_snapshot": self.source["abi_snapshot"], + "current_kandelo_abi": 43, + "guest_layout": self.source["guest_layout"], + "kandelo_commit": self.producer, + "native_homebrew_commit": self.native, + "old_metadata": self.source["old_metadata"], + "old_tap_commit": self.tap, + "source_materialization": { + "kind": "exact-git-tree-v1", + "tree_git_oid": self.tap_tree, + }, + "source_tap_commit": self.tap, + "tap_name": TAP_NAME, + "tap_repository": TAP_REPOSITORY, + }, + "formulae": [formula_record()], + } + self.source_path = self.root / "source.json" + self.run_path = self.root / "run.json" + self.campaign_path = self.root / "campaign.json" + write_json(self.source_path, self.source) + write_json(self.run_path, self.run) + write_json(self.campaign_path, self.campaign) + + def close(self) -> None: + self.temporary.cleanup() + + +class CandidateCampaignTests(unittest.TestCase): + def test_prepare_uses_noncanonical_content_addressed_namespace(self) -> None: + fixture = CandidateCampaignFixture() + self.addCleanup(fixture.close) + output = fixture.root / "prepared" + TOOL.prepare( + types.SimpleNamespace( + source=str(fixture.source_path), + run_evidence=str(fixture.run_path), + campaign=str(fixture.campaign_path), + out=str(output), + ) + ) + tag = (output / "tag.txt").read_text().strip() + self.assertRegex(tag, TOOL.CANDIDATE_TAG) + self.assertNotRegex(tag, TOOL.EXECUTOR.CAMPAIGN_TAG) + release = json.loads((output / "release-manifest.json").read_text()) + self.assertEqual(release["target_commitish"], fixture.tap_authority) + self.assertEqual( + {asset["name"] for asset in release["assets"]}, + {"campaign.json", "candidate-campaign.json"}, + ) + description = fixture.root / "description.json" + TOOL.describe_release( + types.SimpleNamespace( + candidate=str(output / "assets/candidate-campaign.json"), + campaign=str(output / "assets/campaign.json"), + candidate_tag=tag, + out=str(description), + ) + ) + described = json.loads(description.read_text()) + self.assertEqual(described["manifest"]["source"], fixture.source) + self.assertEqual( + described["campaign_sha256"], + TOOL.sha256_file(fixture.campaign_path), + ) + + def test_prepare_rejects_campaign_from_another_producer(self) -> None: + fixture = CandidateCampaignFixture() + self.addCleanup(fixture.close) + fixture.campaign["authority"]["kandelo_commit"] = "e" * 40 + write_json(fixture.campaign_path, fixture.campaign) + with self.assertRaisesRegex( + TOOL.CandidateCampaignError, "authority differs" + ): + TOOL.prepare( + types.SimpleNamespace( + source=str(fixture.source_path), + run_evidence=str(fixture.run_path), + campaign=str(fixture.campaign_path), + out=str(fixture.root / "rejected"), + ) + ) + + def test_exact_merge_rejects_premerge_and_admits_preserved_head(self) -> None: + with tempfile.TemporaryDirectory( + prefix="candidate-campaign-merge-test-" + ) as temporary_name: + root = pathlib.Path(temporary_name) + repository = root / "repository" + repository.mkdir() + git(repository, "init", "-q", "-b", "main") + (repository / "value").write_text("base\n") + base = commit(repository, "base") + git(repository, "checkout", "-q", "-b", "candidate") + (repository / "value").write_text("candidate\n") + producer = commit(repository, "candidate") + producer_tree = git(repository, "rev-parse", "HEAD^{tree}") + producer_root = root / "producer" + run(root, "git", "clone", "-q", str(repository), str(producer_root)) + git(producer_root, "checkout", "-q", producer) + + source = { + "pr_number": 77, + "base_commit": base, + "producer_commit": producer, + "producer_tree": producer_tree, + "workflow_authority_commit": base, + } + with self.assertRaisesRegex( + TOOL.CandidateCampaignError, + r"preserve \[base, exact head\]", + ): + TOOL.validate_exact_merge( + producer_root, + producer_root, + source, + producer, + producer, + ) + + git(repository, "checkout", "-q", "main") + git( + repository, + "-c", + "user.name=Candidate Campaign Test", + "-c", + "user.email=candidate@example.invalid", + "merge", + "--no-ff", + "--no-edit", + "candidate", + ) + merged = git(repository, "rev-parse", "HEAD") + self.assertEqual( + git(repository, "show", "-s", "--format=%P", merged), + f"{base} {producer}", + ) + TOOL.validate_exact_merge( + repository, + producer_root, + source, + merged, + merged, + ) + + def test_admission_validator_binds_merge_source_abi_and_layout(self) -> None: + tag = ( + "homebrew-prefix-campaign-candidate-pr-77-run-900-attempt-2-" + "sha256-" + "d" * 64 + ) + receipt = { + "schema": 1, + "kind": "kandelo-homebrew-prefix-campaign-candidate-admission", + "candidate_tag": tag, + "candidate_sha256": "d" * 64, + "campaign_sha256": "e" * 64, + "producer_commit": "1" * 40, + "merge_commit": "2" * 40, + "validated_against_main": "2" * 40, + "source_tap_commit": "3" * 40, + "tap_workflow_authority_commit": "4" * 40, + "abi": 43, + "abi_snapshot_sha256": "5" * 64, + "guest_layout_sha256": "6" * 64, + "run_id": 900, + "run_attempt": 2, + } + with tempfile.TemporaryDirectory( + prefix="candidate-campaign-admission-test-" + ) as temporary_name: + path = pathlib.Path(temporary_name) / "receipt.json" + write_json(path, receipt) + arguments = types.SimpleNamespace( + receipt=str(path), + candidate_tag=tag, + producer_commit="1" * 40, + merge_commit="2" * 40, + source_tap_commit="3" * 40, + abi=43, + guest_layout_sha256="6" * 64, + ) + TOOL.validate_admission(arguments) + arguments.abi = 42 + with self.assertRaisesRegex( + TOOL.CandidateCampaignError, "publication field abi" + ): + TOOL.validate_admission(arguments) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test-homebrew-candidate-release-receipt.py b/scripts/test-homebrew-candidate-release-receipt.py new file mode 100755 index 0000000000..4e74855671 --- /dev/null +++ b/scripts/test-homebrew-candidate-release-receipt.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +"""Regression tests for durable candidate release receipts.""" + +from __future__ import annotations + +import hashlib +import json +import pathlib +import subprocess +import tempfile +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +TOOL = ROOT / "scripts/homebrew-candidate-release-receipt.py" + + +def sha256(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def write_json(path: pathlib.Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +class ReceiptFixture: + def __init__(self, root: pathlib.Path) -> None: + self.root = root + self.assets = root / "assets" + self.assets.mkdir(parents=True) + payloads = { + "candidate.json": b'{"candidate":true}\n', + "bottle.tar.gz": b"exact bottle bytes", + } + self.tag = "homebrew-bottle-candidate-pr-1-run-2-attempt-3-sha256-" + ( + "a" * 64 + ) + self.target = "b" * 40 + self.repository = "Kandelo-dev/homebrew-tap-core" + receipts = [] + live = [] + for asset_id, name in enumerate(sorted(payloads), start=10): + payload = payloads[name] + (self.assets / name).write_bytes(payload) + url = ( + "https://github.com/Kandelo-dev/homebrew-tap-core/" + f"releases/download/{self.tag}/{name}" + ) + receipts.append( + { + "asset_id": asset_id, + "bytes": len(payload), + "name": name, + "sha256": sha256(payload), + "url": url, + } + ) + live.append( + { + "id": asset_id, + "name": name, + "state": "uploaded", + "size": len(payload), + "digest": f"sha256:{sha256(payload)}", + "browser_download_url": url, + } + ) + self.receipt = { + "schema": 1, + "status": "success", + "visibility": "public-anonymous-readback", + "repository": self.repository, + "tag": self.tag, + "target_commitish": self.target, + "release_id": 9, + "immutable": True, + "assets": receipts, + } + self.release = { + "id": 9, + "tag_name": self.tag, + "target_commitish": self.target, + "immutable": True, + "draft": False, + "prerelease": False, + } + self.receipt_path = root / "receipt.json" + self.release_path = root / "release.json" + self.live_path = root / "live-assets.json" + self.plan_path = root / "plan.json" + write_json(self.receipt_path, self.receipt) + write_json(self.release_path, self.release) + write_json(self.live_path, live) + + def plan(self, expect_success: bool = True) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + [ + "python3", + str(TOOL), + "plan", + "--receipt", + str(self.receipt_path), + "--release", + str(self.release_path), + "--release-assets", + str(self.live_path), + "--repository", + self.repository, + "--tag", + self.tag, + "--target-commit", + self.target, + "--out", + str(self.plan_path), + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if expect_success and result.returncode != 0: + raise AssertionError(result.stderr) + return result + + def verify(self) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "python3", + str(TOOL), + "verify-readback", + "--plan", + str(self.plan_path), + "--asset-root", + str(self.assets), + ], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +class ReceiptTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = pathlib.Path(self.temporary.name) + + def tearDown(self) -> None: + self.temporary.cleanup() + + def fixture(self) -> ReceiptFixture: + return ReceiptFixture(self.root) + + def test_exact_receipt_and_anonymous_readback_are_accepted(self) -> None: + fixture = self.fixture() + fixture.plan() + self.assertEqual(fixture.verify().returncode, 0) + + def test_receipt_rejects_extra_keys(self) -> None: + fixture = self.fixture() + fixture.receipt["untrusted"] = True + write_json(fixture.receipt_path, fixture.receipt) + result = fixture.plan(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("must contain exactly", result.stderr) + + def test_receipt_rejects_duplicate_json_keys(self) -> None: + fixture = self.fixture() + fixture.receipt_path.write_text('{"schema":1,"schema":1}\n') + result = fixture.plan(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("repeats key", result.stderr) + + def test_live_release_identity_must_match_receipt(self) -> None: + changes = { + "id": 10, + "tag_name": "another-tag", + "target_commitish": "c" * 40, + "immutable": False, + "draft": True, + "prerelease": True, + } + for field, changed in changes.items(): + with self.subTest(field=field): + fixture = ReceiptFixture(self.root / field) + fixture.release[field] = changed + write_json(fixture.release_path, fixture.release) + result = fixture.plan(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "release differs from the protected receipt", + result.stderr, + ) + + def test_live_asset_digest_must_match_receipt(self) -> None: + fixture = self.fixture() + live = json.loads(fixture.live_path.read_text()) + live[0]["digest"] = "sha256:" + "f" * 64 + write_json(fixture.live_path, live) + result = fixture.plan(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("differs from the protected receipt", result.stderr) + + def test_live_asset_identity_must_match_receipt(self) -> None: + changes = { + "id": 99, + "size": 99, + "browser_download_url": "https://github.com/wrong/release", + } + for field, changed in changes.items(): + with self.subTest(field=field): + fixture = ReceiptFixture(self.root / field) + live = json.loads(fixture.live_path.read_text()) + live[0][field] = changed + write_json(fixture.live_path, live) + result = fixture.plan(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "differs from the protected receipt", + result.stderr, + ) + + def test_full_live_inventory_is_required(self) -> None: + fixture = self.fixture() + live = json.loads(fixture.live_path.read_text()) + live.pop() + write_json(fixture.live_path, live) + result = fixture.plan(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("inventory differs", result.stderr) + + def test_extra_live_asset_is_rejected(self) -> None: + fixture = self.fixture() + live = json.loads(fixture.live_path.read_text()) + extra = dict(live[-1]) + extra["id"] = 99 + extra["name"] = "unexpected.bin" + extra["browser_download_url"] += ".unexpected" + live.append(extra) + write_json(fixture.live_path, live) + result = fixture.plan(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("inventory differs", result.stderr) + + def test_anonymous_bytes_must_still_match(self) -> None: + fixture = self.fixture() + fixture.plan() + (fixture.assets / "bottle.tar.gz").write_bytes(b"changed") + result = fixture.verify() + self.assertNotEqual(result.returncode, 0) + self.assertIn("changed", result.stderr) + + def test_anonymous_readback_rejects_symlinks(self) -> None: + fixture = self.fixture() + fixture.plan() + bottle = fixture.assets / "bottle.tar.gz" + payload = fixture.root / "outside.bin" + payload.write_bytes(bottle.read_bytes()) + bottle.unlink() + bottle.symlink_to(payload) + result = fixture.verify() + self.assertNotEqual(result.returncode, 0) + self.assertIn("is not regular", result.stderr) + + def test_plan_output_is_not_overwritten(self) -> None: + fixture = self.fixture() + fixture.plan() + result = fixture.plan(expect_success=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn("already exists", result.stderr) + + def test_readback_plan_rejects_extra_keys(self) -> None: + fixture = self.fixture() + fixture.plan() + plan = json.loads(fixture.plan_path.read_text()) + plan["untrusted"] = True + write_json(fixture.plan_path, plan) + result = fixture.verify() + self.assertNotEqual(result.returncode, 0) + self.assertIn("must contain exactly", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test-homebrew-prefix-campaign-publisher.py b/scripts/test-homebrew-prefix-campaign-publisher.py index db66f75890..3e2bacc9c1 100755 --- a/scripts/test-homebrew-prefix-campaign-publisher.py +++ b/scripts/test-homebrew-prefix-campaign-publisher.py @@ -568,6 +568,116 @@ def prepare( class PrefixCampaignPublisherTests(unittest.TestCase): + def test_candidate_campaign_requires_exact_readback_receipt(self) -> None: + fixture = Fixture() + self.addCleanup(fixture.close) + candidate_digest = "d" * 64 + candidate_tag = ( + "homebrew-prefix-campaign-candidate-pr-77-run-900-" + f"attempt-2-sha256-{candidate_digest}" + ) + + def fetch_candidate( + repository: str, + tag: str, + output: pathlib.Path, + receipt: pathlib.Path, + ) -> None: + self.assertEqual(repository, TAP_REPOSITORY) + self.assertEqual(tag, candidate_tag) + shutil.copy2(fixture.campaign_path, output) + write_json( + receipt, + { + "campaign_sha256": sha256( + fixture.campaign_path.read_bytes() + ), + "candidate_sha256": candidate_digest, + "kind": ( + "kandelo-homebrew-prefix-campaign-" + "candidate-readback" + ), + "release_id": 123, + "repository": TAP_REPOSITORY, + "schema": 1, + "tag": candidate_tag, + "target_commitish": "f" * 40, + }, + ) + + dependencies = PUBLISHER.PreparationDependencies( + fetch_campaign=fetch_candidate, + fetch_handoff=fixture.fetch_handoff, + merge_dependency=fixture.merge_dependency, + ) + receipt = PUBLISHER.prepare( + tap_root=fixture.tap, + kandelo_commit=KANDELO_COMMIT, + tap_repository=TAP_REPOSITORY, + tap_name=TAP_NAME, + source_tap_commit=fixture.source_commit, + campaign_tag=candidate_tag, + dependency_request='{"dependencies":[],"schema":1}', + formula="alpha", + arch="wasm32", + work_root=fixture.root / "candidate-publisher-work", + receipt_output=fixture.root / "candidate-publisher-receipt.json", + dependencies=dependencies, + ) + self.assertEqual(receipt["campaign"]["tag"], candidate_tag) + + fixture = Fixture() + self.addCleanup(fixture.close) + + def fetch_bad_candidate( + _repository: str, + _tag: str, + output: pathlib.Path, + receipt: pathlib.Path, + ) -> None: + shutil.copy2(fixture.campaign_path, output) + write_json( + receipt, + { + "campaign_sha256": "0" * 64, + "candidate_sha256": candidate_digest, + "kind": ( + "kandelo-homebrew-prefix-campaign-" + "candidate-readback" + ), + "release_id": 123, + "repository": TAP_REPOSITORY, + "schema": 1, + "tag": candidate_tag, + "target_commitish": "f" * 40, + }, + ) + + with self.assertRaisesRegex( + PUBLISHER.PublisherCampaignError, + "candidate campaign readback receipt is not exact", + ): + PUBLISHER.prepare( + tap_root=fixture.tap, + kandelo_commit=KANDELO_COMMIT, + tap_repository=TAP_REPOSITORY, + tap_name=TAP_NAME, + source_tap_commit=fixture.source_commit, + campaign_tag=candidate_tag, + dependency_request='{"dependencies":[],"schema":1}', + formula="alpha", + arch="wasm32", + work_root=fixture.root / "bad-candidate-publisher-work", + receipt_output=( + fixture.root / "bad-candidate-publisher-receipt.json" + ), + dependencies=PUBLISHER.PreparationDependencies( + fetch_campaign=fetch_bad_candidate, + fetch_handoff=fixture.fetch_handoff, + merge_dependency=fixture.merge_dependency, + ), + ) + def test_dependency_input_accepts_build_and_reuse_handoffs( self, ) -> None: @@ -722,7 +832,11 @@ def test_sealed_target_and_dependency_bottle_become_clean_snapshot( "prefix-campaign-destination-admission-kind=" "anonymous-absence\n" "prefix-campaign-layout-sha256=" - f"{GUEST_LAYOUT_SHA256}\n", + f"{GUEST_LAYOUT_SHA256}\n" + "prefix-campaign-prepared-tap-commit=" + f"{receipt['preparation']['commit']}\n" + "prefix-campaign-prepared-tap-tree=" + f"{receipt['preparation']['tree_git_oid']}\n", ) self.assertEqual( run( diff --git a/scripts/test-homebrew-prefix-campaign.py b/scripts/test-homebrew-prefix-campaign.py index 58f995e140..862b074020 100755 --- a/scripts/test-homebrew-prefix-campaign.py +++ b/scripts/test-homebrew-prefix-campaign.py @@ -1258,19 +1258,54 @@ def test_unchanged_formula_is_required_for_reuse(self) -> None: {"kind": "byte-clean-reuse-candidate", "reasons": []}, ) - def test_metadata_and_selected_sidecar_must_match_current_authority( + def test_older_catalog_for_new_abi_forces_every_variant_to_rebuild( self, ) -> None: + fixture = make_fixture(alpha_source_changed=False) + self.addCleanup(fixture.close) + (fixture.kandelo / "crates/shared/src/lib.rs").write_text( + "pub const ABI_VERSION: u32 = 43;\n" + ) + write_json( + fixture.kandelo / "abi/snapshot.json", {"abi_version": 43} + ) + kandelo_head = commit(fixture.kandelo, "advance fixture to ABI 43") + + result = CAMPAIGN.derive_campaign( + fixture.options(kandelo_commit=kandelo_head), + fixture.dependencies(), + ) + + self.assertEqual(result["authority"]["current_kandelo_abi"], 43) + old_variants = [ + variant + for formula in result["formulae"] + for variant in formula["variants"] + if "old_record" in variant + ] + self.assertTrue(old_variants) + self.assertTrue( + all( + variant["disposition"]["kind"] == "required-rebuild" + and "abi-mismatch" in variant["disposition"]["reasons"] + for variant in old_variants + ) + ) + self.assertEqual( + result["summary"]["byte_clean_reuse_candidates"], 0 + ) + + def test_future_catalog_is_rejected_for_downlevel_candidate(self) -> None: fixture = make_fixture() self.addCleanup(fixture.close) metadata_path = fixture.old_tap / "Kandelo/metadata.json" metadata = json.loads(metadata_path.read_text()) - metadata["kandelo_abi"] = 41 - metadata["release_tag"] = "bottles-abi-v41" + metadata["kandelo_abi"] = 43 + metadata["release_tag"] = "bottles-abi-v43" write_json(metadata_path, metadata) - metadata_head = commit(fixture.old_tap, "mismatch selected metadata ABI") + metadata_head = commit(fixture.old_tap, "future selected metadata ABI") with self.assertRaisesRegex( - CAMPAIGN.CampaignError, "differs from the exact current Kandelo ABI" + CAMPAIGN.CampaignError, "newer than the exact Kandelo ABI" ): CAMPAIGN.derive_campaign( fixture.options( @@ -1280,6 +1315,9 @@ def test_metadata_and_selected_sidecar_must_match_current_authority( fixture.dependencies(), ) + def test_metadata_and_selected_sidecar_must_match_current_authority( + self, + ) -> None: fixture = make_fixture() self.addCleanup(fixture.close) sidecar_path = fixture.old_tap / "Kandelo/formula/alpha.json" @@ -1326,6 +1364,23 @@ def test_abi_snapshot_and_metadata_generator_are_bound(self) -> None: fixture.dependencies(), ) + fixture = make_fixture() + self.addCleanup(fixture.close) + sidecar_path = fixture.old_tap / "Kandelo/formula/alpha.json" + sidecar_value = json.loads(sidecar_path.read_text()) + sidecar_value["kandelo_abi"] = 41 + write_json(sidecar_path, sidecar_value) + sidecar_head = commit( + fixture.old_tap, "mismatch selected sidecar ABI" + ) + with self.assertRaisesRegex( + CAMPAIGN.CampaignError, "sidecar ABI/tap_commit" + ): + CAMPAIGN.derive_campaign( + fixture.options(old_tap_commit=sidecar_head), + fixture.dependencies(), + ) + fixture = make_fixture() self.addCleanup(fixture.close) metadata_path = fixture.old_tap / "Kandelo/metadata.json" diff --git a/scripts/test-homebrew-publish-workflow.sh b/scripts/test-homebrew-publish-workflow.sh index 69ee4cf33b..04999b72fd 100755 --- a/scripts/test-homebrew-publish-workflow.sh +++ b/scripts/test-homebrew-publish-workflow.sh @@ -7550,6 +7550,14 @@ PYTHONDONTWRITEBYTECODE=1 \ python3 "$REPO_ROOT/scripts/test-homebrew-prefix-campaign-executor.py" PYTHONDONTWRITEBYTECODE=1 \ python3 "$REPO_ROOT/scripts/test-homebrew-prefix-campaign-publisher.py" +PYTHONDONTWRITEBYTECODE=1 \ + python3 "$REPO_ROOT/scripts/test-homebrew-candidate-campaign.py" +PYTHONDONTWRITEBYTECODE=1 \ + python3 "$REPO_ROOT/scripts/test-homebrew-bottle-candidate.py" +PYTHONDONTWRITEBYTECODE=1 \ + python3 "$REPO_ROOT/scripts/test-homebrew-candidate-release-receipt.py" +PYTHONDONTWRITEBYTECODE=1 \ + python3 "$REPO_ROOT/scripts/test-homebrew-candidate-caller-pins.py" bash "$REPO_ROOT/scripts/test-homebrew-inspect-bottle.sh" bash "$REPO_ROOT/scripts/test-homebrew-formula-runtime-closure.sh" bash "$REPO_ROOT/scripts/test-homebrew-validate-host-dependency-plan.sh"