From 0e46dad6975993553e213181ac6d2acd5597a3df Mon Sep 17 00:00:00 2001 From: Salvatore Cuzzilla Date: Sat, 11 Jul 2026 09:44:49 +0200 Subject: [PATCH] ci: authenticate Docker Hub pulls in e2e-package; restore full matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e-package job died on every distro with 'toomanyrequests' before a single test ran: GitHub runners share NAT egress IPs whose anonymous Docker Hub quota is chronically exhausted, and every harness image (debian bases, redpanda) is Hub-backed. Locally the same run passes — a residential IP isn't rate-limited. - podman login docker.io in the e2e-package job, reusing the ci.yaml mirror credentials (vars.DOCKERHUB_USERNAME / secrets.DOCKERHUB_TOKEN); degrades to anonymous with a ::warning:: when unset - redpanda image ref switched from docker.redpanda.com (Hub-fronting vanity registry the docker.io login can't cover) to the equivalent docker.io/redpandadata/redpanda:v24.2.7 — one login covers all pulls - full four-distro matrix restored on main pushes: the gate now also produces consumable current-main packages, so per-distro artifacts need the full matrix; fromJSON conditionals dropped --- .github/workflows/release.yml | 48 +++++++++++++++++++++++++---------- doc/Changelog | 2 ++ tests/e2e/run.sh | 2 +- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d52fb0e..5adabd9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,11 +6,11 @@ name: release # * manual dispatch — same matrix, but artifacts go to the workflow run # only (no Release object created). Useful for testing the pipeline # between proper releases. -# * push to main — regression gate: reduced matrix (one deb + one -# rpm distro), build + validate + packaged e2e, artifacts on the run -# only. Catches packaging bugs at merge time instead of at tag time. -# Path-filtered so doc-only commits skip it (paths are not evaluated -# for tag pushes, so releases are unaffected by the filter). +# * push to main — regression gate: full matrix, build + validate + +# packaged e2e, artifacts on the run only. Catches packaging bugs at +# merge time instead of at tag time, and yields current-main packages +# for every distro. Path-filtered so doc-only commits skip it (paths +# are not evaluated for tag pushes, so releases are unaffected). on: push: branches: @@ -50,11 +50,12 @@ jobs: image: ${{ matrix.distro.image }} strategy: fail-fast: false - # Full matrix on tag push / manual dispatch; one deb + one rpm distro - # on main pushes (the regression gate is distro-independent, SOVER - # drift across the full matrix is release-cut territory). matrix: - distro: ${{ (github.ref_type == 'tag' || github.event_name == 'workflow_dispatch') && fromJSON('[{"name":"debian-12 (bookworm)","image":"debian:12"},{"name":"debian-13 (trixie)","image":"debian:13"},{"name":"ubuntu-24.04 (noble)","image":"ubuntu:24.04"},{"name":"fedora","image":"fedora:latest"}]') || fromJSON('[{"name":"debian-12 (bookworm)","image":"debian:12"},{"name":"fedora","image":"fedora:latest"}]') }} + distro: + - { name: "debian-12 (bookworm)", image: "debian:12" } + - { name: "debian-13 (trixie)", image: "debian:13" } + - { name: "ubuntu-24.04 (noble)", image: "ubuntu:24.04" } + - { name: "fedora", image: "fedora:latest" } steps: # actions/checkout needs `git` inside the container; install it @@ -138,9 +139,12 @@ jobs: image: ${{ matrix.distro.image }} strategy: fail-fast: false - # Mirrors the packages-job matrix reduction on main pushes. matrix: - distro: ${{ (github.ref_type == 'tag' || github.event_name == 'workflow_dispatch') && fromJSON('[{"name":"debian-12 (bookworm)","image":"debian:12","tag":"bookworm"},{"name":"debian-13 (trixie)","image":"debian:13","tag":"trixie"},{"name":"ubuntu-24.04 (noble)","image":"ubuntu:24.04","tag":"noble"},{"name":"fedora","image":"fedora:latest","tag":"fc"}]') || fromJSON('[{"name":"debian-12 (bookworm)","image":"debian:12","tag":"bookworm"},{"name":"fedora","image":"fedora:latest","tag":"fc"}]') }} + distro: + - { name: "debian-12 (bookworm)", image: "debian:12", tag: "bookworm" } + - { name: "debian-13 (trixie)", image: "debian:13", tag: "trixie" } + - { name: "ubuntu-24.04 (noble)", image: "ubuntu:24.04", tag: "noble" } + - { name: "fedora", image: "fedora:latest", tag: "fc" } steps: - name: bootstrap (rpm distros) @@ -330,12 +334,30 @@ jobs: runs-on: ubuntu-latest # host runner: needs podman (pre-installed on ubuntu-latest) strategy: fail-fast: false - # Mirrors the packages-job matrix reduction on main pushes. matrix: - distro: ${{ (github.ref_type == 'tag' || github.event_name == 'workflow_dispatch') && fromJSON('[{"name":"debian-12 (bookworm)","tag":"bookworm"},{"name":"debian-13 (trixie)","tag":"trixie"},{"name":"ubuntu-24.04 (noble)","tag":"noble"},{"name":"fedora","tag":"fc"}]') || fromJSON('[{"name":"debian-12 (bookworm)","tag":"bookworm"},{"name":"fedora","tag":"fc"}]') }} + distro: + - { name: "debian-12 (bookworm)", tag: "bookworm" } + - { name: "debian-13 (trixie)", tag: "trixie" } + - { name: "ubuntu-24.04 (noble)", tag: "noble" } + - { name: "fedora", tag: "fc" } steps: - uses: actions/checkout@v5 + # GitHub runners share NAT egress IPs whose anonymous Docker Hub pull + # quota is chronically exhausted — every e2e image (debian bases + + # redpanda) comes from docker.io, so pull authenticated when the + # ci.yaml mirror credentials are present; degrade to anonymous if not. + - name: podman login docker.io (avoid anonymous pull rate limit) + env: + DH_USER: ${{ vars.DOCKERHUB_USERNAME }} + DH_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + if [ -z "${DH_USER}" ] || [ -z "${DH_TOKEN}" ]; then + echo "::warning::vars.DOCKERHUB_USERNAME or secrets.DOCKERHUB_TOKEN unset — anonymous pulls (rate-limit risk)" + exit 0 + fi + echo "${DH_TOKEN}" | podman login docker.io -u "${DH_USER}" --password-stdin + - uses: actions/download-artifact@v7 with: name: packages-${{ matrix.distro.tag }} diff --git a/doc/Changelog b/doc/Changelog index c2ee260..e460566 100644 --- a/doc/Changelog +++ b/doc/Changelog @@ -8,6 +8,8 @@ The keys used are: current (main branch) -- 11-07-2026 + ! Fixing the e2e-package CI job failing on every distro with "toomanyrequests" before a single test ran: GitHub-hosted runners share NAT egress IPs whose anonymous Docker Hub pull quota is chronically exhausted, and every image the harness pulls (debian bases, redpanda) is Docker Hub-backed — locally the same run passes because a residential IP isn't rate-limited. The e2e-package job now does `podman login docker.io` with the same vars.DOCKERHUB_USERNAME / secrets.DOCKERHUB_TOKEN pair the ci.yaml mirror already uses (authenticated accounts get a per-account quota instead of the shared-IP one), degrading to anonymous with a `::warning::` when the credentials are unset. The redpanda image reference in tests/e2e/run.sh switches from docker.redpanda.com (a Docker Hub-fronting vanity registry, which the docker.io login cannot cover) to the equivalent docker.io/redpandadata/redpanda:v24.2.7 so one login covers every pull. + ! Restoring the full four-distro matrix (bookworm/trixie/noble/fedora) on main pushes in release.yml — dropping the bookworm+fedora reduction added with the per-push gate. Rationale for the reversal: the gate's role grew from pure regression signal to also producing consumable current-main packages per distro, and per-distro artifacts are only as useful as the matrix that builds them. Tag/dispatch behavior is unchanged (it was always the full matrix); the fromJSON conditionals simply go away. ! Fixing the `--package` e2e variant, broken since the 28-04-2026 docker/Dockerfile modernization: that commit added tests/e2e/.pkg-stage/ to .dockerignore ("keep the context lean"), one day after the variant was introduced — but Containerfile.pkg COPYs the staged .deb/.rpm from exactly that directory, so every subsequent `tests/e2e/run.sh --package` (and the release.yml e2e-package job) failed at the image build with "no items matching glob". Undetected until now because nothing ran the packaged e2e between the v1.2.0 release cut and the new per-push gate — whose first full run caught it on all four distros (packages + validate green everywhere, e2e-package red everywhere). Removed the exclusion (the dir is transient and .gitignored; its context cost during a package run is one package file) and added .dockerignore to the release.yml paths filter, since it demonstrably shapes what the packaged e2e can build. ! Fixing the `derive version` step in release.yml for non-tag runs: the deb container images (debian:12/13, ubuntu:24.04) default run-steps to dash, which rejects the bash-only `${GITHUB_SHA::7}` substring with "Bad substitution" — the first-ever workflow_dispatch run failed on all three deb legs while fedora (bash default) passed. The bug was latent since the job was introduced: tag pushes take the POSIX-safe `${GITHUB_REF_NAME#v}` branch, so the else-branch had never executed before. Pinned `shell: bash` on the step — the same fix the validate sub-steps already carry for the same dash trap. diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 4339bc8..3287b81 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -163,7 +163,7 @@ echo echo "=== broker (redpanda) ===" podman run -d --rm --name mdt-e2e-broker \ --network "${NETWORK}" --network-alias broker \ - docker.redpanda.com/redpandadata/redpanda:v24.2.7 \ + docker.io/redpandadata/redpanda:v24.2.7 \ redpanda start --overprovisioned --smp 1 --memory 512M \ --reserve-memory 0M --node-id 0 --check=false \ --kafka-addr PLAINTEXT://0.0.0.0:9092 \