From a2b58022ea49fbd6ec922d18d289771f0f958ebe Mon Sep 17 00:00:00 2001 From: Ali Date: Mon, 13 Apr 2026 06:59:24 +0500 Subject: [PATCH 1/2] ci: use ginkgo --flake-attempts instead of nick-invision/retry for e2e tests Replace the nick-invision/retry GitHub Action wrapper around e2e test targets with ginkgo's built-in --flake-attempts=3 flag. This retries only the individual specs that fail rather than re-running the entire suite, saving GitHub API token quota and CI time. Changes: - Makefile: add --flake-attempts=3 to all ginkgo e2e targets (e2e-pat, e2e-gh-token, e2e-gitlab-token, e2e-gitlab, e2e-azure-devops-token) - main.yml: remove retry wrapper from PAT e2e step - integration.yml: remove retry wrapper from GITHUB_TOKEN e2e step - gitlab.yml: remove retry wrapper from both GitLab e2e steps Non-e2e retry wrappers (generate-mocks, generate-docs, publishimage) are intentionally left unchanged as they don't use ginkgo. Fixes #2897 Signed-off-by: Ali --- .github/workflows/gitlab.yml | 14 ++------------ .github/workflows/integration.yml | 9 ++------- .github/workflows/main.yml | 9 ++------- Makefile | 10 +++++----- 4 files changed, 11 insertions(+), 31 deletions(-) diff --git a/.github/workflows/gitlab.yml b/.github/workflows/gitlab.yml index 6fb319ad613..5d7e8801ee1 100644 --- a/.github/workflows/gitlab.yml +++ b/.github/workflows/gitlab.yml @@ -66,24 +66,14 @@ jobs: go mod download - name: Run GitLab tokenless E2E - uses: nick-invision/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 if: github.event_name == 'pull_request' - with: - max_attempts: 3 - retry_on: error - timeout_minutes: 30 - command: make e2e-gitlab + run: make e2e-gitlab - name: Run GitLab PAT E2E # skip if auth token is not available - uses: nick-invision/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 if: ${{ github.event_name == 'push' && github.actor != 'dependabot[bot]' }} env: GITLAB_AUTH_TOKEN: ${{ secrets.GITLAB_TOKEN }} - with: - max_attempts: 3 - retry_on: error - timeout_minutes: 30 - command: make e2e-gitlab-token + run: make e2e-gitlab-token - name: codecov uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # 5.5.3 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4d6da308d30..d128ddfefe1 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -63,15 +63,10 @@ jobs: run: | go mod download - - name: Run GITHUB_TOKEN E2E #using retry because the GitHub token is being throttled. - uses: nick-invision/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 + - name: Run GITHUB_TOKEN E2E env: GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - max_attempts: 3 - retry_on: error - timeout_minutes: 30 - command: make e2e-gh-token + run: make e2e-gh-token - name: codecov uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # 5.5.3 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4b540bcfe23..89f892ca8bc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -72,16 +72,11 @@ jobs: with: files: ./unit-coverage.out verbose: true - - name: Run PAT Token E2E #using retry because the GitHub token is being throttled. - uses: nick-invision/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 + - name: Run PAT Token E2E if: ${{ github.event_name != 'pull_request' && github.actor != 'dependabot[bot]' }} env: GITHUB_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }} - with: - max_attempts: 3 - retry_on: error - timeout_minutes: 30 - command: make e2e-pat + run: make e2e-pat - name: codecov uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # 2.1.0 if: ${{ github.event_name != 'pull_request' || github.actor != 'dependabot[bot]' }} diff --git a/Makefile b/Makefile index 5aafa9c8ee9..5571def55e1 100644 --- a/Makefile +++ b/Makefile @@ -346,24 +346,24 @@ endif e2e-pat: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token e2e-pat: build-scorecard check-env | $(GINKGO) # Run e2e tests. GITHUB_AUTH_TOKEN with personal access token must be exported to run this - TOKEN_TYPE="PAT" $(GINKGO) --race -p -v -coverprofile=e2e-coverage.out -coverpkg=./... -r ./... + TOKEN_TYPE="PAT" $(GINKGO) --race -p -v --flake-attempts=3 -coverprofile=e2e-coverage.out -coverpkg=./... -r ./... e2e-gh-token: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to default GITHUB_TOKEN e2e-gh-token: build-scorecard check-env | $(GINKGO) # Run e2e tests. GITHUB_AUTH_TOKEN set to secrets.GITHUB_TOKEN must be used to run this. - GITLAB_AUTH_TOKEN="" TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./... + GITLAB_AUTH_TOKEN="" TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v --flake-attempts=3 -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./... e2e-gitlab-token: ## Runs e2e tests that require a GITLAB_TOKEN e2e-gitlab-token: build-scorecard check-env-gitlab | $(GINKGO) - TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv -coverprofile=e2e-coverage.out --keep-separate-coverprofiles --focus '.*GitLab' ./... + TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --flake-attempts=3 -coverprofile=e2e-coverage.out --keep-separate-coverprofiles --focus '.*GitLab' ./... e2e-gitlab: ## Runs e2e tests for GitLab only. TOKEN_TYPE is not used (since these are public APIs), but must be set to something e2e-gitlab: build-scorecard | $(GINKGO) - TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="PAT" $(GINKGO) --race -p -vv -coverprofile=e2e-coverage.out --keep-separate-coverprofiles --focus ".*GitLab" ./... + TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="PAT" $(GINKGO) --race -p -vv --flake-attempts=3 -coverprofile=e2e-coverage.out --keep-separate-coverprofiles --focus ".*GitLab" ./... e2e-azure-devops-token: ## Runs e2e tests that require a AZURE_DEVOPS_AUTH_TOKEN e2e-azure-devops-token: build-scorecard check-env-azure-devops | $(GINKGO) - SCORECARD_EXPERIMENTAL=1 TEST_AZURE_DEVOPS_EXTERNAL=1 TOKEN_TYPE="AZURE_DEVOPS_PAT" $(GINKGO) --race -p -vv -coverprofile=e2e-coverage.out --keep-separate-coverprofiles --focus "Azure DevOps" ./... + SCORECARD_EXPERIMENTAL=1 TEST_AZURE_DEVOPS_EXTERNAL=1 TOKEN_TYPE="AZURE_DEVOPS_PAT" $(GINKGO) --race -p -vv --flake-attempts=3 -coverprofile=e2e-coverage.out --keep-separate-coverprofiles --focus "Azure DevOps" ./... e2e-attestor: ## Runs e2e tests for scorecard-attestor cd attestor/e2e; go test -covermode=atomic -coverprofile=e2e-coverage.out; cd ../.. From c6275eb1813e5ed988c730800ec00db81caaa540 Mon Sep 17 00:00:00 2001 From: Ali Date: Fri, 24 Apr 2026 03:54:49 +0500 Subject: [PATCH 2/2] ci: preserve 30-minute e2e timeout with timeout-minutes Signed-off-by: Ali --- .github/workflows/gitlab.yml | 2 ++ .github/workflows/integration.yml | 1 + .github/workflows/main.yml | 1 + 3 files changed, 4 insertions(+) diff --git a/.github/workflows/gitlab.yml b/.github/workflows/gitlab.yml index 5d7e8801ee1..63187e2d9df 100644 --- a/.github/workflows/gitlab.yml +++ b/.github/workflows/gitlab.yml @@ -67,10 +67,12 @@ jobs: - name: Run GitLab tokenless E2E if: github.event_name == 'pull_request' + timeout-minutes: 30 run: make e2e-gitlab - name: Run GitLab PAT E2E # skip if auth token is not available if: ${{ github.event_name == 'push' && github.actor != 'dependabot[bot]' }} + timeout-minutes: 30 env: GITLAB_AUTH_TOKEN: ${{ secrets.GITLAB_TOKEN }} run: make e2e-gitlab-token diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d128ddfefe1..df392047aa1 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -64,6 +64,7 @@ jobs: go mod download - name: Run GITHUB_TOKEN E2E + timeout-minutes: 30 env: GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: make e2e-gh-token diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 89f892ca8bc..a7bcaf414e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,6 +74,7 @@ jobs: verbose: true - name: Run PAT Token E2E if: ${{ github.event_name != 'pull_request' && github.actor != 'dependabot[bot]' }} + timeout-minutes: 30 env: GITHUB_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }} run: make e2e-pat