🌱 ci: use ginkgo --flake-attempts instead of nick-invision/retry for e2e tests

.github/workflows/gitlab.yml

@@ -66,24 +66,16 @@ jobs:
             go mod download
 
       - name: Run GitLab tokenless E2E
-        uses: nick-invision/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0
         if: github.event_name == 'pull_request'
-        with:
-          max_attempts: 3
-          retry_on: error
-          timeout_minutes: 30
-          command: make e2e-gitlab
+        timeout-minutes: 30
+        run: make e2e-gitlab
 
       - name: Run GitLab PAT E2E  # skip if auth token is not available
-        uses: nick-invision/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0
         if: ${{ github.event_name == 'push' && github.actor != 'dependabot[bot]' }}
+        timeout-minutes: 30
         env:
           GITLAB_AUTH_TOKEN: ${{ secrets.GITLAB_TOKEN }}
-        with:
-          max_attempts: 3
-          retry_on: error
-          timeout_minutes: 30
-          command: make e2e-gitlab-token
+        run: make e2e-gitlab-token
 
       - name: codecov
         uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # 5.5.3

.github/workflows/integration.yml

@@ -63,15 +63,11 @@ jobs:
         run: |
             go mod download
 
-      - name: Run GITHUB_TOKEN E2E  #using retry because the GitHub token is being throttled.
-        uses: nick-invision/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0
+      - name: Run GITHUB_TOKEN E2E
+        timeout-minutes: 30
         env:
           GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          max_attempts: 3
-          retry_on: error
-          timeout_minutes: 30
-          command: make e2e-gh-token
+        run: make e2e-gh-token
 
       - name: codecov
         uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # 5.5.3

.github/workflows/main.yml

@@ -72,16 +72,12 @@ jobs:
        with:
          files: ./unit-coverage.out
          verbose: true
-     - name: Run PAT Token E2E  #using retry because the GitHub token is being throttled.
-       uses: nick-invision/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0
+     - name: Run PAT Token E2E
        if: ${{ github.event_name != 'pull_request' && github.actor != 'dependabot[bot]' }}
+       timeout-minutes: 30
        env:
           GITHUB_AUTH_TOKEN: ${{ secrets.GH_AUTH_TOKEN }}
-       with:
-          max_attempts: 3
-          retry_on: error
-          timeout_minutes: 30
-          command: make e2e-pat
+       run: make e2e-pat
      - name: codecov
        uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # 2.1.0
        if: ${{ github.event_name != 'pull_request' || github.actor != 'dependabot[bot]' }}

Makefile

@@ -346,24 +346,24 @@ endif
 e2e-pat: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to GitHub personal access token
 e2e-pat: build-scorecard check-env | $(GINKGO)
 	# Run e2e tests. GITHUB_AUTH_TOKEN with personal access token must be exported to run this
-	TOKEN_TYPE="PAT" $(GINKGO) --race -p -v  -coverprofile=e2e-coverage.out -coverpkg=./... -r ./... 
+	TOKEN_TYPE="PAT" $(GINKGO) --race -p -v --flake-attempts=3 -coverprofile=e2e-coverage.out -coverpkg=./... -r ./...
 
 e2e-gh-token: ## Runs e2e tests. Requires GITHUB_AUTH_TOKEN env var to be set to default GITHUB_TOKEN
 e2e-gh-token: build-scorecard check-env | $(GINKGO)
 	# Run e2e tests. GITHUB_AUTH_TOKEN set to secrets.GITHUB_TOKEN must be used to run this.
-	GITLAB_AUTH_TOKEN="" TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./...
+	GITLAB_AUTH_TOKEN="" TOKEN_TYPE="GITHUB_TOKEN" $(GINKGO) --race -p -v --flake-attempts=3 -coverprofile=e2e-coverage.out --keep-separate-coverprofiles ./...
 
 e2e-gitlab-token: ## Runs e2e tests that require a GITLAB_TOKEN
 e2e-gitlab-token: build-scorecard check-env-gitlab | $(GINKGO)
-	TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv -coverprofile=e2e-coverage.out --keep-separate-coverprofiles --focus '.*GitLab' ./...
+	TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="GITLAB_PAT" $(GINKGO) --race -p -vv --flake-attempts=3 -coverprofile=e2e-coverage.out --keep-separate-coverprofiles --focus '.*GitLab' ./...
 
 e2e-gitlab: ## Runs e2e tests for GitLab only. TOKEN_TYPE is not used (since these are public APIs), but must be set to something
 e2e-gitlab: build-scorecard | $(GINKGO)
-	TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="PAT" $(GINKGO)  --race -p -vv -coverprofile=e2e-coverage.out --keep-separate-coverprofiles --focus ".*GitLab" ./...
+	TEST_GITLAB_EXTERNAL=1 TOKEN_TYPE="PAT" $(GINKGO) --race -p -vv --flake-attempts=3 -coverprofile=e2e-coverage.out --keep-separate-coverprofiles --focus ".*GitLab" ./...
 
 e2e-azure-devops-token: ## Runs e2e tests that require a AZURE_DEVOPS_AUTH_TOKEN
 e2e-azure-devops-token: build-scorecard check-env-azure-devops | $(GINKGO)
-	SCORECARD_EXPERIMENTAL=1 TEST_AZURE_DEVOPS_EXTERNAL=1 TOKEN_TYPE="AZURE_DEVOPS_PAT" $(GINKGO) --race -p -vv -coverprofile=e2e-coverage.out --keep-separate-coverprofiles --focus "Azure DevOps" ./...
+	SCORECARD_EXPERIMENTAL=1 TEST_AZURE_DEVOPS_EXTERNAL=1 TOKEN_TYPE="AZURE_DEVOPS_PAT" $(GINKGO) --race -p -vv --flake-attempts=3 -coverprofile=e2e-coverage.out --keep-separate-coverprofiles --focus "Azure DevOps" ./...
 
 e2e-attestor: ## Runs e2e tests for scorecard-attestor
 	cd attestor/e2e; go test -covermode=atomic -coverprofile=e2e-coverage.out; cd ../..