diff --git a/.github/utils/prepare_release_notification.sh b/.github/utils/prepare_release_notification.sh index e4b562361d2..a9620bf2233 100755 --- a/.github/utils/prepare_release_notification.sh +++ b/.github/utils/prepare_release_notification.sh @@ -4,7 +4,8 @@ # Requires: VERSION, RUN_URL, HAS_FAILURE, GH_TOKEN, GITHUB_REPOSITORY # Optional: IS_RC, IS_FIRST_RC, MAJOR_MINOR, GITHUB_URL, PYPI_URL, DOCKER_URL, # BUMP_VERSION_PR_URL, DC_PIPELINE_TEMPLATES_PR_URL, DC_CUSTOM_NODES_PR_URL, -# HAYSTACK_RUNTIME_PR_URL, GITHUB_WORKSPACE +# HAYSTACK_RUNTIME_PR_URL, DC_PIPELINE_TEMPLATES_RESULT, DC_CUSTOM_NODES_RESULT, +# HAYSTACK_RUNTIME_RESULT, GITHUB_WORKSPACE # Output: slack_payload.json # # This script is used in the release.yml workflow to prepare the notification payload @@ -59,12 +60,20 @@ if [[ "${IS_FIRST_RC:-}" == "true" && -n "${BUMP_VERSION_PR_URL:-}" ]]; then TXT+=$'\n'"- <${BUMP_VERSION_PR_URL}|Bump unstable version and create unstable docs>" fi -# For RCs, include Platform test PRs +# For RCs, include Platform test PRs (or a warning for repos whose bump job failed) if [[ "${IS_RC:-}" == "true" ]]; then PLATFORM_PRS="" - [[ -n "${DC_PIPELINE_TEMPLATES_PR_URL:-}" ]] && PLATFORM_PRS+=$'\n'"- <${DC_PIPELINE_TEMPLATES_PR_URL}|dc-pipeline-templates>" - [[ -n "${DC_CUSTOM_NODES_PR_URL:-}" ]] && PLATFORM_PRS+=$'\n'"- <${DC_CUSTOM_NODES_PR_URL}|deepset-cloud-custom-nodes>" - [[ -n "${HAYSTACK_RUNTIME_PR_URL:-}" ]] && PLATFORM_PRS+=$'\n'"- <${HAYSTACK_RUNTIME_PR_URL}|haystack-runtime>" + platform_line() { + local result="$1" url="$2" label="$3" + if [[ -n "${url}" ]]; then + PLATFORM_PRS+=$'\n'"- <${url}|${label}>" + elif [[ "${result}" != "success" ]]; then + PLATFORM_PRS+=$'\n'"- ${label}: :warning: PR not found, something might have failed, see <${RUN_URL}|workflow logs>" + fi + } + platform_line "${DC_PIPELINE_TEMPLATES_RESULT:-}" "${DC_PIPELINE_TEMPLATES_PR_URL:-}" "dc-pipeline-templates" + platform_line "${DC_CUSTOM_NODES_RESULT:-}" "${DC_CUSTOM_NODES_PR_URL:-}" "deepset-cloud-custom-nodes" + platform_line "${HAYSTACK_RUNTIME_RESULT:-}" "${HAYSTACK_RUNTIME_PR_URL:-}" "haystack-runtime" if [[ -n "${PLATFORM_PRS}" ]]; then TXT+=$'\n\n'":factory: *Test PRs opened on Platform:*${PLATFORM_PRS}" fi diff --git a/.github/utils/update_haystack_dc_custom_nodes.py b/.github/utils/update_haystack_dc_custom_nodes.py deleted file mode 100755 index a372d0ac15f..00000000000 --- a/.github/utils/update_haystack_dc_custom_nodes.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -""" -Update the haystack-ai version in the deepset-cloud-custom-nodes uv.lock file. - -Fetches sdist/wheel hashes from PyPI and updates the haystack-ai package entry, -preserving the existing lock file formatting. -""" - -import argparse -import json -import sys -import urllib.request - -import tomlkit - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("version", help="Version to update to (e.g. 2.26.1-rc1)") - parser.add_argument("lock_file", help="Path to uv.lock") - args = parser.parse_args() - - # PEP 440 normalized version for filenames - new_version = args.version.replace("-", "") - - # Fetch hashes from PyPI - pypi_data = json.load(urllib.request.urlopen(f"https://pypi.org/pypi/haystack-ai/{args.version}/json")) - sdist_sha = wheel_sha = None - for u in pypi_data["urls"]: - if u["packagetype"] == "sdist": - sdist_sha = u["digests"]["sha256"] - elif u["packagetype"] == "bdist_wheel": - wheel_sha = u["digests"]["sha256"] - if not sdist_sha or not wheel_sha: - sys.exit("Could not find sdist or wheel hashes on PyPI") - - with open(args.lock_file) as f: - data = tomlkit.load(f) - - found = False - for pkg in data["package"]: - if pkg["name"] == "haystack-ai": - old_version = pkg["version"] - - pkg["version"] = new_version - - pkg["sdist"]["url"] = pkg["sdist"]["url"].replace(old_version, new_version) - pkg["sdist"]["hash"] = f"sha256:{sdist_sha}" - - wheel = pkg["wheels"][0] - wheel["url"] = wheel["url"].replace(old_version, new_version) - wheel["hash"] = f"sha256:{wheel_sha}" - - found = True - print(f"Updated haystack-ai from {old_version} to {new_version}") - break - - if not found: - sys.exit("haystack-ai package not found in uv.lock") - - with open(args.lock_file, "w") as f: - tomlkit.dump(data, f) diff --git a/.github/utils/wait_for_platform_pr.sh b/.github/utils/wait_for_platform_pr.sh new file mode 100755 index 00000000000..24100bffb88 --- /dev/null +++ b/.github/utils/wait_for_platform_pr.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# wait_for_platform_pr.sh - Wait for the PR opened by a platform repo's Haystack bump workflow +# +# Usage: ./wait_for_platform_pr.sh +# Requires: GH_TOKEN environment variable +# Output: Writes pr_url to $GITHUB_OUTPUT if set, otherwise to stdout +# +# Example: +# ./wait_for_platform_pr.sh deepset-ai/haystack-runtime 2.99.0-rc1 +# +# This script is used in the release.yml workflow after dispatching a platform repo's update +# workflow, which opens a PR from branch "bump/hs". + + +# With the default values, we wait for 5 minutes +MAX_ATTEMPTS="${MAX_ATTEMPTS:-30}" +SLEEP_SECONDS="${SLEEP_SECONDS:-10}" + +set -euo pipefail + +if [[ -z "${GH_TOKEN:-}" ]]; then + echo "❌ GH_TOKEN must be set" + exit 1 +fi + +REPO="$1" +VERSION="$2" +BRANCH="bump/hs${VERSION}" + +echo "⏳ Waiting for a PR from branch ${BRANCH} on ${REPO}" + +for ((i=1; i<=MAX_ATTEMPTS; i++)); do + PR_URL=$(gh pr list -R "${REPO}" --head "${BRANCH}" --state open \ + --json url --jq '.[0].url // empty' 2>/dev/null || true) + if [[ -n "${PR_URL}" ]]; then + echo "✅ Found PR: ${PR_URL}" + OUTPUT_FILE="${GITHUB_OUTPUT:-/dev/stdout}" + echo "pr_url=${PR_URL}" >> "${OUTPUT_FILE}" + exit 0 + fi + echo " Attempt $i/${MAX_ATTEMPTS}: PR not found yet..." + sleep "${SLEEP_SECONDS}" +done + +echo "❌ PR not found. Something might have failed: check https://github.com/${REPO}/actions" +exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 33610ad2e0a..d140d87d5d3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -144,124 +144,60 @@ jobs: echo "docker_url=https://hub.docker.com/r/deepset/haystack/tags?name=base-v${{ env.VERSION }}" } >> "$GITHUB_OUTPUT" - bump-dc-pipeline-templates: + bump-platform-repos: needs: ["parse-validate-version", "check-artifacts"] if: always() && needs.check-artifacts.result == 'success' && needs.parse-validate-version.outputs.is_rc == 'true' runs-on: ubuntu-slim outputs: - pr_url: ${{ steps.create-pr.outputs.pull-request-url }} - env: - VERSION: ${{ needs.parse-validate-version.outputs.version }} - steps: - - name: Checkout dc-pipeline-templates - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: deepset-ai/dc-pipeline-templates - token: ${{ secrets.HAYSTACK_BOT_TOKEN }} - - - name: Update haystack pin - run: sed -i "s/haystack-ai>=.*/haystack-ai>=${VERSION}/" requirements-test.txt - - - name: Create Pull Request - id: create-pr - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 - with: - token: ${{ secrets.HAYSTACK_BOT_TOKEN }} - commit-message: "Bump haystack to ${{ env.VERSION }}" - branch: bump-haystack-${{ env.VERSION }} - base: main - title: "chore: bump haystack to ${{ env.VERSION }}" - add-paths: requirements-test.txt - body: | - Bump haystack pin to `${{ env.VERSION }}` for platform testing. - - bump-deepset-cloud-custom-nodes: - needs: ["parse-validate-version", "check-artifacts"] - if: always() && needs.check-artifacts.result == 'success' && needs.parse-validate-version.outputs.is_rc == 'true' - runs-on: ubuntu-slim - outputs: - pr_url: ${{ steps.create-pr.outputs.pull-request-url }} + dc_pipeline_templates_pr_url: ${{ steps.dc-pipeline-templates.outputs.pr_url }} + dc_pipeline_templates_result: ${{ steps.dc-pipeline-templates.outcome }} + dc_custom_nodes_pr_url: ${{ steps.deepset-cloud-custom-nodes.outputs.pr_url }} + dc_custom_nodes_result: ${{ steps.deepset-cloud-custom-nodes.outcome }} + haystack_runtime_pr_url: ${{ steps.haystack-runtime.outputs.pr_url }} + haystack_runtime_result: ${{ steps.haystack-runtime.outcome }} env: VERSION: ${{ needs.parse-validate-version.outputs.version }} steps: - name: Checkout haystack (for utils) uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - path: haystack - sparse-checkout: .github/utils/update_haystack_dc_custom_nodes.py + sparse-checkout: .github/utils/wait_for_platform_pr.sh sparse-checkout-cone-mode: false - - name: Checkout deepset-cloud-custom-nodes - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: deepset-ai/deepset-cloud-custom-nodes - token: ${{ secrets.HAYSTACK_BOT_TOKEN }} - path: deepset-cloud-custom-nodes - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.13" - - - name: Install tomlkit + - name: Trigger platform update workflows + env: + GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }} run: | - python -m pip install --upgrade pip - pip install tomlkit --uploaded-prior-to=P1D - - - name: Update haystack-ai in uv.lock - run: python haystack/.github/utils/update_haystack_dc_custom_nodes.py "${{ env.VERSION }}" deepset-cloud-custom-nodes/uv.lock + gh workflow run update-haystack-version.yaml -R deepset-ai/dc-pipeline-templates -f haystack_version="$VERSION" + gh workflow run update-haystack-version.yaml -R deepset-ai/deepset-cloud-custom-nodes -f haystack_version="$VERSION" + gh workflow run update-package-version.yaml -R deepset-ai/haystack-runtime -f haystack_version="$VERSION" - - name: Create Pull Request - id: create-pr - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 - with: - token: ${{ secrets.HAYSTACK_BOT_TOKEN }} - path: deepset-cloud-custom-nodes - commit-message: "Bump haystack to ${{ env.VERSION }}" - branch: bump-haystack-${{ env.VERSION }} - base: main - title: "chore: bump haystack to ${{ env.VERSION }}" - add-paths: uv.lock - body: | - Bump haystack pin to `${{ env.VERSION }}` for platform testing. + - name: Wait for dc-pipeline-templates PR + id: dc-pipeline-templates + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }} + run: .github/utils/wait_for_platform_pr.sh deepset-ai/dc-pipeline-templates "$VERSION" - bump-haystack-runtime: - needs: ["parse-validate-version", "check-artifacts"] - if: always() && needs.check-artifacts.result == 'success' && needs.parse-validate-version.outputs.is_rc == 'true' - runs-on: ubuntu-slim - outputs: - pr_url: ${{ steps.wait-pr.outputs.pr_url }} - env: - VERSION: ${{ needs.parse-validate-version.outputs.version }} - steps: - - name: Trigger "Update package version" workflow + - name: Wait for deepset-cloud-custom-nodes PR + id: deepset-cloud-custom-nodes + continue-on-error: true env: GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }} - run: | - gh workflow run update-package-version.yaml \ - -R deepset-ai/haystack-runtime \ - -f haystack_version="${{ env.VERSION }}" + run: .github/utils/wait_for_platform_pr.sh deepset-ai/deepset-cloud-custom-nodes "$VERSION" - - name: Wait for PR - id: wait-pr + - name: Wait for haystack-runtime PR + id: haystack-runtime + continue-on-error: true env: GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }} + run: .github/utils/wait_for_platform_pr.sh deepset-ai/haystack-runtime "$VERSION" + + - name: Fail if any platform PR is missing + if: steps.dc-pipeline-templates.outcome == 'failure' || steps.deepset-cloud-custom-nodes.outcome == 'failure' || steps.haystack-runtime.outcome == 'failure' run: | - TRIGGER_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) - for i in $(seq 1 30); do - PR_URL=$(gh pr list -R deepset-ai/haystack-runtime \ - --head "bump/hs${{ env.VERSION }}" \ - --json url,createdAt \ - --jq ".[] | select(.createdAt >= \"$TRIGGER_TIME\") | .url") - if [[ -n "$PR_URL" ]]; then - echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT" - echo "Found PR: $PR_URL" - exit 0 - fi - echo "Attempt $i: PR not found yet, waiting 10s..." - sleep 10 - done - echo "PR not found after 5 minutes" + echo "::error::One or more platform PRs were not found. Something might have failed; check the logs above." + exit 1 notify: needs: @@ -269,9 +205,7 @@ jobs: - "branch-off" - "create-release-tag" - "check-artifacts" - - "bump-dc-pipeline-templates" - - "bump-deepset-cloud-custom-nodes" - - "bump-haystack-runtime" + - "bump-platform-repos" if: always() runs-on: ubuntu-slim steps: @@ -283,7 +217,8 @@ jobs: VERSION: ${{ inputs.version }} GH_TOKEN: ${{ github.token }} RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - HAS_FAILURE: ${{ contains(needs.*.result, 'failure') }} + # HAS_FAILURE = the release failed. Failing PRs on Platform repos are reported on Slack but do not mean the release failed. + HAS_FAILURE: ${{ needs.parse-validate-version.result == 'failure' || needs.branch-off.result == 'failure' || needs.create-release-tag.result == 'failure' || needs.check-artifacts.result == 'failure' }} IS_RC: ${{ needs.parse-validate-version.outputs.is_rc }} IS_FIRST_RC: ${{ needs.parse-validate-version.outputs.is_first_rc }} MAJOR_MINOR: ${{ needs.parse-validate-version.outputs.major_minor }} @@ -291,9 +226,12 @@ jobs: PYPI_URL: ${{ needs.check-artifacts.outputs.pypi_url }} DOCKER_URL: ${{ needs.check-artifacts.outputs.docker_url }} BUMP_VERSION_PR_URL: ${{ needs.branch-off.outputs.bump_version_pr_url }} - DC_PIPELINE_TEMPLATES_PR_URL: ${{ needs.bump-dc-pipeline-templates.outputs.pr_url }} - DC_CUSTOM_NODES_PR_URL: ${{ needs.bump-deepset-cloud-custom-nodes.outputs.pr_url }} - HAYSTACK_RUNTIME_PR_URL: ${{ needs.bump-haystack-runtime.outputs.pr_url }} + DC_PIPELINE_TEMPLATES_PR_URL: ${{ needs.bump-platform-repos.outputs.dc_pipeline_templates_pr_url }} + DC_CUSTOM_NODES_PR_URL: ${{ needs.bump-platform-repos.outputs.dc_custom_nodes_pr_url }} + HAYSTACK_RUNTIME_PR_URL: ${{ needs.bump-platform-repos.outputs.haystack_runtime_pr_url }} + DC_PIPELINE_TEMPLATES_RESULT: ${{ needs.bump-platform-repos.outputs.dc_pipeline_templates_result }} + DC_CUSTOM_NODES_RESULT: ${{ needs.bump-platform-repos.outputs.dc_custom_nodes_result }} + HAYSTACK_RUNTIME_RESULT: ${{ needs.bump-platform-repos.outputs.haystack_runtime_result }} run: .github/utils/prepare_release_notification.sh - name: Send release notification to Slack