diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..af7696c --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,17 @@ +# Configuration for actionlint (https://github.com/rhysd/actionlint). +paths: + .github/workflows/pack-release.yaml: + ignore: + # actionlint's type definitions for the `job` context (still true as + # of v1.7.12 / main) only include check_run_id, container, services, + # and status. GitHub added job.workflow_ref / job.workflow_sha / + # job.workflow_repository / job.workflow_file_path afterwards, so + # actionlint doesn't know about them yet and flags real, documented + # properties as undefined. Confirmed against GitHub's own docs + # (https://docs.github.com/en/actions/reference/contexts-reference#job-context), + # which show this exact pattern -- a reusable workflow checking out + # its own repo via job.workflow_repository/job.workflow_sha -- as the + # canonical example usage. Remove these two lines once actionlint + # ships support for the job workflow-identity properties. + - 'property "workflow_repository" is not defined in object type' + - 'property "workflow_sha" is not defined in object type' diff --git a/.github/scripts/pin_image_tags.py b/.github/scripts/pin_image_tags.py new file mode 100644 index 0000000..05df5c7 --- /dev/null +++ b/.github/scripts/pin_image_tags.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Pin container image tags in a Helm values.yaml file to a release tag. + +Used by pack-release.yaml to rewrite the working copy of a chart's +values.yaml so that its image tags point at the sha- tag built +for the release commit, right before packaging. The change is made only +in the workflow's working directory; it is never committed back to the +source repository. + +Usage: + python pin_image_tags.py [ ...] + +Each path is a dot-separated key path into the YAML document, e.g. +"operator.image.tag". The walk sets the final key in the path (the +"tag" leaf) to , leaving every sibling key and comment intact. + +Uses ruamel.yaml's round-trip mode so comments, quoting, and key order +in the original file are preserved. Fails loudly (non-zero exit) if any +segment of a path does not exist in the document -- a missing path is +almost always a typo'd values.yaml key and should stop the release +rather than silently no-op. +""" +import sys + +from ruamel.yaml import YAML + + +def pin(values_file, sha_tag, paths): + """Set the leaf key of each dotted path in values_file to sha_tag. + + Args: + values_file: path to a values.yaml file, read and rewritten in place. + sha_tag: the value to assign to each path's leaf key, e.g. "sha-abc1234". + paths: dotted key paths such as "operator.image.tag". The last + segment is the leaf that gets set; everything before it is + walked as nested mapping keys. + + Raises: + KeyError: if any segment of any path is missing from the document. + """ + yaml = YAML() + yaml.preserve_quotes = True + + with open(values_file) as f: + data = yaml.load(f) + + for dotted in paths: + keys = dotted.split(".") + node = data + for key in keys[:-1]: + if not isinstance(node, dict) or key not in node: + raise KeyError( + f"path '{dotted}' not found in {values_file}: " + f"no key '{key}'" + ) + node = node[key] + + leaf = keys[-1] + if not isinstance(node, dict) or leaf not in node: + raise KeyError( + f"path '{dotted}' not found in {values_file}: " + f"no key '{leaf}'" + ) + node[leaf] = sha_tag + + with open(values_file, "w") as f: + yaml.dump(data, f) + + +def main(argv): + if len(argv) < 4: + print( + "usage: pin_image_tags.py [ ...]", + file=sys.stderr, + ) + return 2 + + values_file, sha_tag, *paths = argv[1:] + try: + pin(values_file, sha_tag, paths) + except Exception as exc: + print(f"pin_image_tags: error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/.github/scripts/test_pin_image_tags.py b/.github/scripts/test_pin_image_tags.py new file mode 100644 index 0000000..b87f7ac --- /dev/null +++ b/.github/scripts/test_pin_image_tags.py @@ -0,0 +1,86 @@ +"""Tests for pin_image_tags.py. + +Run with: python -m pytest .github/scripts/test_pin_image_tags.py -v +""" +import textwrap + +import pytest +from ruamel.yaml import YAML + +from pin_image_tags import pin + +SAMPLE_VALUES = textwrap.dedent( + """\ + operator: + image: + repository: quay.io/nebari/llm-d-operator + tag: latest # bumped by CI on each release + replicas: 1 + + worker: + image: + repository: quay.io/nebari/llm-d-worker + tag: latest + """ +) + + +@pytest.fixture +def values_file(tmp_path): + path = tmp_path / "values.yaml" + path.write_text(SAMPLE_VALUES) + return path + + +def _load(values_file): + yaml = YAML() + with open(values_file) as f: + return yaml.load(f) + + +def test_pin_sets_tag(values_file): + pin(str(values_file), "sha-abc1234", ["operator.image.tag"]) + + data = _load(values_file) + assert data["operator"]["image"]["tag"] == "sha-abc1234" + # untouched path is left alone + assert data["worker"]["image"]["tag"] == "latest" + + +def test_pin_preserves_comment(values_file): + pin(str(values_file), "sha-abc1234", ["operator.image.tag"]) + + text = values_file.read_text() + # ruamel may re-flow the whitespace before an inline comment when the + # value's length changes, but the comment text itself must survive. + assert "tag: sha-abc1234" in text + assert "# bumped by CI on each release" in text + + +def test_pin_multiple_paths(values_file): + pin(str(values_file), "sha-def5678", ["operator.image.tag", "worker.image.tag"]) + + data = _load(values_file) + assert data["operator"]["image"]["tag"] == "sha-def5678" + assert data["worker"]["image"]["tag"] == "sha-def5678" + + +def test_missing_leaf_raises(values_file): + with pytest.raises(Exception): + pin(str(values_file), "sha-abc1234", ["operator.image.missing"]) + + +def test_missing_intermediate_key_raises(values_file): + with pytest.raises(Exception): + pin(str(values_file), "sha-abc1234", ["nonexistent.image.tag"]) + + +def test_original_file_unchanged_on_partial_success_is_not_guaranteed(values_file): + # Documents current behavior: a later path failing does not roll back + # earlier writes made in-memory before the dump. Since dump only + # happens once at the end, a failure means no dump happens at all and + # the file on disk is untouched. + original = values_file.read_text() + with pytest.raises(Exception): + pin(str(values_file), "sha-abc1234", ["operator.image.tag", "nonexistent.path.tag"]) + assert values_file.read_text() == original diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml new file mode 100644 index 0000000..1ca4658 --- /dev/null +++ b/.github/workflows/lint-test.yaml @@ -0,0 +1,58 @@ +name: lint-test + +# CI for this repo's own reusable workflows and helper scripts. +# +# Scoped to the workflows introduced/maintained under this effort +# (pack-build-image.yaml, pack-release.yaml, and this file) rather than +# the whole .github/workflows/ directory: the other pre-existing +# workflows here (sync-issue-templates.yaml, sync-project-priority.yaml) +# already have unrelated actionlint findings (a floating, EOL +# actions/checkout@v3 pin and a few shellcheck info-level notes) that +# predate this change and are out of scope for it. Widen the file list +# below once those are cleaned up separately. + +on: + pull_request: + push: + branches: + - main + +jobs: + actionlint: + name: actionlint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Run actionlint + run: | + set -euo pipefail + docker run --rm -v "$PWD:/repo" -w /repo \ + rhysd/actionlint@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667 \ + -color \ + .github/workflows/pack-build-image.yaml \ + .github/workflows/pack-release.yaml \ + .github/workflows/lint-test.yaml + + pytest: + name: pytest + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + set -euo pipefail + python3 -m pip install "ruamel.yaml==0.19.1" pytest + + - name: Run pytest + run: | + set -euo pipefail + python3 -m pytest .github/scripts/ diff --git a/.github/workflows/pack-build-image.yaml b/.github/workflows/pack-build-image.yaml new file mode 100644 index 0000000..d5f38ce --- /dev/null +++ b/.github/workflows/pack-build-image.yaml @@ -0,0 +1,98 @@ +name: pack-build-image + +on: + workflow_call: + inputs: + image: + description: "Image name suffix, e.g. 'operator' or 'frontend'." + required: true + type: string + context: + description: "Docker build context path." + required: true + type: string + dockerfile: + description: "Path to the Dockerfile. Defaults to '/Dockerfile'." + required: false + type: string + default: "" + target: + description: "Optional multi-stage build target." + required: false + type: string + default: "" + platforms: + description: "Comma-separated platforms to build for." + required: false + type: string + default: "linux/amd64" + push: + description: "Whether to push the built image. Set false for pull_request builds." + required: false + type: boolean + default: true + secrets: + QUAY_TOKEN: + required: false + +env: + GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ inputs.image }} + QUAY_IMAGE: quay.io/nebari/${{ github.event.repository.name }}-${{ inputs.image }} + +jobs: + build: + name: Build and push ${{ inputs.image }} + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Short sha + id: sha + run: echo "short=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + + - name: Log in to GHCR + if: ${{ inputs.push }} + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to Quay + if: ${{ inputs.push }} + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: quay.io + username: ${{ vars.QUAY_USERNAME }} + password: ${{ secrets.QUAY_TOKEN }} + + - name: Metadata (tags) + id: meta + uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 + with: + images: | + ${{ env.GHCR_IMAGE }} + ${{ env.QUAY_IMAGE }} + tags: | + type=raw,value=sha-${{ steps.sha.outputs.short }} + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: ${{ inputs.context }} + file: ${{ inputs.dockerfile || format('{0}/Dockerfile', inputs.context) }} + target: ${{ inputs.target }} + platforms: ${{ inputs.platforms }} + push: ${{ inputs.push }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.GHCR_IMAGE }}:cache + cache-to: ${{ inputs.push && format('type=registry,ref={0}:cache,mode=max', env.GHCR_IMAGE) || '' }} diff --git a/.github/workflows/pack-release.yaml b/.github/workflows/pack-release.yaml new file mode 100644 index 0000000..7f90195 --- /dev/null +++ b/.github/workflows/pack-release.yaml @@ -0,0 +1,227 @@ +name: pack-release + +# Reusable release workflow for Nebari software packs. +# +# Given a chart source directory living in the calling repository, this +# workflow: +# 1. Reads the chart version from /Chart.yaml. +# 2. Skips everything below if a GitHub Release for - +# already exists (idempotent re-runs, e.g. after a workflow retry). +# 3. Pins the given `tag-paths` (dotted values.yaml keys, e.g. +# "operator.image.tag") to sha- of the release commit, in +# the checked-out working copy only. This is never committed back to +# the calling repository -- it only affects the packaged chart and the +# copy synced to nebari-dev/helm-repository in the final step. +# 4. Packages the chart with `helm package` and attaches the resulting +# .tgz to a GitHub Release in the calling repository. +# 5. Syncs the (now pinned) chart source to the central +# nebari-dev/helm-repository via the shared sync-chart action, which +# opens a pull request there. +# +# Images: this workflow pins the tag-paths to the RELEASE COMMIT's short +# sha (sha- of the commit that changed Chart.yaml). The caller +# MUST therefore ensure images for that exact commit get built and +# pushed -- in practice, trigger the caller's image build on Chart.yaml +# (or chart) changes too, not only on source-code changes, so that a +# release commit that only bumps the version still produces the images +# the published chart will reference. Those builds run in parallel with +# this workflow on the same push to `main`. There is no hard +# cross-workflow gate in v1 -- this workflow does not wait for the image +# build to finish before pinning/publishing; the images land before the +# chart is consumable downstream (the OCI publish happens later, after +# the sync-chart PR merges in nebari-dev/helm-repository). If build times +# ever lag badly enough to matter, add a wait step (e.g. polling the +# build workflow run) in a future revision. +# +# Quay.io chart repositories: sync-chart only copies chart source into +# nebari-dev/helm-repository and opens a PR there -- it does not touch +# quay.io itself. The actual `helm package` + ensure-quay-repos + OCI push +# to quay.io/nebari/charts happens automatically inside +# nebari-dev/helm-repository's own release-helm-charts.yml once that PR +# merges, using a QUAY_API_TOKEN secret scoped to that repo alone. This +# workflow deliberately does not add its own ensure-quay-repos step: it +# has no access to that token, and calling it here would be redundant +# with (and could race) the downstream automation that already runs it. + +on: + workflow_call: + inputs: + chart-path: + description: >- + Path to the Helm chart source directory in the calling repository + (must contain a Chart.yaml). + required: true + type: string + chart-name: + description: >- + Chart name used for the GitHub Release tag + (-) and as the destination directory name + in nebari-dev/helm-repository. + required: true + type: string + tag-paths: + description: >- + Newline-separated list of dotted values.yaml paths to pin to the + release's sha- tag, for example: + operator.image.tag + worker.image.tag + required: true + type: string + secrets: + NEBARI_HELM_REPO_TOKEN: + description: >- + Fine-grained PAT with contents + pull-request write on + nebari-dev/helm-repository. Used only by the sync-chart step. + required: true + +jobs: + release: + name: Release ${{ inputs.chart-name }} + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: Read chart metadata + id: chart + env: + CHART_PATH: ${{ inputs.chart-path }} + CHART_NAME: ${{ inputs.chart-name }} + run: | + set -euo pipefail + chart_yaml="${CHART_PATH}/Chart.yaml" + if [ ! -f "$chart_yaml" ]; then + echo "::error::Chart.yaml not found at ${chart_yaml}. Check 'chart-path'." + exit 1 + fi + + version=$(grep -E '^version:' "$chart_yaml" | head -1 | awk '{print $2}' | tr -d "\"'" || true) + if [ -z "$version" ]; then + echo "::error::Could not read 'version' from ${chart_yaml}." + exit 1 + fi + + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "tag=${CHART_NAME}-$version" >> "$GITHUB_OUTPUT" + + - name: Skip if already released + id: exists + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + TAG: ${{ steps.chart.outputs.tag }} + run: | + set -euo pipefail + if gh release view "$TAG" >/dev/null 2>&1; then + echo "::notice::Release $TAG already exists, skipping." + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + # Reusable workflows can check out their OWN repo (as opposed to the + # caller's) via the `job` context: job.workflow_repository and + # job.workflow_sha identify exactly the nebari-dev/.github commit + # that the calling workflow pinned to invoke this file, which is + # documented for exactly this case -- "when a reusable workflow + # needs to access files co-located with the workflow definition." + # (job.workflow_ref/workflow_sha/workflow_repository are not + # available on GHES, but nebari-dev is on github.com, so that + # restriction doesn't apply here.) This lets the pin step below run + # the real, unit-tested .github/scripts/pin_image_tags.py instead + # of an inlined duplicate. + # + # nebari-dev/.github is a public repository, so the default + # GITHUB_TOKEN is sufficient to check it out -- no extra secret is + # needed. Checked out to its own path (_dot-github) so it doesn't + # collide with the caller's checkout at the workspace root, which + # is where `chart-path` continues to resolve from. + - name: Check out the reusable workflow's repo (for its scripts) + if: steps.exists.outputs.exists == 'false' + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: _dot-github + + - name: Set up Python + if: steps.exists.outputs.exists == 'false' + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + + - name: Pin image tags to release sha + if: steps.exists.outputs.exists == 'false' + env: + CHART_PATH: ${{ inputs.chart-path }} + TAG_PATHS: ${{ inputs.tag-paths }} + run: | + set -euo pipefail + python3 -m pip install "ruamel.yaml==0.19.1" + + short_sha="${GITHUB_SHA::7}" + mapfile -t paths < <(printf '%s\n' "$TAG_PATHS" | sed '/^[[:space:]]*$/d') + echo "Pinning ${#paths[@]} path(s) to sha-${short_sha}: ${paths[*]}" + + python3 _dot-github/.github/scripts/pin_image_tags.py \ + "${CHART_PATH}/values.yaml" "sha-${short_sha}" "${paths[@]}" + + - name: Set up Helm + if: steps.exists.outputs.exists == 'false' + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 + + - name: Package chart + if: steps.exists.outputs.exists == 'false' + env: + CHART_PATH: ${{ inputs.chart-path }} + run: | + set -euo pipefail + helm dependency update "${CHART_PATH}" + helm package "${CHART_PATH}" --destination . + + - name: Determine prerelease flag + id: pre + if: steps.exists.outputs.exists == 'false' + env: + VERSION: ${{ steps.chart.outputs.version }} + run: | + set -euo pipefail + case "$VERSION" in + *-*) echo "flag=--prerelease" >> "$GITHUB_OUTPUT" ;; + *) echo "flag=" >> "$GITHUB_OUTPUT" ;; + esac + + - name: Create GitHub Release + if: steps.exists.outputs.exists == 'false' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + TAG: ${{ steps.chart.outputs.tag }} + PRERELEASE_FLAG: ${{ steps.pre.outputs.flag }} + run: | + set -euo pipefail + shopt -s nullglob + tgz_files=(*.tgz) + if [ "${#tgz_files[@]}" -ne 1 ]; then + echo "::error::Expected exactly one .tgz file to release, found ${#tgz_files[@]}: ${tgz_files[*]}" + exit 1 + fi + + release_args=(--title "$TAG" --generate-notes) + if [ -n "$PRERELEASE_FLAG" ]; then + release_args+=("$PRERELEASE_FLAG") + fi + + gh release create "$TAG" "${release_args[@]}" "${tgz_files[0]}" + + - name: Sync chart to nebari-dev/helm-repository + if: steps.exists.outputs.exists == 'false' + uses: nebari-dev/helm-repository/.github/actions/sync-chart@5cbd23a45c014bf2fa34b4683d4e5ac70ad34fa4 # main 2026-07-03 + with: + token: ${{ secrets.NEBARI_HELM_REPO_TOKEN }} + chart-path: ${{ inputs.chart-path }} + chart-name: ${{ inputs.chart-name }}