From 81a3793e420484fa7bd5e4fab46a4f2e6b70aba7 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Wed, 15 Jul 2026 14:58:26 +0200 Subject: [PATCH 1/7] operator: make model-downloader init image configurable via LLM_MODEL_DOWNLOADER_IMAGE --- .../internal/controller/reconcilers/storage.go | 16 ++++++++++++++-- .../controller/reconcilers/storage_test.go | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/operator/internal/controller/reconcilers/storage.go b/operator/internal/controller/reconcilers/storage.go index 5ac5a95..541605e 100644 --- a/operator/internal/controller/reconcilers/storage.go +++ b/operator/internal/controller/reconcilers/storage.go @@ -2,6 +2,7 @@ package reconcilers import ( "fmt" + "os" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -13,9 +14,20 @@ import ( const ( modelStorageVolumeName = "model-storage" modelCachePath = "/model-cache" - hfInitContainerImage = "ghcr.io/nebari-dev/nebari-llm-serving-pack/model-downloader:latest" + + defaultModelDownloaderImage = "ghcr.io/nebari-dev/llm-serving-pack/model-downloader:latest" ) +// modelDownloaderImage returns the image to use for the model-downloader init +// container. It can be overridden via the LLM_MODEL_DOWNLOADER_IMAGE env var; +// otherwise it falls back to defaultModelDownloaderImage. +func modelDownloaderImage() string { + if v := os.Getenv("LLM_MODEL_DOWNLOADER_IMAGE"); v != "" { + return v + } + return defaultModelDownloaderImage +} + // StorageResult holds the storage-related Kubernetes resources for an LLMModel. type StorageResult struct { // PVC is the PersistentVolumeClaim to create, or nil if not needed. @@ -144,7 +156,7 @@ func buildHFInitContainer(model *llmv1alpha1.LLMModel) corev1.Container { container := corev1.Container{ Name: "model-downloader", - Image: hfInitContainerImage, + Image: modelDownloaderImage(), Args: args, VolumeMounts: []corev1.VolumeMount{ { diff --git a/operator/internal/controller/reconcilers/storage_test.go b/operator/internal/controller/reconcilers/storage_test.go index 97cfffc..05c8f73 100644 --- a/operator/internal/controller/reconcilers/storage_test.go +++ b/operator/internal/controller/reconcilers/storage_test.go @@ -31,6 +31,22 @@ func makeHFModel(storage llmv1alpha1.StorageSpec) *llmv1alpha1.LLMModel { } } +func TestModelDownloaderImage(t *testing.T) { + const def = "ghcr.io/nebari-dev/llm-serving-pack/model-downloader:latest" + tests := []struct{ name, env, want string }{ + {"default when unset", "", def}, + {"override from env", "quay.io/nebari/llm-serving-pack-model-downloader:sha-abc1234", "quay.io/nebari/llm-serving-pack-model-downloader:sha-abc1234"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("LLM_MODEL_DOWNLOADER_IMAGE", tt.env) + if got := modelDownloaderImage(); got != tt.want { + t.Fatalf("got %q want %q", got, tt.want) + } + }) + } +} + func TestBuildStorageSpec(t *testing.T) { //nolint:gocyclo // table-driven test t.Parallel() From f7bbb5ba38c39720268e82d3e48057126902615d Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:04:02 +0200 Subject: [PATCH 2/7] chart: decouple image tags from appVersion; add configurable model-downloader image --- .../templates/frontend-deployment.yaml | 2 +- .../templates/key-manager-deployment.yaml | 2 +- .../templates/operator-deployment.yaml | 4 ++- charts/nebari-llm-serving/values.yaml | 32 +++++++++++-------- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/charts/nebari-llm-serving/templates/frontend-deployment.yaml b/charts/nebari-llm-serving/templates/frontend-deployment.yaml index 8697228..26f1667 100644 --- a/charts/nebari-llm-serving/templates/frontend-deployment.yaml +++ b/charts/nebari-llm-serving/templates/frontend-deployment.yaml @@ -34,7 +34,7 @@ spec: # ClusterIP. This is the service the NebariApp targets; auth is handled # client-side by keycloak-js (PKCE/S256) and validated by the key-manager. - name: frontend - image: {{ .Values.frontend.image.repository }}:{{ .Values.frontend.image.tag | default .Chart.AppVersion }} + image: {{ .Values.frontend.image.repository }}:{{ .Values.frontend.image.tag }} imagePullPolicy: {{ .Values.frontend.image.pullPolicy | default "IfNotPresent" }} ports: - name: http diff --git a/charts/nebari-llm-serving/templates/key-manager-deployment.yaml b/charts/nebari-llm-serving/templates/key-manager-deployment.yaml index 6f994dc..547cabd 100644 --- a/charts/nebari-llm-serving/templates/key-manager-deployment.yaml +++ b/charts/nebari-llm-serving/templates/key-manager-deployment.yaml @@ -20,7 +20,7 @@ spec: serviceAccountName: {{ include "nebari-llm-serving.fullname" . }}-key-manager containers: - name: key-manager - image: {{ .Values.keyManager.image.repository }}:{{ .Values.keyManager.image.tag | default .Chart.AppVersion }} + image: {{ .Values.keyManager.image.repository }}:{{ .Values.keyManager.image.tag }} imagePullPolicy: {{ .Values.keyManager.image.pullPolicy | default "IfNotPresent" }} ports: - containerPort: 8080 diff --git a/charts/nebari-llm-serving/templates/operator-deployment.yaml b/charts/nebari-llm-serving/templates/operator-deployment.yaml index 6d7182a..4a4bae5 100644 --- a/charts/nebari-llm-serving/templates/operator-deployment.yaml +++ b/charts/nebari-llm-serving/templates/operator-deployment.yaml @@ -19,7 +19,7 @@ spec: serviceAccountName: {{ include "nebari-llm-serving.fullname" . }}-operator containers: - name: manager - image: {{ .Values.operator.image.repository }}:{{ .Values.operator.image.tag | default .Chart.AppVersion }} + image: {{ .Values.operator.image.repository }}:{{ .Values.operator.image.tag }} imagePullPolicy: {{ .Values.operator.image.pullPolicy | default "IfNotPresent" }} env: - name: LLM_BASE_DOMAIN @@ -48,6 +48,8 @@ spec: value: {{ .Values.defaults.serving.image | quote }} - name: LLM_DEFAULT_EPP_IMAGE value: {{ .Values.defaults.epp.image | quote }} + - name: LLM_MODEL_DOWNLOADER_IMAGE + value: "{{ .Values.modelDownloader.image.repository }}:{{ .Values.modelDownloader.image.tag }}" - name: LLM_DEFAULT_STORAGE_CLASS_NAME value: {{ .Values.defaults.storage.storageClassName | quote }} - name: LLM_CLUSTER_ISSUER_NAME diff --git a/charts/nebari-llm-serving/values.yaml b/charts/nebari-llm-serving/values.yaml index 03eceba..1816ba3 100644 --- a/charts/nebari-llm-serving/values.yaml +++ b/charts/nebari-llm-serving/values.yaml @@ -72,11 +72,9 @@ keyManager: groups: - llm image: - repository: ghcr.io/nebari-dev/nebari-llm-serving-pack/key-manager - # tag defaults to .Chart.AppVersion when empty so the chart version and - # the image version always move together. Override only when testing a - # specific image build (e.g. tag: 0.1.0-alpha.2 or tag: sha-abc1234). - tag: "" + repository: ghcr.io/nebari-dev/llm-serving-pack/key-manager + # Floating on main; pinned to a build sha at release time (see cicd-and-releasing). + tag: "latest" pullPolicy: Always auditInterval: 5m oidcUserinfoURL: "" @@ -176,10 +174,9 @@ frontend: # Set enabled=false to skip the frontend (e.g. an API-only test install). enabled: true image: - repository: ghcr.io/nebari-dev/nebari-llm-serving-pack/frontend - # tag defaults to .Chart.AppVersion when empty so the chart and image - # versions move together. Override to pull a specific build. - tag: "" + repository: ghcr.io/nebari-dev/llm-serving-pack/frontend + # Floating on main; pinned to a build sha at release time (see cicd-and-releasing). + tag: "latest" pullPolicy: Always # port nginx listens on. Must be > 1024 - the container runs as non-root # (UID 101) with all capabilities dropped and cannot bind privileged ports. @@ -202,13 +199,20 @@ frontend: operator: image: - repository: ghcr.io/nebari-dev/nebari-llm-serving-pack/operator - # tag defaults to .Chart.AppVersion when empty so the chart version and - # the image version always move together. Override only when testing a - # specific image build (e.g. tag: 0.1.0-alpha.2 or tag: sha-abc1234). - tag: "" + repository: ghcr.io/nebari-dev/llm-serving-pack/operator + # Floating on main; pinned to a build sha at release time (see cicd-and-releasing). + tag: "latest" pullPolicy: Always +# modelDownloader - the init/job image the operator uses to fetch model +# weights before a serving Deployment starts. Rendered into the operator's +# LLM_MODEL_DOWNLOADER_IMAGE env var (operator-deployment.yaml) rather than +# hardcoded, so it can be re-pinned without rebuilding the operator image. +modelDownloader: + image: + repository: ghcr.io/nebari-dev/llm-serving-pack/model-downloader + # Floating on main; pinned to a build sha at release time (see cicd-and-releasing). + tag: "latest" defaults: serving: From 9d7df5612265125f3f03389d70446db0cd6a0f6d Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:08:42 +0200 Subject: [PATCH 3/7] ci: consume reusable build/release workflows; add categorized changelog config Replace the per-image build steps in build-images.yaml with thin caller jobs that invoke the shared nebari-dev/.github pack-build-image reusable workflow, add a release.yaml caller for pack-release, and add .github/release.yml with changelog categories keyed on this repo's labels. The @feat/pack-release-workflows ref is a bring-up ref and must be re-pinned to v1 before this PR merges. --- .github/release.yml | 14 ++ .github/workflows/build-images.yaml | 202 ++++++++-------------------- .github/workflows/release.yaml | 22 +++ 3 files changed, 89 insertions(+), 149 deletions(-) create mode 100644 .github/release.yml create mode 100644 .github/workflows/release.yaml diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..d04b5d8 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,14 @@ +changelog: + categories: + - title: Features + labels: ["type: enhancement 💅🏼", "type: exploration 🔮"] + - title: Bug Fixes + labels: ["type: bug 🐛"] + - title: Documentation + labels: ["area: documentation 📖"] + - title: Dependencies + labels: ["area: dependencies 📦"] + - title: Maintenance + labels: ["type: maintenance 🛠", "area: ci 👷🏽‍♀️"] + - title: Other Changes + labels: ["*"] diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 50ad70c..b8560ee 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -1,156 +1,60 @@ -name: Build Images +name: Build Docker Images on: push: branches: [main] - tags: ["v*"] + paths: + - "operator/**" + - "key-manager/**" + - "frontend/**" + - "model-downloader/**" + - ".github/workflows/build-images.yaml" + pull_request: + paths: + - "operator/**" + - "key-manager/**" + - "frontend/**" + - "model-downloader/**" workflow_dispatch: -env: - REGISTRY: ghcr.io - IMAGE_PREFIX: ghcr.io/nebari-dev/nebari-llm-serving-pack - jobs: - build-operator: - name: Build and push operator image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Log in to GHCR - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata - id: meta - uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 - with: - images: ${{ env.IMAGE_PREFIX }}/operator - tags: | - type=sha - type=ref,event=branch - type=semver,pattern=v{{version}} - type=raw,value=latest,enable={{is_default_branch}} - - - name: Build and push operator image - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 - with: - context: operator/ - file: operator/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - build-model-downloader: - name: Build and push model-downloader image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Log in to GHCR - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata - id: meta - uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 - with: - images: ${{ env.IMAGE_PREFIX }}/model-downloader - tags: | - type=sha - type=ref,event=branch - type=semver,pattern=v{{version}} - type=raw,value=latest,enable={{is_default_branch}} - - - name: Build and push model-downloader image - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 - with: - context: model-downloader/ - file: model-downloader/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - build-key-manager: - name: Build and push key-manager image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Log in to GHCR - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata - id: meta - uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 - with: - images: ${{ env.IMAGE_PREFIX }}/key-manager - tags: | - type=sha - type=ref,event=branch - type=semver,pattern=v{{version}} - type=raw,value=latest,enable={{is_default_branch}} - - - name: Build and push key-manager image - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 - with: - context: . - file: key-manager/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - build-frontend: - name: Build and push frontend image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Log in to GHCR - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata - id: meta - uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 - with: - images: ${{ env.IMAGE_PREFIX }}/frontend - tags: | - type=sha - type=ref,event=branch - type=semver,pattern=v{{version}} - type=raw,value=latest,enable={{is_default_branch}} - - - name: Build and push frontend image - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 - with: - context: frontend/ - file: frontend/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + operator: + # TODO: re-pin to the v1 tag before this PR merges (bring-up ref only). + uses: nebari-dev/.github/.github/workflows/pack-build-image.yaml@feat/pack-release-workflows + with: + image: operator + context: operator/ + push: ${{ github.event_name != 'pull_request' }} + secrets: + QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} + + model-downloader: + # TODO: re-pin to the v1 tag before this PR merges (bring-up ref only). + uses: nebari-dev/.github/.github/workflows/pack-build-image.yaml@feat/pack-release-workflows + with: + image: model-downloader + context: model-downloader/ + push: ${{ github.event_name != 'pull_request' }} + secrets: + QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} + + key-manager: + # TODO: re-pin to the v1 tag before this PR merges (bring-up ref only). + uses: nebari-dev/.github/.github/workflows/pack-build-image.yaml@feat/pack-release-workflows + with: + image: key-manager + context: "." + dockerfile: key-manager/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + secrets: + QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} + + frontend: + # TODO: re-pin to the v1 tag before this PR merges (bring-up ref only). + uses: nebari-dev/.github/.github/workflows/pack-build-image.yaml@feat/pack-release-workflows + with: + image: frontend + context: frontend/ + push: ${{ github.event_name != 'pull_request' }} + secrets: + QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..eec25a6 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,22 @@ +name: Release Chart + +on: + push: + branches: [main] + paths: + - "charts/nebari-llm-serving/Chart.yaml" + +jobs: + release: + # TODO: re-pin to the v1 tag before this PR merges (bring-up ref only). + uses: nebari-dev/.github/.github/workflows/pack-release.yaml@feat/pack-release-workflows + with: + chart-path: charts/nebari-llm-serving + chart-name: nebari-llm-serving + tag-paths: | + operator.image.tag + keyManager.image.tag + frontend.image.tag + modelDownloader.image.tag + secrets: + NEBARI_HELM_REPO_TOKEN: ${{ secrets.NEBARI_HELM_REPO_TOKEN }} From e971243becc26e3b0b464157cdaa065a9e7e39f0 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:23:46 +0200 Subject: [PATCH 4/7] docs: rewrite release process for pin-at-release OCI publishing --- docs/src/content/docs/cicd-and-releasing.md | 80 ++++++++++++--------- 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/docs/src/content/docs/cicd-and-releasing.md b/docs/src/content/docs/cicd-and-releasing.md index 45d4468..c0bd980 100644 --- a/docs/src/content/docs/cicd-and-releasing.md +++ b/docs/src/content/docs/cicd-and-releasing.md @@ -1,11 +1,11 @@ --- title: CI/CD and Releasing --- -This page documents every GitHub Actions workflow in the repository, the container images they produce, and the manual release process used to cut new versions. It is aimed at contributors and maintainers. +This page documents every GitHub Actions workflow in the repository, the container images they produce, and the release process used to cut new versions. It is aimed at contributors and maintainers. ## Workflows -The repository has six workflows located in `.github/workflows/`. +This section covers the workflows in `.github/workflows/` relevant to CI and releasing. The image build and chart release workflows are thin callers of reusable workflows shared across Nebari packs, defined in [`nebari-dev/.github`](https://github.com/nebari-dev/.github): `pack-build-image.yaml` and `pack-release.yaml`. This repository's own `build-images.yaml` and `release.yaml` files just supply the pack-specific inputs (image names, build contexts, the chart path, which `values.yaml` keys to pin); the actual build, pin, package, and publish steps live in the shared workflows, so every pack that adopts them gets the same behavior and bug fixes. ### Test (`test.yaml`) @@ -38,32 +38,38 @@ The Helm lint job runs `helm lint charts/nebari-llm-serving/` against the chart ### Build Images (`build-images.yaml`) -**Triggers:** push to `main`, push of any `v*` tag, manual `workflow_dispatch`. +**Triggers:** push to `main` touching `operator/`, `key-manager/`, `frontend/`, `model-downloader/`, or the workflow file itself; pull requests touching those same component paths (builds the image but does not push it); manual `workflow_dispatch`. -Builds and pushes four images to GitHub Container Registry (`ghcr.io`) under the prefix `ghcr.io/nebari-dev/nebari-llm-serving-pack`: +Each of the four images (operator, model-downloader, key-manager, frontend - React SPA served by nginx) has its own job in `build-images.yaml`, and each job is a one-line call into the shared `pack-build-image.yaml` workflow, passing the image name, build context, and Dockerfile path. The shared workflow does the actual work: it builds with Docker Buildx and, on a push, pushes the image to two registries: | Image | Build context | Dockerfile | |-------|--------------|------------| -| `ghcr.io/nebari-dev/nebari-llm-serving-pack/operator` | `operator/` | `operator/Dockerfile` | -| `ghcr.io/nebari-dev/nebari-llm-serving-pack/model-downloader` | `model-downloader/` | `model-downloader/Dockerfile` | -| `ghcr.io/nebari-dev/nebari-llm-serving-pack/key-manager` | `.` (repo root) | `key-manager/Dockerfile` | -| `ghcr.io/nebari-dev/nebari-llm-serving-pack/frontend` | `frontend/` | `frontend/Dockerfile` | +| `operator` | `operator/` | `operator/Dockerfile` | +| `model-downloader` | `model-downloader/` | `model-downloader/Dockerfile` | +| `key-manager` | `.` (repo root) | `key-manager/Dockerfile` | +| `frontend` | `frontend/` | `frontend/Dockerfile` | -The `build-frontend` job (React SPA → nginx) is added alongside the existing image jobs. +- GitHub Container Registry, as `ghcr.io/nebari-dev/llm-serving-pack/` +- Quay, as `quay.io/nebari/llm-serving-pack-` -Each job uses `docker/metadata-action` to derive tags automatically: +`docker/metadata-action` derives the tags automatically: | Condition | Tag applied | |-----------|------------| -| Any push | `sha-` | -| Push to `main` branch | branch name (`main`) and `:latest` | -| Push of a `v*` tag | semantic version from the tag, `v` prefix preserved (e.g. `v0.1.0-alpha.9`) | +| Any push (or manual dispatch) | `sha-` | +| Push to the default branch (`main`) | also `latest` | -The `latest` tag only applies when building from the default branch. Version tags applied on a `v*` push take the form `v{{version}}` as extracted by `docker/metadata-action`'s semver pattern. +Pull request builds always run with pushing disabled, so a PR proves the image builds without publishing anything under that PR's commit. -Authentication to GHCR uses the workflow's automatic `GITHUB_TOKEN` with `packages: write` permission. +Authentication uses the workflow's automatic `GITHUB_TOKEN` for GHCR (`packages: write`) and a `QUAY_TOKEN` secret plus a `QUAY_USERNAME` repository variable for Quay. -The chart's `values.yaml` does not set a default tag for the operator, key-manager, and frontend images; it leaves `tag: ""` and falls back to `.Chart.AppVersion` at render time. This means `helm install` without a tag override pulls whatever image version matches the chart's `appVersion`. +Nothing here produces a version-numbered tag. On `main`, the chart's `values.yaml` hardcodes `tag: "latest"` for the operator, key-manager, frontend, and model-downloader images - that is deliberate, and covered in [Chart Versioning](#chart-versioning) and [Releasing](#releasing) below. + +### Release Chart (`release.yaml`) + +**Triggers:** push to `main` that touches `charts/nebari-llm-serving/Chart.yaml`. + +This is the workflow that actually cuts a release. It is a thin caller of the shared `pack-release.yaml` workflow: `release.yaml` just tells it which chart to package (`charts/nebari-llm-serving`), what to call it (`nebari-llm-serving`), and which `values.yaml` keys hold image tags that need to be pinned (`operator.image.tag`, `keyManager.image.tag`, `frontend.image.tag`, `modelDownloader.image.tag`). The full mechanics of what the shared workflow does are described in [Releasing](#releasing) below. ### Docs (`docs.yml`) @@ -91,10 +97,10 @@ The four images built by `build-images.yaml` and their roles in the pack: | Image | Purpose | |-------|---------| -| `ghcr.io/nebari-dev/nebari-llm-serving-pack/operator` | Kubernetes controller that reconciles `LLMModel` CRs | -| `ghcr.io/nebari-dev/nebari-llm-serving-pack/model-downloader` | Init container that downloads model weights into a PVC before the serving pod starts | -| `ghcr.io/nebari-dev/nebari-llm-serving-pack/key-manager` | Key manager REST API for managing per-user API keys | -| `ghcr.io/nebari-dev/nebari-llm-serving-pack/frontend` | LLM serving pack React UI (nginx) - serves the SPA and proxies `/api` to the key-manager | +| `ghcr.io/nebari-dev/llm-serving-pack/operator` | Kubernetes controller that reconciles `LLMModel` CRs | +| `ghcr.io/nebari-dev/llm-serving-pack/model-downloader` | Init container that downloads model weights into a PVC before the serving pod starts | +| `ghcr.io/nebari-dev/llm-serving-pack/key-manager` | Key manager REST API for managing per-user API keys | +| `ghcr.io/nebari-dev/llm-serving-pack/frontend` | LLM serving pack React UI (nginx) - serves the SPA and proxies `/api` to the key-manager | A fifth image, `ghcr.io/llm-d/llm-d-cuda`, is the upstream llm-d GPU serving image. Its version is set in `values.yaml` under `defaults.serving.image` and is not built by this repository. See the [llm-d project](https://github.com/llm-d/llm-d) for its release history. @@ -102,25 +108,35 @@ A fifth image, `ghcr.io/llm-d/llm-d-cuda`, is the upstream llm-d GPU serving ima The Helm chart lives at `charts/nebari-llm-serving/` and has two version fields in `Chart.yaml`: -- **`version`** - the chart packaging version (SemVer). Helm uses this for `helm repo` indexing and upgrade resolution. -- **`appVersion`** - the application version string, which the chart uses as the default image tag for the operator, key-manager, and frontend when no explicit tag override is given. +- **`version`** - the chart packaging version (SemVer). Bumping this field and merging the change to `main` is what cuts a release; see [Releasing](#releasing) below. +- **`appVersion`** - the application version string. The chart only uses it to set the `app.kubernetes.io/version` label on the resources it renders. It has no effect on which image gets deployed. + +Both fields are conventionally bumped together (e.g. to `0.1.0-alpha.9`), but only `version` drives the release automation. The current version is `0.1.0-alpha.9`. + +Image tags are not derived from `appVersion`, and they are not hand-edited in the repository either. On `main`, every image tag in `values.yaml` (operator, key manager, frontend, model downloader) is pinned to `latest` on purpose: the real, immutable tag only gets written when a release is packaged, and that step never touches the repo tree. See [Releasing](#releasing) for where that pinning happens. + +## Releasing + +Cutting a release is one action: open a pull request that bumps `version` in `charts/nebari-llm-serving/Chart.yaml`, and merge it. Everything else is automatic - no git tag to push by hand, no editing image tags, no release step that happens outside a PR merged to `main`. -Both fields are kept in sync; every release bumps them together to the same value (e.g. `0.1.0-alpha.9`). The current version is `0.1.0-alpha.9`. +**What happens when that PR merges:** -## Release Process +1. `build-images.yaml` builds images for the merge commit and pushes them to `ghcr.io` and `quay.io`, tagged `sha-` (see [Build Images](#build-images-build-imagesyaml) above). +2. `release.yaml` runs. It reads the new `version` from `Chart.yaml`, pins that exact sha into the four image tags in the chart's `values.yaml` (operator, key manager, frontend, model downloader), packages the chart with `helm package`, and creates a GitHub Release tagged `nebari-llm-serving-` with auto-generated notes and the packaged chart attached. It then opens a pull request that syncs the pinned chart source to `nebari-dev/helm-repository`; once that PR merges, the chart is published to `oci://quay.io/nebari/charts/nebari-llm-serving`. +3. A pre-release version (anything with a `-` in it, e.g. `0.2.0-alpha.1`) is marked as a pre-release on the GitHub Release automatically. -Releases follow a manual workflow. As of `v0.1.0-alpha.9` there is no automated Helm repository publish step; the release consists of: +**Why `values.yaml` on `main` shows `tag: "latest"`:** the tag floats on purpose. Pinning happens in step 2 above, when a version is packaged, not in the repository tree. The published chart, not the source, is the pinned record of what a given version actually runs. To see exactly which image shas a released version deploys: -1. **Bump chart version** - edit `charts/nebari-llm-serving/Chart.yaml` and update both `version` and `appVersion` to the new value. Commit with a message like `chore(release): cut v0.1.0-alpha.X`. -2. **Push a `v*` tag** - push an annotated or lightweight tag matching the new version (e.g. `git tag v0.1.0-alpha.9 && git push origin v0.1.0-alpha.9`). This tag triggers `build-images.yaml`, which pushes images with the version tag to GHCR. -3. **Create a GitHub release** - the release is created manually (or as a draft) on the GitHub releases page. Release notes summarise the PRs included since the previous release. +``` +helm show values oci://quay.io/nebari/charts/nebari-llm-serving --version +``` -There is currently **no automated Helm chart repository or OCI chart publish step**. Users install the chart directly from a local checkout or from the repository source. A future task will add automated chart publishing to a Helm OCI registry or GitHub Pages-hosted index. +**Reproducibility:** installing a given `--version` always deploys the same shas, because those shas are baked into the packaged chart rather than read live from `main`. Re-running the release pipeline for a version that has already been released is a no-op: it checks whether a GitHub Release for that chart and version already exists, and skips everything if so. -To install from the current release, see the [Installation](/installation/) guide. To work with a local development build, see the [Local Development](/local-development/) page. +To install a released version, see the [Installation](/installation/) guide. To work with a local development build, which reads `values.yaml` as committed (so `latest`-tagged images), see the [Local Development](/local-development/) page. ## Known Gaps - There is no chart-testing (`ct lint`) step that validates the chart against multiple Kubernetes versions. Contributions adding `helm/chart-testing-action` are welcome. -- There is no automated Helm chart publish. The release process is entirely manual after the images are pushed. -- The `v0.1.0-alpha.9` GitHub release is currently a draft and has not been formally published. +- The release workflow does not wait for `build-images.yaml` to finish before pinning image shas into the chart. Both are triggered by the same push to `main` and normally finish independently, but there is no hard gate between them: if the image build for that commit is slow or fails, the released chart is still pinned to that sha, and the image only becomes pullable once the build catches up. +- `build-images.yaml` only triggers on changes under `operator/`, `key-manager/`, `frontend/`, or `model-downloader/`; it does not watch `charts/**`. A release PR that touches only `Chart.yaml` will not trigger a fresh image build for that commit, so the pinned sha could have no matching image. Cut releases from a PR that also changes at least one component, or manually run `build-images.yaml` via `workflow_dispatch` against the release commit first. From e9fbb6d14f8d228f411e49b4e496605deefa07ff Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:28:21 +0200 Subject: [PATCH 5/7] ci: build images on Chart.yaml bump so release-commit shas exist Add charts/nebari-llm-serving/Chart.yaml to the push trigger paths in build-images.yaml so the release commit (a Chart.yaml version bump) produces sha- images before the release workflow pins the chart to that sha. Update the CI/CD docs to reflect that a release now triggers both workflows in parallel instead of describing this as a known gap. --- .github/workflows/build-images.yaml | 6 ++++++ docs/src/content/docs/cicd-and-releasing.md | 12 ++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index b8560ee..79e4949 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -9,6 +9,12 @@ on: - "frontend/**" - "model-downloader/**" - ".github/workflows/build-images.yaml" + # The release workflow pins images to sha-, so the + # release commit (a Chart.yaml bump) must produce images with that + # sha. Building here on the bump ensures they exist. This runs in + # parallel with the release workflow (no hard gate in v1), so images + # are published around the same time the chart becomes consumable. + - "charts/nebari-llm-serving/Chart.yaml" pull_request: paths: - "operator/**" diff --git a/docs/src/content/docs/cicd-and-releasing.md b/docs/src/content/docs/cicd-and-releasing.md index c0bd980..d76fab8 100644 --- a/docs/src/content/docs/cicd-and-releasing.md +++ b/docs/src/content/docs/cicd-and-releasing.md @@ -38,7 +38,9 @@ The Helm lint job runs `helm lint charts/nebari-llm-serving/` against the chart ### Build Images (`build-images.yaml`) -**Triggers:** push to `main` touching `operator/`, `key-manager/`, `frontend/`, `model-downloader/`, or the workflow file itself; pull requests touching those same component paths (builds the image but does not push it); manual `workflow_dispatch`. +**Triggers:** push to `main` touching `operator/`, `key-manager/`, `frontend/`, `model-downloader/`, the workflow file itself, or `charts/nebari-llm-serving/Chart.yaml`; pull requests touching those same component paths (builds the image but does not push it; chart-only PRs do not trigger a build); manual `workflow_dispatch`. + +The `Chart.yaml` path is in the push trigger for a specific reason: a release is cut by bumping `version` in that file (see [Releasing](#releasing)), and the release workflow pins the release commit's images to `sha-`. Triggering a build on the same push that cuts the release ensures those shas exist. Each of the four images (operator, model-downloader, key-manager, frontend - React SPA served by nginx) has its own job in `build-images.yaml`, and each job is a one-line call into the shared `pack-build-image.yaml` workflow, passing the image name, build context, and Dockerfile path. The shared workflow does the actual work: it builds with Docker Buildx and, on a push, pushes the image to two registries: @@ -121,10 +123,14 @@ Cutting a release is one action: open a pull request that bumps `version` in `ch **What happens when that PR merges:** -1. `build-images.yaml` builds images for the merge commit and pushes them to `ghcr.io` and `quay.io`, tagged `sha-` (see [Build Images](#build-images-build-imagesyaml) above). +The merge commit (the `Chart.yaml` bump) touches a path that both `build-images.yaml` and `release.yaml` watch, so it triggers both workflows in parallel: + +1. `build-images.yaml` builds images for the merge commit and pushes them to `ghcr.io` and `quay.io`, tagged `sha-` (see [Build Images](#build-images-build-imagesyaml) above). This is what guarantees the sha the release pins in the next step actually has matching images. 2. `release.yaml` runs. It reads the new `version` from `Chart.yaml`, pins that exact sha into the four image tags in the chart's `values.yaml` (operator, key manager, frontend, model downloader), packages the chart with `helm package`, and creates a GitHub Release tagged `nebari-llm-serving-` with auto-generated notes and the packaged chart attached. It then opens a pull request that syncs the pinned chart source to `nebari-dev/helm-repository`; once that PR merges, the chart is published to `oci://quay.io/nebari/charts/nebari-llm-serving`. 3. A pre-release version (anything with a `-` in it, e.g. `0.2.0-alpha.1`) is marked as a pre-release on the GitHub Release automatically. +There is no hard gate between the two workflows in v1: `release.yaml` does not wait for `build-images.yaml` to finish before pinning shas. In practice both jobs kick off from the same push and finish close together. If the image build for that commit fails or lags, the release still pins the sha, and that failure is visible immediately: the `build-images.yaml` run shows red, and any deploy of that version fails to pull the missing image. The fix is to re-cut the release (bump `version` again) once the underlying build issue is resolved, rather than to retroactively patch a published chart. + **Why `values.yaml` on `main` shows `tag: "latest"`:** the tag floats on purpose. Pinning happens in step 2 above, when a version is packaged, not in the repository tree. The published chart, not the source, is the pinned record of what a given version actually runs. To see exactly which image shas a released version deploys: ``` @@ -138,5 +144,3 @@ To install a released version, see the [Installation](/installation/) guide. To ## Known Gaps - There is no chart-testing (`ct lint`) step that validates the chart against multiple Kubernetes versions. Contributions adding `helm/chart-testing-action` are welcome. -- The release workflow does not wait for `build-images.yaml` to finish before pinning image shas into the chart. Both are triggered by the same push to `main` and normally finish independently, but there is no hard gate between them: if the image build for that commit is slow or fails, the released chart is still pinned to that sha, and the image only becomes pullable once the build catches up. -- `build-images.yaml` only triggers on changes under `operator/`, `key-manager/`, `frontend/`, or `model-downloader/`; it does not watch `charts/**`. A release PR that touches only `Chart.yaml` will not trigger a fresh image build for that commit, so the pinned sha could have no matching image. Cut releases from a PR that also changes at least one component, or manually run `build-images.yaml` via `workflow_dispatch` against the release commit first. From 70d1bf94d43ced05090c32eec550773b20fdc716 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:43:38 +0200 Subject: [PATCH 6/7] docs: update README and ui-development image paths to llm-serving-pack Follows the repo rename; only the ghcr.io image paths change (the github.com repo URLs are redirect-handled and left for the repo-rename cleanup). Go module path untouched. --- README.md | 10 +++++----- docs/src/content/docs/ui-development.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8a124f2..f4eaf4c 100644 --- a/README.md +++ b/README.md @@ -236,10 +236,10 @@ Admin applies LLMModel CR | Image | Description | |-------|-------------| -| `ghcr.io/nebari-dev/nebari-llm-serving-pack/operator` | LLM operator - reconciles LLMModel CRDs | -| `ghcr.io/nebari-dev/nebari-llm-serving-pack/key-manager` | Key manager REST API | -| `ghcr.io/nebari-dev/nebari-llm-serving-pack/frontend` | LLM serving pack React UI (nginx) | -| `ghcr.io/nebari-dev/nebari-llm-serving-pack/model-downloader` | Model download init container (distroless, pixi-managed) | +| `ghcr.io/nebari-dev/llm-serving-pack/operator` | LLM operator - reconciles LLMModel CRDs | +| `ghcr.io/nebari-dev/llm-serving-pack/key-manager` | Key manager REST API | +| `ghcr.io/nebari-dev/llm-serving-pack/frontend` | LLM serving pack React UI (nginx) | +| `ghcr.io/nebari-dev/llm-serving-pack/model-downloader` | Model download init container (distroless, pixi-managed) | ### Infrastructure requirements @@ -281,7 +281,7 @@ make teardown ### Key manager UI -The key manager web UI is a [React](https://react.dev) + TypeScript app (Vite, Tailwind, shadcn/ui) in [`frontend/`](frontend/). In production it ships as its own nginx image (`ghcr.io/nebari-dev/nebari-llm-serving-pack/frontend`) that serves the SPA and proxies `/api` to the API-only key-manager; it is not embedded in the Go binary. For a one-command dev loop that needs **no Keycloak**: +The key manager web UI is a [React](https://react.dev) + TypeScript app (Vite, Tailwind, shadcn/ui) in [`frontend/`](frontend/). In production it ships as its own nginx image (`ghcr.io/nebari-dev/llm-serving-pack/frontend`) that serves the SPA and proxies `/api` to the API-only key-manager; it is not embedded in the Go binary. For a one-command dev loop that needs **no Keycloak**: ```bash # One-time: copy dev/.env.example to dev/.env and set OPENROUTER_API_KEY diff --git a/docs/src/content/docs/ui-development.md b/docs/src/content/docs/ui-development.md index d4bc336..c9ad82d 100644 --- a/docs/src/content/docs/ui-development.md +++ b/docs/src/content/docs/ui-development.md @@ -90,7 +90,7 @@ npm run check # biome lint + format ## Shipping a change The UI ships as its own image -(`ghcr.io/nebari-dev/nebari-llm-serving-pack/frontend`, nginx serving the built +(`ghcr.io/nebari-dev/llm-serving-pack/frontend`, nginx serving the built bundle) - it is no longer embedded in the Go key-manager binary. Committing your edits to `frontend/` is all that is needed for them to ship: CI builds and publishes the frontend image on merge, and a chart upgrade rolls it out. From dbef333f64f34339c884d36dbdb1aa852ed6bbdf Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Wed, 15 Jul 2026 18:24:27 +0200 Subject: [PATCH 7/7] ci: pin reusable release workflows to nebari-dev/.github@v1 Now that nebari-dev/.github#43 is merged and tagged v1, pin the caller refs to the release tag and drop the bring-up TODOs. --- .github/workflows/build-images.yaml | 12 ++++-------- .github/workflows/release.yaml | 3 +-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 79e4949..af93dbe 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -25,8 +25,7 @@ on: jobs: operator: - # TODO: re-pin to the v1 tag before this PR merges (bring-up ref only). - uses: nebari-dev/.github/.github/workflows/pack-build-image.yaml@feat/pack-release-workflows + uses: nebari-dev/.github/.github/workflows/pack-build-image.yaml@v1 with: image: operator context: operator/ @@ -35,8 +34,7 @@ jobs: QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} model-downloader: - # TODO: re-pin to the v1 tag before this PR merges (bring-up ref only). - uses: nebari-dev/.github/.github/workflows/pack-build-image.yaml@feat/pack-release-workflows + uses: nebari-dev/.github/.github/workflows/pack-build-image.yaml@v1 with: image: model-downloader context: model-downloader/ @@ -45,8 +43,7 @@ jobs: QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} key-manager: - # TODO: re-pin to the v1 tag before this PR merges (bring-up ref only). - uses: nebari-dev/.github/.github/workflows/pack-build-image.yaml@feat/pack-release-workflows + uses: nebari-dev/.github/.github/workflows/pack-build-image.yaml@v1 with: image: key-manager context: "." @@ -56,8 +53,7 @@ jobs: QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} frontend: - # TODO: re-pin to the v1 tag before this PR merges (bring-up ref only). - uses: nebari-dev/.github/.github/workflows/pack-build-image.yaml@feat/pack-release-workflows + uses: nebari-dev/.github/.github/workflows/pack-build-image.yaml@v1 with: image: frontend context: frontend/ diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index eec25a6..cc04741 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -8,8 +8,7 @@ on: jobs: release: - # TODO: re-pin to the v1 tag before this PR merges (bring-up ref only). - uses: nebari-dev/.github/.github/workflows/pack-release.yaml@feat/pack-release-workflows + uses: nebari-dev/.github/.github/workflows/pack-release.yaml@v1 with: chart-path: charts/nebari-llm-serving chart-name: nebari-llm-serving