SHARD-232: package the guest kernel: a reproducible build, a release workflow, and a checksummed fetch

.github/workflows/kernel.yml

@@ -0,0 +1,76 @@
+name: kernel
+
+# Builds each guest kernel twice and proves the hashes match the ones shard was built with. A PR that
+# touches the build runs that much; a manual dispatch also publishes under the tag services/kernel names.
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - Makefile
+      - packaging/kernel/**
+      - services/kernel/**
+      - .github/workflows/kernel.yml
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: kernel ${{ matrix.arch }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [arm64, amd64]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: build twice and compare
+        run: make kernel-reproducible ARCH=${{ matrix.arch }}
+
+      - name: print the hash
+        run: cat bin/kernel/${{ matrix.arch }}/*.sha256
+
+      - name: check the hash shard expects
+        run: |
+          want=$(go run ./packaging/kernel/tag ${{ matrix.arch }})
+          got=$(cut -d' ' -f1 bin/kernel/${{ matrix.arch }}/*.sha256)
+          test "$want" = "$got" || { echo "services/kernel expects $want, built $got"; exit 1; }
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: kernel-${{ matrix.arch }}
+          path: bin/kernel/${{ matrix.arch }}/*
+
+  release:
+    name: release
+    if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main'
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+
+      - name: publish
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          tag=$(go run ./packaging/kernel/tag)
+          cat dist/*.sha256 > dist/SHA256SUMS
+          gh release create "$tag" --target "$GITHUB_SHA" --title "guest kernel $tag" --notes "Built by the kernel workflow from packaging/kernel. Reproducible: each file was built twice." dist/Image-arm64 dist/vmlinux-amd64 dist/SHA256SUMS

AGENTS.md

@@ -25,6 +25,8 @@ make itest                   integration tests for ITEST_PKG, on the devbox
 make e2e                     the whole lifecycle on this host, as root, over a daemon it starts (SHARD-17)
 make devbox-e2e              the same script, on the devbox; PROVIDER=sysbox or PROVIDER=runc picks the substrate (SHARD-90, SHARD-224)
 make devbox-demo             record scripts/demo.sh on the devbox into docs/demo.cast (SHARD-36)
+make kernel ARCH=arm64       build the guest kernel for one arch in Docker, into bin/kernel (SHARD-232)
+make kernel-reproducible     the same twice, and fail if the two hashes differ
 make lint                    golangci-lint (v2: brew install golangci-lint)
 make lint-fix                apply the fixes golangci-lint can make
 make fmt                     apply formatting
@@ -76,6 +78,8 @@ services/provider/firecracker/  implements models.Provider on Firecracker
 services/provider/conformance/  the test suite every substrate must pass
 
 packaging/systemd/         the unit for the daemon, and the one for the TCP front
+packaging/kernel/          the guest kernel build: pinned image, config per arch, release tag helper
+services/kernel/           the guest kernel fetch: release URL, checksum, the dev override
 docs/
 ```
 

Makefile

@@ -13,7 +13,13 @@ PROVIDER ?= gvisor
 # Which packages `make itest` runs on the box. Narrow it while you work on one ticket.
 ITEST_PKG ?= ./services/network/... ./services/provider/gvisor/... ./services/provider/sysbox/... ./services/provider/runc/...
 
-.PHONY: all build build-linux build-shard-init build-shard-init-linux test test-integration e2e-test vet lint lint-fix fmt fmt-check vuln check clean devbox-sync devbox-test itest e2e devbox-e2e devbox-demo
+include packaging/kernel/version.mk
+# The guest kernel arch to build: arm64 or amd64.
+ARCH ?= arm64
+KERNEL_OUT := bin/kernel
+KERNEL_IMAGE := packaging-kernel-builder
+
+.PHONY: all build build-linux build-shard-init build-shard-init-linux test test-integration e2e-test vet lint lint-fix fmt fmt-check vuln check clean devbox-sync devbox-test itest e2e devbox-e2e devbox-demo kernel kernel-reproducible
 
 all: check build
 
@@ -101,3 +107,21 @@ check: fmt-check vet lint test e2e-test
 
 clean:
 	rm -rf bin
+
+# One guest kernel, built in the pinned amd64 image so the bytes match CI wherever it runs (SHARD-232).
+kernel:
+	docker build --platform linux/amd64 -q -t $(KERNEL_IMAGE) packaging/kernel >/dev/null
+	mkdir -p $(KERNEL_OUT)/$(ARCH) $(KERNEL_OUT)/cache
+	docker run --rm --platform linux/amd64 \
+		-e KERNEL_VERSION=$(KERNEL_VERSION) -e KERNEL_SHA256=$(KERNEL_SHA256) \
+		-v $(CURDIR)/packaging/kernel:/config:ro \
+		-v $(CURDIR)/$(KERNEL_OUT)/cache:/cache \
+		-v $(CURDIR)/$(KERNEL_OUT)/$(ARCH):/out \
+		$(KERNEL_IMAGE) $(ARCH)
+
+# Builds twice and refuses a hash that moved; the release workflow runs this before it publishes.
+kernel-reproducible:
+	$(MAKE) kernel ARCH=$(ARCH)
+	cp $(KERNEL_OUT)/$(ARCH)/*.sha256 $(KERNEL_OUT)/$(ARCH).first.sha256
+	$(MAKE) kernel ARCH=$(ARCH)
+	diff $(KERNEL_OUT)/$(ARCH).first.sha256 $(KERNEL_OUT)/$(ARCH)/*.sha256

docs/kernel.md

@@ -0,0 +1,64 @@
+# The guest kernel
+
+A microVM substrate boots a kernel shard ships, never one from the host. There is one Linux release
+per shard version, built once per architecture, and every build of it is byte-identical. The VZ
+provider boots the arm64 one; Firecracker will boot the amd64 one (SHARD-232, shared with M8).
+
+## What is in it
+
+`packaging/kernel/config-<arch>` is a full `.config` with no modules and no initrd: virtio-blk,
+virtio-net, virtio-vsock, virtio-console, virtio-pci and virtio-mmio, ext4, overlay, squashfs,
+cgroups, namespaces and seccomp are built in. Both started as Cloud Hypervisor's `ch_defconfig` at
+their `ch-6.12.8` tag, which hypeman boots on Virtualization.framework in production, and
+`olddefconfig` carries them to the pinned release. A change to either file is a new `Build`.
+
+## How it is built
+
+```
+make kernel ARCH=arm64          one build, into bin/kernel/arm64/Image-arm64 and its .sha256
+make kernel ARCH=amd64          the same, bin/kernel/amd64/vmlinux-amd64
+make kernel-reproducible        two builds, and a diff of the two hashes
+```
+
+The build runs in `packaging/kernel/Dockerfile`, a `debian:13` image pinned by digest, with apt
+pointed at a dated `snapshot.debian.org` archive and every package at an exact version, and always
+`linux/amd64`, so a Mac and a GitHub runner produce the same bytes: same compiler, same cross
+compiler for arm64, and `KBUILD_BUILD_TIMESTAMP`, `KBUILD_BUILD_USER`, `KBUILD_BUILD_HOST` and
+`SOURCE_DATE_EPOCH` fixed. The source tarball is fetched from `cdn.kernel.org` once into
+`bin/kernel/cache` and checked against the sha256 in `packaging/kernel/version.mk`. That file and
+`services/kernel.Version` must agree; a unit test says so.
+
+On a Mac the build is emulated and takes a while; nothing in it needs a Mac, so a Linux box with
+Docker is the faster place.
+
+## How it is released and fetched
+
+The `kernel` workflow (`.github/workflows/kernel.yml`, manual dispatch only) builds each arch
+twice, compares the hashes, checks them against the ones `services/kernel` was built with, and
+publishes `Image-arm64`, `vmlinux-amd64` and `SHA256SUMS` under the release tag `kernel-<version>-<build>`.
+A hash that does not match what the Go code expects fails the workflow, so a release can never
+carry a kernel the daemon would refuse.
+
+The daemon fetches the file for the host arch into `<root>/kernel/<tag>/` on first use, hashes it
+before every boot, and refuses one that changed: `kernel checksum mismatch`. `shard inspect` shows
+the tag in `kernel` on a sandbox that booted one.
+
+### The dev path
+
+Before the first release, or to boot a kernel that is not released, a daemon takes one from disk:
+
+```
+SHARD_KERNEL=/path/to/Image-arm64 SHARD_KERNEL_SHA256=<its sha256> shard daemon ...
+```
+
+The hash is still checked, against the value given. Both variables or neither; one alone is an
+error at start. This is for a developer with a fresh build, not for an install.
+
+## Bumping it
+
+1. Change `KERNEL_VERSION` and `KERNEL_SHA256` in `packaging/kernel/version.mk`, from
+   `https://cdn.kernel.org/pub/linux/kernel/v6.x/sha256sums.asc`.
+2. Set `Version` in `services/kernel/kernel.go` to the same, and `Build` to 1; a config-only
+   change keeps `Version` and adds one to `Build`.
+3. `make kernel-reproducible ARCH=arm64` and `ARCH=amd64`; put the two hashes in `artifacts`.
+4. Merge, then run the `kernel` workflow on `main`. It refuses if a hash differs from step 3.

docs/provider-vz.md

@@ -41,10 +41,10 @@ Rejected: VMs in the daemon process. Two sandboxes per host, and every sandbox d
 
 ### The kernel is ours, and it is a raw arm64 Image
 
-shard ships one Linux kernel per architecture (SHARD-232): virtio-blk, virtio-net, virtio-vsock,
-virtio-console, ext4 and overlay built in, no modules, no initrd, versioned and checksummed with the
-release, downloaded into the shard root on first use. The Firecracker provider boots the same amd64
-kernel; the arm64 one is this substrate's.
+shard ships one Linux kernel per architecture (SHARD-232, `docs/kernel.md`): virtio-blk, virtio-net,
+virtio-vsock, virtio-console, ext4 and overlay built in, no modules, no initrd, versioned and
+checksummed with the release, downloaded into the shard root on first use. The Firecracker provider
+boots the same amd64 kernel; the arm64 one is this substrate's.
 
 The framework's Linux boot loader takes a raw arm64 `Image` and nothing else. A distribution kernel
 does not fit: Alpine's `vmlinuz-virt` is an EFI zboot wrapper (a PE file, `zimg` magic at offset 4,

models/sandbox.go

@@ -15,7 +15,9 @@ type Sandbox struct {
 	Name     string `json:"name,omitempty"`
 	Image    string `json:"image"`
 	Provider string `json:"provider"`
-	State    State  `json:"state"`
+	// Kernel is the guest kernel a microVM substrate booted, as its release tag; empty on a container substrate.
+	Kernel string `json:"kernel,omitempty"`
+	State  State  `json:"state"`
 	// ExitStatus is the last entrypoint exit, nil until one happens. A sandbox has none of its own.
 	ExitStatus *ExitStatus `json:"exit_status,omitempty"`
 	// StoppedReason says why shard stopped it when no operator did, empty otherwise.

packaging/kernel/Dockerfile

@@ -0,0 +1,32 @@
+# The one toolchain every guest kernel is built with; the digest pins the filesystem, the snapshot pins apt.
+FROM --platform=linux/amd64 debian:13@sha256:9cc080028c43b27d2074d63a5f9caf7166d731494965616c1a6d2827a004585c
+
+# A dated snapshot never moves, and apt still checks its signed Release file; only its Valid-Until has lapsed.
+ARG SNAPSHOT=20260918T000000Z
+RUN echo "deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/${SNAPSHOT} trixie main" > /etc/apt/sources.list \
+    && rm -f /etc/apt/sources.list.d/debian.sources \
+    && apt-get -o Acquire::Retries=5 update \
+    && apt-get -o Acquire::Retries=5 install -y --no-install-recommends \
+    bc=1.07.1-4 \
+    binutils=2.44-3 \
+    binutils-aarch64-linux-gnu=2.44-3 \
+    bison=2:3.8.2+dfsg-1+b2 \
+    build-essential=12.12 \
+    ca-certificates=20250419 \
+    cpio=2.15+dfsg-2 \
+    curl=8.14.1-2+deb13u5 \
+    flex=2.6.4-8.2+b4 \
+    gcc-14=14.2.0-19 \
+    gcc-14-aarch64-linux-gnu=14.2.0-19cross1 \
+    gcc-aarch64-linux-gnu=4:14.2.0-1 \
+    kmod=34.2-2 \
+    libc6-dev=2.41-12+deb13u4 \
+    libelf-dev=0.192-4 \
+    libssl-dev=3.5.7-1~deb13u2 \
+    make=4.4.1-2 \
+    python3=3.13.5-1 \
+    xz-utils=5.8.1-1+deb13u1 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY build.sh /build.sh
+ENTRYPOINT ["/build.sh"]

packaging/kernel/build.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+# Builds one guest kernel inside the pinned image. Called by `make kernel ARCH=<arch>`, never by hand.
+set -eu
+
+ARCH="${1:?arch: arm64 or amd64}"
+KERNEL_VERSION="${KERNEL_VERSION:?}"
+KERNEL_SHA256="${KERNEL_SHA256:?}"
+SRC=/src
+OUT=/out
+
+case "$ARCH" in
+arm64) KARCH=arm64; CROSS=aarch64-linux-gnu-; TARGET=Image; ARTIFACT=arch/arm64/boot/Image; OUTNAME=Image-arm64 ;;
+amd64) KARCH=x86_64; CROSS=; TARGET=vmlinux; ARTIFACT=vmlinux; OUTNAME=vmlinux-amd64 ;;
+*) echo "unknown arch $ARCH" >&2; exit 2 ;;
+esac
+
+tarball="/cache/linux-$KERNEL_VERSION.tar.xz"
+mkdir -p /cache "$OUT"
+if [ ! -f "$tarball" ]; then
+	curl -fsSL -o "$tarball.part" "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-$KERNEL_VERSION.tar.xz"
+	mv "$tarball.part" "$tarball"
+fi
+echo "$KERNEL_SHA256  $tarball" | sha256sum -c - >/dev/null
+
+rm -rf "$SRC"
+mkdir -p "$SRC"
+tar -xJf "$tarball" -C "$SRC" --strip-components=1
+cp "/config/config-$ARCH" "$SRC/.config"
+
+# A fixed timestamp, user, host and version are what make two builds byte-identical.
+export KBUILD_BUILD_TIMESTAMP='Thu Jan  1 00:00:00 UTC 1970'
+export KBUILD_BUILD_USER=shard
+export KBUILD_BUILD_HOST=shard
+export KBUILD_BUILD_VERSION=1
+export SOURCE_DATE_EPOCH=0
+
+cd "$SRC"
+make ARCH="$KARCH" CROSS_COMPILE="$CROSS" olddefconfig >/dev/null
+make ARCH="$KARCH" CROSS_COMPILE="$CROSS" -j"$(nproc)" "$TARGET" >/dev/null
+
+cp "$ARTIFACT" "$OUT/$OUTNAME"
+(cd "$OUT" && sha256sum "$OUTNAME" > "$OUTNAME.sha256")
+cat "$OUT/$OUTNAME.sha256"