diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go
index c44c513c660..b5156617710 100644
--- a/aks-node-controller/app.go
+++ b/aks-node-controller/app.go
@@ -50,8 +50,24 @@ type App struct {
hotfixVersionPath string
// aptSourcesDir overrides the default APT sources directory for testing.
aptSourcesDir string
+ // aptTrustedKeyringsDir overrides the default APT trusted keyrings directory for testing.
+ aptTrustedKeyringsDir string
+ // yumReposDir overrides the default RPM repositories directory for testing.
+ yumReposDir string
// osReleasePath overrides the default /etc/os-release path for testing.
osReleasePath string
+ // goArch overrides runtime.GOARCH for repository-path tests.
+ goArch string
+ // repositoryTempDir overrides where repository downloads and extraction are staged.
+ repositoryTempDir string
+ // vhdBinaryPath, hotfixBinaryPath, and pkgBinaryPath override ANC binary paths for testing.
+ vhdBinaryPath string
+ hotfixBinaryPath string
+ pkgBinaryPath string
+ // verifyRepositorySignature overrides gpgv-backed repository signature verification.
+ verifyRepositorySignature func(ctx context.Context, signedPath, signaturePath string, keyrings []string) error
+ // extractRepositoryPackage overrides package extraction for deterministic unit tests.
+ extractRepositoryPackage func(ctx context.Context, format, packagePath, destination string) error
// nodeCustomDataPath overrides the default nodecustomdata path for testing.
nodeCustomDataPath string
// nodeConfigPath overrides the default AKSNodeConfig path for testing. It is the
@@ -74,13 +90,6 @@ type App struct {
// grpcDialContext overrides how the gRPC LPS client dials, letting tests point the client at
// an in-process (bufconn) server. When nil, the real TLS dial to the apiserver front is used.
grpcDialContext func(ctx context.Context, target string) (net.Conn, error)
- // httpDownload overrides the real HTTP GET for download-hotfix artifact fetching, letting
- // unit tests inject canned binary content or errors without real networking. When nil, the
- // real HTTP download is used.
- httpDownload func(ctx context.Context, url string) ([]byte, error)
- // downloadDir overrides the directory where artifact downloads are staged. When empty,
- // defaults to filepath.Dir(hotfixBinaryPath). Used for testing.
- downloadDir string
}
// provision.json values are emitted as strings by the shell jq invocation.
diff --git a/aks-node-controller/checkhotfix.go b/aks-node-controller/checkhotfix.go
index 0820b76a5b2..f2d132762d1 100644
--- a/aks-node-controller/checkhotfix.go
+++ b/aks-node-controller/checkhotfix.go
@@ -211,7 +211,7 @@ func (a *App) checkHotfix(ctx context.Context) (checkHotfixOutcome, error) {
// value keeps the reported outcome consistent with what download-hotfix will actually read:
// a pointer with no entry for this node's base stages nothing resolvable, so it must report
// noHotfixForBase, not LPSRead.
- staged := hotfixConfig{Hotfixes: cfg.Hotfixes, Artifacts: cfg.Artifacts}
+ staged := hotfixConfig{Hotfixes: cfg.Hotfixes}
if err := writeHotfixConfig(hotfixPath, staged); err != nil {
return outcomeFailed, fmt.Errorf("writing hotfix config: %w", err)
@@ -444,8 +444,7 @@ func (a *App) coldStartHotfixConfig() (hotfixConfig, bool, error) {
// Lenient parse: the AKSNodeConfig is protojson, but the cold-start pointer is an
// out-of-contract top-level object, so parse it permissively with encoding/json.
var lenient struct {
- Hotfixes map[string]string `json:"hotfixes"`
- Artifacts map[string]map[string]artifactInfo `json:"artifacts"`
+ Hotfixes map[string]string `json:"hotfixes"`
}
if err := json.Unmarshal(raw, &lenient); err != nil {
return hotfixConfig{}, false, fmt.Errorf("parsing cold-start hotfixes from node config: %w", err)
@@ -453,7 +452,7 @@ func (a *App) coldStartHotfixConfig() (hotfixConfig, bool, error) {
if len(lenient.Hotfixes) == 0 {
return hotfixConfig{}, false, nil
}
- return hotfixConfig{Hotfixes: lenient.Hotfixes, Artifacts: lenient.Artifacts}, true, nil
+ return hotfixConfig{Hotfixes: lenient.Hotfixes}, true, nil
}
// writeHotfixConfig stages the LPS-served hotfixes map to the path download-hotfix reads.
@@ -482,21 +481,13 @@ func writeHotfixConfig(path string, cfg hotfixConfig) error {
hotfixes = map[string]string{}
}
out := struct {
- Version string `json:"version,omitempty"`
- ScriptsVersion string `json:"scripts_version,omitempty"`
- Hotfixes map[string]string `json:"hotfixes"`
- Artifacts map[string]map[string]artifactInfo `json:"artifacts,omitempty"`
+ Version string `json:"version,omitempty"`
+ ScriptsVersion string `json:"scripts_version,omitempty"`
+ Hotfixes map[string]string `json:"hotfixes"`
}{
Version: existing.Version,
ScriptsVersion: existing.ScriptsVersion,
Hotfixes: hotfixes,
- Artifacts: cfg.Artifacts,
- }
- // Preserve existing artifacts when the incoming config has none (e.g. LPS response
- // doesn't include artifacts yet). This mirrors the Version/ScriptsVersion preservation
- // and avoids erasing artifacts that cloud-init originally wrote.
- if out.Artifacts == nil {
- out.Artifacts = existing.Artifacts
}
data, err := json.Marshal(out)
if err != nil {
diff --git a/aks-node-controller/checkhotfix_test.go b/aks-node-controller/checkhotfix_test.go
index ace49a87f81..e7a62a30398 100644
--- a/aks-node-controller/checkhotfix_test.go
+++ b/aks-node-controller/checkhotfix_test.go
@@ -682,13 +682,15 @@ func TestWriteHotfixConfig_EmptyMapKeepsStableKey(t *testing.T) {
}
}
-// TestWriteHotfixConfig_PreservesExistingVersionAndScriptsVersion is the unit-level guard for
-// the read-modify-write: given a pre-existing file (as cloud-init writes) carrying version and
-// scripts_version, writeHotfixConfig must keep those fields and only replace the hotfixes map.
-func TestWriteHotfixConfig_PreservesExistingVersionAndScriptsVersion(t *testing.T) {
+// TestWriteHotfixConfig_PreservesVersionsAndDropsArtifacts guards the read-modify-write:
+// version fields remain compatible with cloud-init, while the retired artifacts contract is removed.
+func TestWriteHotfixConfig_PreservesVersionsAndDropsArtifacts(t *testing.T) {
path := filepath.Join(t.TempDir(), "hotfix.json")
- require.NoError(t, os.WriteFile(path, []byte(
- `{"version":"202604.01.5","scripts_version":"202604.01.7","hotfixes":{"202604.01":"202604.01.5"}}`), 0644))
+ existing := `{"version":"202604.01.5","scripts_version":"202604.01.7",` +
+ `"hotfixes":{"202604.01":"202604.01.5"},` +
+ `"artifacts":{"202604.01.5":{"linux-ubuntu-22.04-amd64":` +
+ `{"url":"https://packages.microsoft.com/fake.deb","sha256":"abc123"}}}}`
+ require.NoError(t, os.WriteFile(path, []byte(existing), 0644))
require.NoError(t, writeHotfixConfig(path, hotfixConfig{Hotfixes: map[string]string{"202604.01": "202604.01.9"}}))
diff --git a/aks-node-controller/hotfix.go b/aks-node-controller/hotfix.go
index def7761611d..b335f6c0015 100644
--- a/aks-node-controller/hotfix.go
+++ b/aks-node-controller/hotfix.go
@@ -1,17 +1,10 @@
package main
import (
- "bytes"
"context"
- "crypto/sha256"
- "encoding/hex"
"encoding/json"
- "errors"
"fmt"
- "io"
"log/slog"
- "net/http"
- "net/url"
"os"
"os/exec"
"path/filepath"
@@ -19,7 +12,6 @@ import (
"strings"
"time"
- "github.com/Azure/agentbaker/aks-node-controller/common"
"github.com/Masterminds/semver/v3"
)
@@ -36,9 +28,6 @@ const (
hotfixBinaryPath = "/opt/azure/containers/aks-node-controller-hotfix"
// pkgBinaryPath is where apt/dnf package installs the binary.
pkgBinaryPath = "/usr/bin/aks-node-controller"
-
- // HTTP download settings.
- downloadTimeout = 30 * time.Second
)
// downloadHotfix installs the requested hotfix and stages it alongside the VHD-baked binary.
@@ -112,37 +101,81 @@ func (a *App) downloadBinaryHotfixIfNeeded(ctx context.Context, cfg *hotfixConfi
slog.Info("downloading ANC hotfix", "current", Version, "target", hotfixVersion)
- // Prefer direct HTTP download when an artifact descriptor is available. This avoids the
- // package manager's repository refresh, version resolution, and package installation,
- // reducing hotfix latency while retaining SHA-256 verification. Transient download
- // failures fall back to the package manager below.
- if err := a.tryDirectDownload(ctx, cfg, hotfixVersion); err == nil {
+ routeStart := time.Now()
+ if err := a.tryRepositoryDownload(ctx, hotfixVersion); err == nil {
return nil
} else if isIntegrityError(err) {
- return err
+ a.removeStaleHotfix()
+ return fmt.Errorf("repository integrity check failed for hotfix version %s: %w", hotfixVersion, err)
+ } else {
+ slog.Warn("safe repository download unavailable, falling back to package manager",
+ "version", hotfixVersion, "error", err)
}
- // Fallback: install via package manager (apt-get or dnf/tdnf).
if err := a.installFromPMC(ctx, hotfixVersion); err != nil {
return fmt.Errorf("install hotfix version %s: %w", hotfixVersion, err)
}
- if err := copyBinaryAlongside(pkgBinaryPath, hotfixBinaryPath, vhdBinaryPath); err != nil {
+ if err := copyBinaryAlongside(a.pkgPath(), a.hotfixPath(), a.vhdPath()); err != nil {
return fmt.Errorf("stage hotfix binary: %w", err)
}
- slog.Info("downloaded ANC hotfix", "target", hotfixVersion, "path", hotfixBinaryPath)
+ // Mirrors the fast path's durationMs so the two can be compared from node logs.
+ slog.Info("downloaded ANC hotfix", "target", hotfixVersion, "path", a.hotfixPath(),
+ "durationMs", time.Since(routeStart).Milliseconds())
return nil
}
-// artifactInfo describes a directly-downloadable package artifact with its integrity digest.
-type artifactInfo struct {
- URL string `json:"url"`
- SHA256 string `json:"sha256"`
+func (a *App) vhdPath() string {
+ if a.vhdBinaryPath != "" {
+ return a.vhdBinaryPath
+ }
+ return vhdBinaryPath
+}
+
+func (a *App) hotfixPath() string {
+ if a.hotfixBinaryPath != "" {
+ return a.hotfixBinaryPath
+ }
+ return hotfixBinaryPath
+}
+
+func (a *App) pkgPath() string {
+ if a.pkgBinaryPath != "" {
+ return a.pkgBinaryPath
+ }
+ return pkgBinaryPath
}
-// hotfixConfig is the JSON structure of the hotfix configuration file.
-// Using JSON allows future extension (e.g., adding checksum, source URL) without format changes.
+// removeStaleHotfix disarms a previously staged hotfix binary after an integrity failure,
+// so the launcher falls back to the VHD-baked ANC instead of re-running the stale copy.
+//
+// Removal is the intent, but it is not the guarantee: the launcher selects the hotfix on
+// `[ -x ]` alone and ignores this process's exit status, so a remove that fails would
+// silently leave the stale binary armed. Clearing the executable bits closes that gate
+// independently, giving a second way to disarm when unlink cannot succeed (e.g. an
+// immutable attribute or a read-only mount). The staged binary is only ever written by
+// copyBinaryAlongside from a package-manager-verified install, so the risk being contained
+// here is running a stale-but-authentic ANC, not attacker-controlled code.
+func (a *App) removeStaleHotfix() {
+ path := a.hotfixPath()
+ err := os.Remove(path)
+ if err == nil || os.IsNotExist(err) {
+ return
+ }
+ slog.Warn("failed to remove stale hotfix binary after repository integrity failure",
+ "path", path, "error", err)
+
+ if chmodErr := os.Chmod(path, 0o600); chmodErr != nil {
+ slog.Error("stale hotfix binary remains executable after repository integrity failure",
+ "path", path, "removeError", err, "chmodError", chmodErr)
+ return
+ }
+ slog.Warn("cleared executable bits on stale hotfix binary that could not be removed",
+ "path", path)
+}
+
+// hotfixConfig is the version-only JSON structure shared with LPS and cloud-init.
type hotfixConfig struct {
// Version is the legacy single-version pointer. It is still honored when Hotfixes
// is empty, preserving backward compatibility with the original config shape.
@@ -157,12 +190,6 @@ type hotfixConfig struct {
// whose key is absent gets no hotfix (default deny). When non-empty, this map
// takes precedence over Version.
Hotfixes map[string]string `json:"hotfixes,omitempty"`
-
- // Artifacts maps a hotfix version to per-platform/architecture artifact descriptors for direct
- // HTTP download. When present and matching, the download path bypasses the package
- // manager entirely. The outer key is the hotfix version (e.g. "202607.02.2"), the
- // inner key is "GOOS-ID-VERSION_ID-GOARCH" (e.g. "linux-ubuntu-22.04-amd64").
- Artifacts map[string]map[string]artifactInfo `json:"artifacts,omitempty"`
}
// hotfixBaseFromVersion extracts the "YYYYMM.DD" base from an ANC version string of
@@ -237,7 +264,11 @@ func (a *App) parseLinuxPlatformInfo() (platformInfo, error) {
if err != nil {
return platformInfo{}, fmt.Errorf("reading %s: %w", osReleasePath, err)
}
- info := platformInfo{OS: "linux", Arch: runtime.GOARCH}
+ arch := a.goArch
+ if arch == "" {
+ arch = runtime.GOARCH
+ }
+ info := platformInfo{OS: "linux", Arch: arch}
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "ID=") {
@@ -271,7 +302,7 @@ func (a *App) detectPackageManager() (packageManager, error) {
switch info.ID {
case "ubuntu":
return pkgMgrApt, nil
- case "azurelinux", "mariner":
+ case osIDAzureLinux, osIDMariner:
return preferredRpmManager(), nil
default:
return "", fmt.Errorf("unsupported OS: %s", info.ID)
@@ -433,241 +464,6 @@ func copyBinaryAlongside(src, dst, refPath string) error {
return nil
}
-// tryDirectDownload attempts to download the hotfix binary directly via HTTP using the
-// artifact descriptor. Returns nil on success, an integrityError on validation failure
-// (caller must NOT fallback), or a regular error on network/transient failure (caller may fallback).
-// Returns a non-nil non-integrity error when no artifact is available (signals fallback).
-func (a *App) tryDirectDownload(ctx context.Context, cfg *hotfixConfig, hotfixVersion string) error {
- artifact, artifactKey := a.resolveArtifact(cfg, hotfixVersion)
- if artifact == nil {
- return fmt.Errorf("no artifact descriptor available")
- }
-
- slog.Info("artifact descriptor found, attempting direct HTTP download",
- "version", hotfixVersion, "key", artifactKey, "url", artifact.URL)
-
- tmpPath, err := a.downloadAndVerify(ctx, artifact.URL, artifact.SHA256)
- if err != nil {
- if isIntegrityError(err) {
- // Remove any previously staged hotfix binary so the wrapper falls back to the
- // VHD-baked ANC — a stale hotfix binary must not run after an integrity failure.
- if removeErr := os.Remove(hotfixBinaryPath); removeErr != nil && !os.IsNotExist(removeErr) {
- slog.Warn("failed to remove stale hotfix binary on integrity error",
- "path", hotfixBinaryPath, "error", removeErr)
- }
- return fmt.Errorf("artifact integrity check failed for %s: %w", hotfixVersion, err)
- }
- slog.Warn("direct HTTP download failed, falling back to package manager",
- "version", hotfixVersion, "error", err)
- return err
- }
-
- if err := copyBinaryAlongside(tmpPath, hotfixBinaryPath, vhdBinaryPath); err != nil {
- os.Remove(tmpPath)
- // Staging failure after successful download+verify is a hard error — do not fallback
- // to package manager since we already verified the binary integrity.
- return newIntegrityError("stage hotfix binary from artifact: %v", err)
- }
- os.Remove(tmpPath)
- slog.Info("downloaded ANC hotfix via direct HTTP", "target", hotfixVersion, "path", hotfixBinaryPath)
- return nil
-}
-
-// integrityError marks errors where the downloaded content failed validation.
-// These must NOT fallback to the package manager — the node should keep its VHD-baked ANC.
-type integrityError struct {
- msg string
-}
-
-func (e *integrityError) Error() string { return e.msg }
-
-func newIntegrityError(format string, args ...any) error {
- return &integrityError{msg: fmt.Sprintf(format, args...)}
-}
-
-func isIntegrityError(err error) bool {
- var ie *integrityError
- return errors.As(err, &ie)
-}
-
-// resolveArtifact looks up the artifact descriptor for the given hotfix version and current
-// platform/architecture. Returns nil if no artifact is available (caller should fallback to pkg mgr).
-func (a *App) resolveArtifact(cfg *hotfixConfig, hotfixVersion string) (*artifactInfo, string) {
- if len(cfg.Artifacts) == 0 {
- return nil, ""
- }
- perArch, ok := cfg.Artifacts[hotfixVersion]
- if !ok || len(perArch) == 0 {
- return nil, ""
- }
- key, err := a.buildArtifactKey()
- if err != nil {
- slog.Warn("cannot build artifact key, skipping direct download", "error", err)
- return nil, ""
- }
- ai, ok := perArch[key]
- if !ok {
- return nil, key
- }
- return &ai, key
-}
-
-// buildArtifactKey constructs the platform/architecture lookup key for the artifacts map.
-// Format: "GOOS-ID-VERSION_ID-GOARCH" (e.g. "linux-ubuntu-22.04-amd64").
-func (a *App) buildArtifactKey() (string, error) {
- info, err := a.parseLinuxPlatformInfo()
- if err != nil {
- return "", err
- }
- if info.VersionID == "" {
- return "", fmt.Errorf("VERSION_ID not found in os-release")
- }
- return fmt.Sprintf("%s-%s-%s-%s", info.OS, info.ID, info.VersionID, info.Arch), nil
-}
-
-// validateArtifactURL ensures the URL is HTTPS and the host is in the PMC allowlist.
-func validateArtifactURL(rawURL string) error {
- u, err := url.Parse(rawURL)
- if err != nil {
- return newIntegrityError("invalid artifact URL %q: %v", rawURL, err)
- }
- if u.Scheme != "https" {
- return newIntegrityError("artifact URL must be HTTPS, got %q", u.Scheme)
- }
- host := strings.ToLower(u.Hostname())
- switch host {
- case "packages.microsoft.com":
- return nil
- default:
- return newIntegrityError("artifact URL host %q not in allowlist", host)
- }
-}
-
-// downloadAndVerify downloads the artifact from the given URL, streams it to a temp file
-// while computing its SHA-256 digest, and compares to the expected value. Returns the path
-// to a temp file containing the verified binary. The caller is responsible for removing or
-// renaming the temp file after staging.
-func (a *App) downloadAndVerify(ctx context.Context, artifactURL, expectedSHA256 string) (string, error) {
- if err := validateArtifactURL(artifactURL); err != nil {
- return "", err
- }
- expectedSHA256 = strings.TrimSpace(strings.ToLower(expectedSHA256))
- if expectedSHA256 == "" {
- return "", newIntegrityError("artifact SHA-256 is empty")
- }
-
- // Create temp file for streaming.
- dir := a.downloadDir
- if dir == "" {
- dir = filepath.Dir(hotfixBinaryPath)
- }
- tmp, err := os.CreateTemp(dir, ".aks-node-controller-download-*")
- if err != nil {
- return "", fmt.Errorf("creating temp file in %s: %w", dir, err)
- }
- tmpPath := tmp.Name()
-
- // Cleanup on any failure path.
- success := false
- defer func() {
- if !success {
- tmp.Close()
- os.Remove(tmpPath)
- }
- }()
-
- // Get a reader for the artifact content.
- reader, err := a.getArtifactReader(ctx, artifactURL)
- if err != nil {
- return "", fmt.Errorf("HTTP download %s: %w", artifactURL, err)
- }
- defer reader.Close()
-
- // Stream through SHA-256 hasher into temp file in a single pass — no full-body buffer.
- hasher := sha256.New()
- if _, err := io.Copy(tmp, io.TeeReader(reader, hasher)); err != nil {
- return "", fmt.Errorf("writing temp file %s: %w", tmpPath, err)
- }
- if err := tmp.Close(); err != nil {
- return "", fmt.Errorf("closing temp file %s: %w", tmpPath, err)
- }
-
- // Verify SHA-256.
- actualSHA256 := hex.EncodeToString(hasher.Sum(nil))
- if actualSHA256 != expectedSHA256 {
- os.Remove(tmpPath)
- return "", newIntegrityError("SHA-256 mismatch: expected %s, got %s", expectedSHA256, actualSHA256)
- }
-
- success = true
- return tmpPath, nil
-}
-
-// getArtifactReader returns a ReadCloser for the artifact content. It uses the injectable
-// httpDownload hook for testing (wrapping []byte in a reader), or performs a real streaming
-// HTTP GET.
-func (a *App) getArtifactReader(ctx context.Context, artifactURL string) (io.ReadCloser, error) {
- if a.httpDownload != nil {
- data, err := a.httpDownload(ctx, artifactURL)
- if err != nil {
- return nil, err
- }
- return io.NopCloser(bytes.NewReader(data)), nil
- }
- return a.doHTTPStream(ctx, artifactURL)
-}
-
-// doHTTPStream performs a real streaming HTTP GET and returns the response body.
-// The caller must close the returned ReadCloser.
-func (a *App) doHTTPStream(ctx context.Context, artifactURL string) (io.ReadCloser, error) {
- dlCtx, cancel := context.WithTimeout(ctx, downloadTimeout)
-
- transport := common.NewBaseTransport(common.HTTPTransportOptions{
- DialTimeout: 10 * time.Second,
- TLSHandshakeTimeout: 10 * time.Second,
- ResponseHeaderTimeout: 10 * time.Second,
- })
- client := &http.Client{
- Transport: transport,
- CheckRedirect: func(req *http.Request, via []*http.Request) error {
- if err := validateArtifactURL(req.URL.String()); err != nil {
- return fmt.Errorf("redirect to disallowed host: %w", err)
- }
- return nil
- },
- }
-
- req, err := http.NewRequestWithContext(dlCtx, http.MethodGet, artifactURL, nil)
- if err != nil {
- cancel()
- return nil, err
- }
- resp, err := client.Do(req)
- if err != nil {
- cancel()
- return nil, err
- }
- if resp.StatusCode != http.StatusOK {
- resp.Body.Close()
- cancel()
- return nil, fmt.Errorf("HTTP %d from %s", resp.StatusCode, artifactURL)
- }
- // Wrap body to cancel context on close.
- return &cancelOnClose{ReadCloser: resp.Body, cancel: cancel}, nil
-}
-
-// cancelOnClose wraps an io.ReadCloser to call a cancel func on Close.
-type cancelOnClose struct {
- io.ReadCloser
- cancel context.CancelFunc
-}
-
-func (c *cancelOnClose) Close() error {
- err := c.ReadCloser.Close()
- c.cancel()
- return err
-}
-
// shouldUpgradeToHotfix returns true when the current ANC version should be upgraded
// to the hotfix version. This is true only when both versions share the same YYYYMM.DD
// base and the hotfix has a strictly higher PATCH number (patch-only matching).
diff --git a/aks-node-controller/hotfix_test.go b/aks-node-controller/hotfix_test.go
index a9b283d7070..d8a6acafeb3 100644
--- a/aks-node-controller/hotfix_test.go
+++ b/aks-node-controller/hotfix_test.go
@@ -2,11 +2,9 @@ package main
import (
"context"
- "fmt"
"os"
"os/exec"
"path/filepath"
- "runtime"
"strings"
"testing"
@@ -282,14 +280,24 @@ func TestDownloadHotfix_DevVersionSkips(t *testing.T) {
assert.False(t, installCalled, "should skip when Version is 'dev' (parse error)")
}
-func TestDownloadHotfix_MatchingBaseUpgrades(t *testing.T) {
+func TestDownloadHotfix_MatchingBaseWithArtifactsUsesPackageManager(t *testing.T) {
origVersion := Version
Version = "202604.01.0"
defer func() { Version = origVersion }()
dir := t.TempDir()
path := filepath.Join(dir, "hotfix-config.json")
- require.NoError(t, os.WriteFile(path, []byte(`{"version": "202604.01.1"}`), 0o644))
+ require.NoError(t, os.WriteFile(path, []byte(`{
+ "version": "202604.01.1",
+ "artifacts": {
+ "202604.01.1": {
+ "linux-ubuntu-22.04-amd64": {
+ "url": "https://packages.microsoft.com/fake.deb",
+ "sha256": "abc123"
+ }
+ }
+ }
+ }`), 0o644))
aptDir := filepath.Join(dir, "sources.list.d")
require.NoError(t, os.MkdirAll(aptDir, 0755))
@@ -314,7 +322,7 @@ func TestDownloadHotfix_MatchingBaseUpgrades(t *testing.T) {
// but install should have been called.
err := tt.App.downloadHotfix(context.Background())
require.Error(t, err)
- assert.True(t, installCalled, "should proceed when base matches and hotfix patch is higher")
+ assert.True(t, installCalled, "artifacts must be ignored and the package manager must install the hotfix")
}
func TestDownloadHotfix_UnreadableFileFailsOpen(t *testing.T) {
@@ -652,385 +660,43 @@ func TestShouldUpgradeToHotfix(t *testing.T) {
}
}
-func TestReadHotfixConfig_ParsesArtifacts(t *testing.T) {
- path := filepath.Join(t.TempDir(), "hotfix-config.json")
- require.NoError(t, os.WriteFile(path, []byte(`{
- "hotfixes": {"202607.02": "202607.02.2"},
- "artifacts": {
- "202607.02.2": {
- "linux-ubuntu-22.04-amd64": {
- "url": "https://packages.microsoft.com/ubuntu/22.04/prod/pool/main/a/aks-node-controller/aks-node-controller_0.202607.02.2_amd64.deb",
- "sha256": "abc123"
- }
- }
- }
- }`), 0o644))
- cfg, err := readHotfixConfig(path)
- require.NoError(t, err)
- require.Contains(t, cfg.Artifacts, "202607.02.2")
- require.Contains(t, cfg.Artifacts["202607.02.2"], "linux-ubuntu-22.04-amd64")
- assert.Equal(t, "abc123", cfg.Artifacts["202607.02.2"]["linux-ubuntu-22.04-amd64"].SHA256)
-}
-
-func TestReadHotfixConfig_NoArtifactsFieldBackwardCompat(t *testing.T) {
- path := filepath.Join(t.TempDir(), "hotfix-config.json")
- require.NoError(t, os.WriteFile(path, []byte(`{"version": "202604.01.1"}`), 0o644))
- cfg, err := readHotfixConfig(path)
- require.NoError(t, err)
- assert.Equal(t, "202604.01.1", cfg.Version)
- assert.Nil(t, cfg.Artifacts)
-}
-
-func TestBuildArtifactKey(t *testing.T) {
- t.Run("ubuntu", func(t *testing.T) {
- dir := t.TempDir()
- path := filepath.Join(dir, "os-release")
- require.NoError(t, os.WriteFile(path, []byte("ID=ubuntu\nVERSION_ID=\"22.04\"\n"), 0o644))
- a := &App{osReleasePath: path}
- key, err := a.buildArtifactKey()
- require.NoError(t, err)
- assert.Equal(t, fmt.Sprintf("linux-ubuntu-22.04-%s", runtime.GOARCH), key)
- })
-
- t.Run("azurelinux", func(t *testing.T) {
+func TestRemoveStaleHotfix(t *testing.T) {
+ t.Run("removes the staged binary", func(t *testing.T) {
dir := t.TempDir()
- path := filepath.Join(dir, "os-release")
- require.NoError(t, os.WriteFile(path, []byte("ID=azurelinux\nVERSION_ID=\"3.0\"\n"), 0o644))
- a := &App{osReleasePath: path}
- key, err := a.buildArtifactKey()
- require.NoError(t, err)
- assert.Equal(t, fmt.Sprintf("linux-azurelinux-3.0-%s", runtime.GOARCH), key)
- })
-
- t.Run("missing VERSION_ID errors", func(t *testing.T) {
- dir := t.TempDir()
- path := filepath.Join(dir, "os-release")
- require.NoError(t, os.WriteFile(path, []byte("ID=ubuntu\n"), 0o644))
- a := &App{osReleasePath: path}
- _, err := a.buildArtifactKey()
- require.Error(t, err)
- assert.Contains(t, err.Error(), "VERSION_ID not found")
- })
-}
+ path := filepath.Join(dir, "aks-node-controller-hotfix")
+ require.NoError(t, os.WriteFile(path, []byte("stale"), 0o755))
-func TestValidateArtifactURL(t *testing.T) {
- t.Run("valid PMC URL", func(t *testing.T) {
- assert.NoError(t, validateArtifactURL("https://packages.microsoft.com/ubuntu/22.04/prod/pool/main/a/aks.deb"))
- })
+ app := &App{hotfixBinaryPath: path}
+ app.removeStaleHotfix()
- t.Run("HTTP rejected", func(t *testing.T) {
- err := validateArtifactURL("http://packages.microsoft.com/foo.deb")
- require.Error(t, err)
- assert.True(t, isIntegrityError(err))
- assert.Contains(t, err.Error(), "HTTPS")
+ _, err := os.Stat(path)
+ assert.True(t, os.IsNotExist(err), "stale hotfix binary should be gone")
})
- t.Run("non-PMC host rejected", func(t *testing.T) {
- err := validateArtifactURL("https://evil.com/foo.deb")
- require.Error(t, err)
- assert.True(t, isIntegrityError(err))
- assert.Contains(t, err.Error(), "allowlist")
+ t.Run("absent binary is a no-op", func(t *testing.T) {
+ app := &App{hotfixBinaryPath: filepath.Join(t.TempDir(), "missing")}
+ assert.NotPanics(t, app.removeStaleHotfix)
})
- t.Run("empty URL rejected", func(t *testing.T) {
- err := validateArtifactURL("")
- require.Error(t, err)
- })
-}
-
-func TestDownloadHotfix_ArtifactHTTPSuccess(t *testing.T) {
- origVersion := Version
- Version = "202607.02.0"
- defer func() { Version = origVersion }()
-
- dir := t.TempDir()
- binaryContent := []byte("hotfix-binary-content")
- sha := "3ab698426c19090c43a48950dcd94d196122b11149423f230b1234cda75e3293"
-
- path := filepath.Join(dir, "hotfix-config.json")
- artifactKey := fmt.Sprintf("linux-ubuntu-22.04-%s", runtime.GOARCH)
- configJSON := fmt.Sprintf(`{
- "hotfixes": {"202607.02": "202607.02.2"},
- "artifacts": {
- "202607.02.2": {
- %q: {
- "url": "https://packages.microsoft.com/fake.deb",
- "sha256": %q
- }
- }
- }
- }`, artifactKey, sha)
- require.NoError(t, os.WriteFile(path, []byte(configJSON), 0o644))
-
- osReleasePath := filepath.Join(dir, "os-release")
- require.NoError(t, os.WriteFile(osReleasePath, []byte("ID=ubuntu\nVERSION_ID=\"22.04\"\n"), 0o644))
-
- // Create VHD binary so copyBinaryAlongside can derive permissions.
- vhdBin := filepath.Join(dir, "aks-node-controller")
- require.NoError(t, os.WriteFile(vhdBin, []byte("original"), 0o755))
-
- installCalled := false
- tt := NewTestApp(t, TestAppConfig{
- RunFunc: func(cmd *exec.Cmd) error {
- installCalled = true
- return nil
- },
- })
- tt.App.hotfixVersionPath = path
- tt.App.osReleasePath = osReleasePath
- tt.App.downloadDir = dir
- tt.App.httpDownload = func(ctx context.Context, url string) ([]byte, error) {
- return binaryContent, nil
- }
-
- // copyBinaryAlongside will fail because vhdBinaryPath (/opt/azure/containers/aks-node-controller)
- // doesn't exist in tests. This is treated as a hard failure (integrity error) — no apt fallback.
- err := tt.App.downloadHotfix(context.Background())
- require.Error(t, err)
- assert.Contains(t, err.Error(), "stage hotfix binary from artifact")
- assert.False(t, installCalled, "should use HTTP download, not package manager")
-}
-
-func TestDownloadHotfix_ArtifactSHAMismatchHardFail(t *testing.T) {
- origVersion := Version
- Version = "202607.02.0"
- defer func() { Version = origVersion }()
-
- dir := t.TempDir()
- artifactKey := fmt.Sprintf("linux-ubuntu-22.04-%s", runtime.GOARCH)
-
- path := filepath.Join(dir, "hotfix-config.json")
- configJSON := fmt.Sprintf(`{
- "hotfixes": {"202607.02": "202607.02.2"},
- "artifacts": {
- "202607.02.2": {
- %q: {
- "url": "https://packages.microsoft.com/fake.deb",
- "sha256": "0000000000000000000000000000000000000000000000000000000000000000"
- }
- }
- }
- }`, artifactKey)
- require.NoError(t, os.WriteFile(path, []byte(configJSON), 0o644))
-
- osReleasePath := filepath.Join(dir, "os-release")
- require.NoError(t, os.WriteFile(osReleasePath, []byte("ID=ubuntu\nVERSION_ID=\"22.04\"\n"), 0o644))
-
- installCalled := false
- tt := NewTestApp(t, TestAppConfig{
- RunFunc: func(cmd *exec.Cmd) error {
- installCalled = true
- return nil
- },
- })
- tt.App.hotfixVersionPath = path
- tt.App.osReleasePath = osReleasePath
- tt.App.downloadDir = dir
- tt.App.httpDownload = func(ctx context.Context, url string) ([]byte, error) {
- return []byte("hotfix-binary-content"), nil
- }
-
- err := tt.App.downloadHotfix(context.Background())
- require.Error(t, err)
- assert.Contains(t, err.Error(), "integrity")
- assert.False(t, installCalled, "should NOT fallback to package manager on SHA mismatch")
-}
-
-func TestDownloadHotfix_ArtifactHTTPErrorFallsBackToApt(t *testing.T) {
- origVersion := Version
- Version = "202607.02.0"
- defer func() { Version = origVersion }()
-
- dir := t.TempDir()
- artifactKey := fmt.Sprintf("linux-ubuntu-22.04-%s", runtime.GOARCH)
-
- path := filepath.Join(dir, "hotfix-config.json")
- configJSON := fmt.Sprintf(`{
- "hotfixes": {"202607.02": "202607.02.2"},
- "artifacts": {
- "202607.02.2": {
- %q: {
- "url": "https://packages.microsoft.com/fake.deb",
- "sha256": "abc123"
- }
- }
+ // The launcher selects the hotfix on `[ -x ]` alone and ignores this process's exit
+ // status, so removal failing must not leave the stale binary runnable.
+ t.Run("disarms the binary when removal fails", func(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("root bypasses directory write permissions, so unlink cannot be made to fail")
}
- }`, artifactKey)
- require.NoError(t, os.WriteFile(path, []byte(configJSON), 0o644))
-
- osReleasePath := filepath.Join(dir, "os-release")
- require.NoError(t, os.WriteFile(osReleasePath, []byte("ID=ubuntu\nVERSION_ID=\"22.04\"\n"), 0o644))
-
- aptDir := filepath.Join(dir, "sources.list.d")
- require.NoError(t, os.MkdirAll(aptDir, 0o755))
- require.NoError(t, os.WriteFile(filepath.Join(aptDir, "microsoft-prod.list"), []byte("deb ..."), 0o644))
-
- installCalled := false
- tt := NewTestApp(t, TestAppConfig{
- RunFunc: func(cmd *exec.Cmd) error {
- installCalled = true
- return nil
- },
- })
- tt.App.hotfixVersionPath = path
- tt.App.osReleasePath = osReleasePath
- tt.App.aptSourcesDir = aptDir
- tt.App.downloadDir = dir
- tt.App.httpDownload = func(ctx context.Context, url string) ([]byte, error) {
- return nil, fmt.Errorf("connection refused")
- }
-
- // Will fail at copyBinaryAlongside (pkgBinaryPath doesn't exist), but apt should be called.
- err := tt.App.downloadHotfix(context.Background())
- require.Error(t, err)
- assert.True(t, installCalled, "should fallback to package manager on HTTP network error")
-}
-
-func TestDownloadHotfix_NoArtifactsFallsBackToApt(t *testing.T) {
- origVersion := Version
- Version = "202604.01.0"
- defer func() { Version = origVersion }()
-
- dir := t.TempDir()
- path := filepath.Join(dir, "hotfix-config.json")
- // No artifacts field — legacy config.
- require.NoError(t, os.WriteFile(path, []byte(`{"version": "202604.01.1"}`), 0o644))
-
- osReleasePath := filepath.Join(dir, "os-release")
- require.NoError(t, os.WriteFile(osReleasePath, []byte("ID=ubuntu\n"), 0o644))
-
- aptDir := filepath.Join(dir, "sources.list.d")
- require.NoError(t, os.MkdirAll(aptDir, 0o755))
- require.NoError(t, os.WriteFile(filepath.Join(aptDir, "microsoft-prod.list"), []byte("deb ..."), 0o644))
-
- installCalled := false
- tt := NewTestApp(t, TestAppConfig{
- RunFunc: func(cmd *exec.Cmd) error {
- installCalled = true
- return nil
- },
- })
- tt.App.hotfixVersionPath = path
- tt.App.osReleasePath = osReleasePath
- tt.App.aptSourcesDir = aptDir
-
- // Will fail at copyBinaryAlongside, but apt should be called.
- err := tt.App.downloadHotfix(context.Background())
- require.Error(t, err)
- assert.True(t, installCalled, "should use package manager when no artifacts field")
-}
-
-func TestDownloadHotfix_ArtifactInvalidURLHardFail(t *testing.T) {
- origVersion := Version
- Version = "202607.02.0"
- defer func() { Version = origVersion }()
+ dir := t.TempDir()
+ path := filepath.Join(dir, "aks-node-controller-hotfix")
+ require.NoError(t, os.WriteFile(path, []byte("stale"), 0o755))
- dir := t.TempDir()
- artifactKey := fmt.Sprintf("linux-ubuntu-22.04-%s", runtime.GOARCH)
+ // Unlink needs write permission on the parent directory; chmod on the file does not.
+ require.NoError(t, os.Chmod(dir, 0o500))
+ t.Cleanup(func() { _ = os.Chmod(dir, 0o700) })
- path := filepath.Join(dir, "hotfix-config.json")
- configJSON := fmt.Sprintf(`{
- "hotfixes": {"202607.02": "202607.02.2"},
- "artifacts": {
- "202607.02.2": {
- %q: {
- "url": "http://evil.com/malicious.deb",
- "sha256": "abc123"
- }
- }
- }
- }`, artifactKey)
- require.NoError(t, os.WriteFile(path, []byte(configJSON), 0o644))
+ app := &App{hotfixBinaryPath: path}
+ app.removeStaleHotfix()
- osReleasePath := filepath.Join(dir, "os-release")
- require.NoError(t, os.WriteFile(osReleasePath, []byte("ID=ubuntu\nVERSION_ID=\"22.04\"\n"), 0o644))
-
- installCalled := false
- tt := NewTestApp(t, TestAppConfig{
- RunFunc: func(cmd *exec.Cmd) error {
- installCalled = true
- return nil
- },
+ info, err := os.Stat(path)
+ require.NoError(t, err, "removal was expected to fail, leaving the file in place")
+ assert.Zero(t, info.Mode().Perm()&0o111, "stale hotfix binary must not remain executable")
})
- tt.App.hotfixVersionPath = path
- tt.App.osReleasePath = osReleasePath
-
- err := tt.App.downloadHotfix(context.Background())
- require.Error(t, err)
- assert.Contains(t, err.Error(), "integrity")
- assert.False(t, installCalled, "should NOT fallback to package manager on invalid URL")
-}
-
-func TestDownloadAndVerify_Success(t *testing.T) {
- dir := t.TempDir()
- binaryContent := []byte("hotfix-binary-content")
- sha := "3ab698426c19090c43a48950dcd94d196122b11149423f230b1234cda75e3293"
-
- a := &App{
- downloadDir: dir,
- httpDownload: func(ctx context.Context, url string) ([]byte, error) {
- return binaryContent, nil
- },
- }
-
- tmpPath, err := a.downloadAndVerify(context.Background(),
- "https://packages.microsoft.com/test-binary", sha)
- require.NoError(t, err)
- defer os.Remove(tmpPath)
-
- // Verify the staged file has the correct content.
- data, err := os.ReadFile(tmpPath)
- require.NoError(t, err)
- assert.Equal(t, binaryContent, data)
-}
-
-func TestDownloadAndVerify_SHAMismatch(t *testing.T) {
- dir := t.TempDir()
-
- a := &App{
- downloadDir: dir,
- httpDownload: func(ctx context.Context, url string) ([]byte, error) {
- return []byte("tampered-content"), nil
- },
- }
-
- _, err := a.downloadAndVerify(context.Background(),
- "https://packages.microsoft.com/test-binary",
- "0000000000000000000000000000000000000000000000000000000000000000")
- require.Error(t, err)
- assert.True(t, isIntegrityError(err))
- assert.Contains(t, err.Error(), "SHA-256 mismatch")
-
- // Verify no temp files left behind.
- entries, err := os.ReadDir(dir)
- require.NoError(t, err)
- for _, e := range entries {
- assert.False(t, strings.HasPrefix(e.Name(), ".aks-node-controller-download-"),
- "temp file should be cleaned up on SHA mismatch: %s", e.Name())
- }
-}
-
-func TestDownloadAndVerify_HTTPError(t *testing.T) {
- dir := t.TempDir()
-
- a := &App{
- downloadDir: dir,
- httpDownload: func(ctx context.Context, url string) ([]byte, error) {
- return nil, fmt.Errorf("connection timeout")
- },
- }
-
- _, err := a.downloadAndVerify(context.Background(),
- "https://packages.microsoft.com/test-binary", "abc123")
- require.Error(t, err)
- assert.False(t, isIntegrityError(err))
- assert.Contains(t, err.Error(), "connection timeout")
-
- // Verify no temp files left behind.
- entries, err := os.ReadDir(dir)
- require.NoError(t, err)
- for _, e := range entries {
- assert.False(t, strings.HasPrefix(e.Name(), ".aks-node-controller-download-"),
- "temp file should be cleaned up on HTTP error: %s", e.Name())
- }
}
diff --git a/aks-node-controller/repository_hotfix.go b/aks-node-controller/repository_hotfix.go
new file mode 100644
index 00000000000..3d6600a023f
--- /dev/null
+++ b/aks-node-controller/repository_hotfix.go
@@ -0,0 +1,1691 @@
+package main
+
+import (
+ "archive/tar"
+ "bufio"
+ "bytes"
+ "compress/gzip"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/xml"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "os/exec"
+ pathpkg "path"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/Azure/agentbaker/aks-node-controller/common"
+)
+
+const (
+ defaultYumReposDir = "/etc/yum.repos.d"
+ defaultAptTrustedKeyringsDir = "/etc/apt/trusted.gpg.d"
+ repositoryRequestTimeout = 30 * time.Second
+ repositoryMetadataMaxBytes = 128 << 20
+ repositoryPackageMaxBytes = 512 << 20
+ repositoryBinaryMaxBytes = 128 << 20
+ repositoryCommandTimeout = 60 * time.Second
+ ancPackageName = "aks-node-controller"
+ ancPackageBinaryRelativePath = "usr/bin/aks-node-controller"
+
+ archAMD64 = "amd64"
+ archARM64 = "arm64"
+
+ osIDAzureLinux = "azurelinux"
+ osIDMariner = "mariner"
+)
+
+type integrityError struct {
+ msg string
+}
+
+func isSHA256Hex(value string) bool {
+ if len(value) != sha256.Size*2 {
+ return false
+ }
+ _, err := hex.DecodeString(value)
+ return err == nil
+}
+
+func (e *integrityError) Error() string { return e.msg }
+
+func newIntegrityError(format string, args ...any) error {
+ return &integrityError{msg: fmt.Sprintf(format, args...)}
+}
+
+func isIntegrityError(err error) bool {
+ var target *integrityError
+ return errors.As(err, &target)
+}
+
+type unsupportedRepositoryError struct {
+ msg string
+}
+
+func (e *unsupportedRepositoryError) Error() string { return e.msg }
+
+func newUnsupportedRepositoryError(format string, args ...any) error {
+ return &unsupportedRepositoryError{msg: fmt.Sprintf(format, args...)}
+}
+
+type downloadedRepositoryFile struct {
+ path string
+ sha256 string
+ size int64
+}
+
+type repositoryPackageMetadata struct {
+ sha256 string
+}
+
+type repositoryDownloadPlan struct {
+ format string
+ packageURL string
+ trustedOrigin *url.URL
+ resolveMetadata func(context.Context) (repositoryPackageMetadata, error)
+}
+
+// fetchPackageAndMetadata downloads the package and resolves its authenticated metadata
+// concurrently, cancelling the peer as soon as either fails. Without that cancellation a
+// fast failure (e.g. a 404 on the package) still waits out the other branch -- gpgv's 60s
+// command timeout plus a 30s metadata request -- before the package-manager fallback can
+// start, directly extending node provisioning.
+//
+// Cancellation makes error classification load-bearing. downloadBinaryHotfixIfNeeded
+// treats integrity errors as terminal: it disarms the staged hotfix and skips the
+// fallback. A cancelled peer must therefore never be reported as integrity, or a benign
+// 404 would masquerade as tampering. Conversely, a real integrity error from either
+// branch must outrank operational failures, even if the operational failure triggered
+// cancellation first.
+//
+// The returned file is the caller's to remove, including on the error paths.
+func (a *App) fetchPackageAndMetadata(
+ ctx context.Context,
+ plan repositoryDownloadPlan,
+) (downloadedRepositoryFile, repositoryPackageMetadata, error) {
+ branchCtx, cancelBranches := context.WithCancel(ctx)
+ defer cancelBranches()
+
+ var (
+ packageFile downloadedRepositoryFile
+ metadata repositoryPackageMetadata
+ packageErr error
+ metadataErr error
+ cancelOnce sync.Once
+ wg sync.WaitGroup
+ )
+ failBranch := func(err error) {
+ if err == nil {
+ return
+ }
+ cancelOnce.Do(cancelBranches)
+ }
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ packageFile, packageErr = a.downloadRepositoryFile(
+ branchCtx, plan.packageURL, plan.trustedOrigin, repositoryPackageMaxBytes)
+ failBranch(packageErr)
+ }()
+ go func() {
+ defer wg.Done()
+ metadata, metadataErr = plan.resolveMetadata(branchCtx)
+ failBranch(metadataErr)
+ }()
+ wg.Wait()
+
+ // A dead caller context outranks whichever branch happened to notice it first.
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return packageFile, metadata, fmt.Errorf("repository fast path cancelled: %w", ctxErr)
+ }
+ if err := preferredRepositoryDownloadError(packageErr, metadataErr); err != nil {
+ return packageFile, metadata, err
+ }
+ return packageFile, metadata, nil
+}
+
+func preferredRepositoryDownloadError(packageErr, metadataErr error) error {
+ branches := []struct {
+ label string
+ err error
+ }{
+ {label: "download repository package", err: packageErr},
+ {label: "resolve authenticated repository metadata", err: metadataErr},
+ }
+ for _, branch := range branches {
+ if branch.err != nil && !isRepositoryCancellationError(branch.err) && isIntegrityError(branch.err) {
+ return fmt.Errorf("%s: %w", branch.label, branch.err)
+ }
+ }
+ for _, branch := range branches {
+ if branch.err != nil && !isRepositoryCancellationError(branch.err) {
+ return fmt.Errorf("%s: %w", branch.label, branch.err)
+ }
+ }
+ for _, branch := range branches {
+ if branch.err != nil {
+ return fmt.Errorf("%s: %w", branch.label, branch.err)
+ }
+ }
+ return nil
+}
+
+func isRepositoryCancellationError(err error) bool {
+ return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
+}
+
+func (a *App) tryRepositoryDownload(ctx context.Context, hotfixVersion string) error {
+ start := time.Now()
+ info, err := a.parseLinuxPlatformInfo()
+ if err != nil {
+ return newUnsupportedRepositoryError("determine platform: %v", err)
+ }
+
+ var plan repositoryDownloadPlan
+ switch info.ID {
+ case "ubuntu":
+ plan, err = a.ubuntuRepositoryPlan(info, hotfixVersion)
+ case osIDAzureLinux, osIDMariner:
+ plan, err = a.rpmRepositoryPlan(info, hotfixVersion)
+ default:
+ err = newUnsupportedRepositoryError("unsupported repository platform %q", info.ID)
+ }
+ if err != nil {
+ return err
+ }
+
+ packageFile, metadata, err := a.fetchPackageAndMetadata(ctx, plan)
+ if packageFile.path != "" {
+ defer os.Remove(packageFile.path)
+ }
+ if err != nil {
+ return err
+ }
+ if !strings.EqualFold(packageFile.sha256, metadata.sha256) {
+ return newIntegrityError("package SHA-256 mismatch: expected %s, got %s",
+ metadata.sha256, packageFile.sha256)
+ }
+
+ extractDir, err := os.MkdirTemp(a.repositoryStagingDir(), ".aks-node-controller-extract-*")
+ if err != nil {
+ return fmt.Errorf("create package extraction directory: %w", err)
+ }
+ defer os.RemoveAll(extractDir)
+
+ if err := a.extractPackage(ctx, plan.format, packageFile.path, extractDir); err != nil {
+ return fmt.Errorf("extract authenticated %s package: %w", plan.format, err)
+ }
+ extractedBinary := filepath.Join(extractDir, filepath.FromSlash(ancPackageBinaryRelativePath))
+ if err := copyBinaryAlongside(extractedBinary, a.hotfixPath(), a.vhdPath()); err != nil {
+ return fmt.Errorf("stage extracted ANC binary: %w", err)
+ }
+
+ // durationMs makes the fast path measurable in the field against the package-manager
+ // path, which is the whole reason this code exists.
+ slog.Info("downloaded ANC hotfix through authenticated repository fast path",
+ "target", hotfixVersion, "format", plan.format, "path", a.hotfixPath(),
+ "durationMs", time.Since(start).Milliseconds())
+ return nil
+}
+
+func (a *App) repositoryStagingDir() string {
+ if a.repositoryTempDir != "" {
+ return a.repositoryTempDir
+ }
+ return filepath.Dir(a.hotfixPath())
+}
+
+func validateRepositoryURL(rawURL string) (*url.URL, error) {
+ u, err := url.Parse(rawURL)
+ if err != nil {
+ return nil, newUnsupportedRepositoryError("parse repository URL %q: %v", rawURL, err)
+ }
+ if u.User != nil || u.Host == "" || (u.Scheme != "https" && u.Scheme != "http") {
+ return nil, newUnsupportedRepositoryError("repository URL %q is not an HTTP(S) origin", rawURL)
+ }
+ if u.Scheme == "http" && !isTrustedLocalRepositoryHost(u.Hostname()) {
+ return nil, newUnsupportedRepositoryError(
+ "plain HTTP repository %q is not a trusted local source", u.Hostname())
+ }
+ u.Fragment = ""
+ return u, nil
+}
+
+func asRepositoryBase(u *url.URL) *url.URL {
+ base := *u
+ if !strings.HasSuffix(base.Path, "/") {
+ base.Path += "/"
+ }
+ return &base
+}
+
+func isTrustedLocalRepositoryHost(host string) bool {
+ if strings.EqualFold(host, "localhost") {
+ return true
+ }
+ ip := net.ParseIP(strings.Trim(host, "[]"))
+ return ip != nil && (ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast())
+}
+
+func sameRepositoryOrigin(a, b *url.URL) bool {
+ return strings.EqualFold(a.Scheme, b.Scheme) && strings.EqualFold(a.Host, b.Host)
+}
+
+func isWithinRepositoryBase(base, candidate *url.URL) bool {
+ return sameRepositoryOrigin(base, candidate) &&
+ strings.HasPrefix(candidate.EscapedPath(), base.EscapedPath())
+}
+
+func resolveRepositoryURL(base *url.URL, relative string) (string, error) {
+ ref, err := url.Parse(relative)
+ if err != nil {
+ return "", newUnsupportedRepositoryError("parse repository-relative URL %q: %v", relative, err)
+ }
+ if ref.IsAbs() || ref.Host != "" || strings.HasPrefix(ref.Path, "/") ||
+ ref.RawQuery != "" || ref.Fragment != "" {
+ return "", newIntegrityError("repository path is not relative: %q", relative)
+ }
+ cleaned := pathpkg.Clean(ref.Path)
+ if cleaned == ".." || strings.HasPrefix(cleaned, "../") {
+ return "", newIntegrityError("repository path escapes configured base: %q", relative)
+ }
+ ref.Path = cleaned
+ resolved := base.ResolveReference(ref)
+ if !isWithinRepositoryBase(base, resolved) {
+ return "", newIntegrityError("repository metadata escaped configured base path: %q", relative)
+ }
+ return resolved.String(), nil
+}
+
+func (a *App) downloadRepositoryFile(
+ ctx context.Context,
+ rawURL string,
+ trustedOrigin *url.URL,
+ maxBytes int64,
+) (downloadedRepositoryFile, error) {
+ u, err := validateRepositoryURL(rawURL)
+ if err != nil {
+ return downloadedRepositoryFile{}, err
+ }
+ if trustedOrigin == nil || !sameRepositoryOrigin(trustedOrigin, u) {
+ return downloadedRepositoryFile{}, newIntegrityError("download URL is outside configured repository origin: %s", rawURL)
+ }
+
+ if err = os.MkdirAll(a.repositoryStagingDir(), 0o755); err != nil {
+ return downloadedRepositoryFile{}, fmt.Errorf("create repository staging directory: %w", err)
+ }
+ tmp, err := os.CreateTemp(a.repositoryStagingDir(), ".aks-node-controller-repository-*")
+ if err != nil {
+ return downloadedRepositoryFile{}, fmt.Errorf("create repository temp file: %w", err)
+ }
+ tmpPath := tmp.Name()
+ success := false
+ defer func() {
+ _ = tmp.Close()
+ if !success {
+ _ = os.Remove(tmpPath)
+ }
+ }()
+
+ requestCtx, cancel := context.WithTimeout(ctx, repositoryRequestTimeout)
+ defer cancel()
+ transport := common.NewBaseTransport(common.HTTPTransportOptions{
+ DialTimeout: 10 * time.Second,
+ TLSHandshakeTimeout: 10 * time.Second,
+ ResponseHeaderTimeout: 10 * time.Second,
+ })
+ transport.DisableCompression = true
+ defer transport.CloseIdleConnections()
+ client := &http.Client{
+ Transport: transport,
+ CheckRedirect: func(req *http.Request, _ []*http.Request) error {
+ redirectURL, redirectErr := validateRepositoryURL(req.URL.String())
+ if redirectErr != nil {
+ return redirectErr
+ }
+ if !isWithinRepositoryBase(trustedOrigin, redirectURL) {
+ return fmt.Errorf("redirect outside configured repository base")
+ }
+ return nil
+ },
+ }
+ req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, u.String(), nil)
+ if err != nil {
+ return downloadedRepositoryFile{}, fmt.Errorf("create GET request: %w", err)
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ return downloadedRepositoryFile{}, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return downloadedRepositoryFile{}, fmt.Errorf("HTTP %d from %s", resp.StatusCode, u.Redacted())
+ }
+
+ hasher := sha256.New()
+ written, err := io.Copy(io.MultiWriter(tmp, hasher), io.LimitReader(resp.Body, maxBytes+1))
+ if err != nil {
+ return downloadedRepositoryFile{}, fmt.Errorf("stream %s: %w", u.Redacted(), err)
+ }
+ if written > maxBytes {
+ return downloadedRepositoryFile{}, fmt.Errorf("repository response exceeds %d bytes", maxBytes)
+ }
+ if err := tmp.Close(); err != nil {
+ return downloadedRepositoryFile{}, fmt.Errorf("close repository temp file: %w", err)
+ }
+ success = true
+ return downloadedRepositoryFile{
+ path: tmpPath,
+ sha256: hex.EncodeToString(hasher.Sum(nil)),
+ size: written,
+ }, nil
+}
+
+func (a *App) verifyRepoSignature(
+ ctx context.Context,
+ signedPath string,
+ signaturePath string,
+ keyrings []string,
+) error {
+ if len(keyrings) == 0 {
+ return newUnsupportedRepositoryError("repository has no configured signing key")
+ }
+ if a.verifyRepositorySignature != nil {
+ if err := a.verifyRepositorySignature(ctx, signedPath, signaturePath, keyrings); err != nil {
+ return newIntegrityError("repository signature verification failed: %v", err)
+ }
+ return nil
+ }
+
+ preparedKeyrings, cleanup, err := a.prepareGPGVKeyrings(ctx, keyrings)
+ if err != nil {
+ return err
+ }
+ defer cleanup()
+
+ args := make([]string, 0, 2*len(preparedKeyrings)+2)
+ for _, keyring := range preparedKeyrings {
+ args = append(args, "--keyring", keyring)
+ }
+ if signaturePath != "" {
+ args = append(args, signaturePath)
+ }
+ args = append(args, signedPath)
+ if err := a.runRepositoryCommand(ctx, "gpgv", args...); err != nil {
+ if errors.Is(err, exec.ErrNotFound) {
+ return newUnsupportedRepositoryError("gpgv is not installed: %v", err)
+ }
+ if isRepositoryCancellationError(err) {
+ return err
+ }
+ return newIntegrityError("gpgv verification failed: %v", err)
+ }
+ return nil
+}
+
+func (a *App) prepareGPGVKeyrings(
+ ctx context.Context,
+ keyrings []string,
+) ([]string, func(), error) {
+ var prepared []string
+ var temporary []string
+ cleanup := func() {
+ for _, path := range temporary {
+ _ = os.Remove(path)
+ }
+ }
+ for _, keyring := range keyrings {
+ data, err := os.ReadFile(keyring)
+ if err != nil {
+ cleanup()
+ return nil, func() {}, newUnsupportedRepositoryError(
+ "read repository keyring %s: %v", keyring, err)
+ }
+ if !bytes.Contains(data, []byte("-----BEGIN PGP PUBLIC KEY BLOCK-----")) {
+ prepared = append(prepared, keyring)
+ continue
+ }
+ output, err := os.CreateTemp(a.repositoryStagingDir(), ".aks-node-controller-keyring-*.gpg")
+ if err != nil {
+ cleanup()
+ return nil, func() {}, fmt.Errorf("create repository keyring temp file: %w", err)
+ }
+ outputPath := output.Name()
+ if err := output.Close(); err != nil {
+ _ = os.Remove(outputPath)
+ cleanup()
+ return nil, func() {}, fmt.Errorf("close repository keyring temp file: %w", err)
+ }
+ temporary = append(temporary, outputPath)
+ if err := a.runRepositoryCommand(
+ ctx, "gpg", "--batch", "--yes", "--dearmor", "--output", outputPath, keyring,
+ ); err != nil {
+ cleanup()
+ return nil, func() {}, newUnsupportedRepositoryError(
+ "dearmor repository key %s: %v", keyring, err)
+ }
+ prepared = append(prepared, outputPath)
+ }
+ return prepared, cleanup, nil
+}
+
+func (a *App) runRepositoryCommand(ctx context.Context, name string, args ...string) error {
+ commandCtx, cancel := context.WithTimeout(ctx, repositoryCommandTimeout)
+ defer cancel()
+ cmd := exec.CommandContext(commandCtx, name, args...)
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ if err := a.cmdRun(cmd); err != nil {
+ if commandCtx.Err() != nil {
+ return commandCtx.Err()
+ }
+ return err
+ }
+ return nil
+}
+
+func (a *App) extractPackage(ctx context.Context, format, packagePath, destination string) error {
+ if a.extractRepositoryPackage != nil {
+ return a.extractRepositoryPackage(ctx, format, packagePath, destination)
+ }
+ switch format {
+ case "deb":
+ return a.extractDeb(ctx, packagePath, destination)
+ case "rpm":
+ return a.extractRPM(ctx, packagePath, destination)
+ default:
+ return newUnsupportedRepositoryError("unsupported package format %q", format)
+ }
+}
+
+func (a *App) extractDeb(ctx context.Context, packagePath, destination string) error {
+ commandCtx, cancel := context.WithTimeout(ctx, repositoryCommandTimeout)
+ defer cancel()
+
+ dpkgDeb := exec.CommandContext(commandCtx, "dpkg-deb", "--fsys-tarfile", packagePath)
+ dpkgDeb.Stderr = os.Stderr
+ stdout, err := dpkgDeb.StdoutPipe()
+ if err != nil {
+ return fmt.Errorf("create dpkg-deb stdout pipe: %w", err)
+ }
+ if err := dpkgDeb.Start(); err != nil {
+ return fmt.Errorf("start dpkg-deb: %w", err)
+ }
+
+ found, extractErr := extractRepositoryTarMember(stdout, destination)
+ if found {
+ cancel()
+ }
+ waitErr := dpkgDeb.Wait()
+ if extractErr != nil {
+ return fmt.Errorf("extract deb package member: %w", extractErr)
+ }
+ if !found {
+ if commandCtx.Err() != nil {
+ return commandCtx.Err()
+ }
+ if waitErr != nil {
+ return fmt.Errorf("dpkg-deb: %w", waitErr)
+ }
+ return fmt.Errorf("deb package does not contain %s", ancPackageBinaryRelativePath)
+ }
+ return nil
+}
+
+func extractRepositoryTarMember(tarStream io.Reader, destination string) (bool, error) {
+ reader := tar.NewReader(tarStream)
+ for {
+ header, err := reader.Next()
+ if errors.Is(err, io.EOF) {
+ return false, nil
+ }
+ if err != nil {
+ return false, err
+ }
+ if !isANCBinaryTarMember(header.Name) {
+ continue
+ }
+ return true, extractRepositoryANCBinary(reader, header, destination)
+ }
+}
+
+func isANCBinaryTarMember(name string) bool {
+ return strings.TrimPrefix(name, "./") == ancPackageBinaryRelativePath
+}
+
+func extractRepositoryANCBinary(reader io.Reader, header *tar.Header, destination string) error {
+ if header.Typeflag != tar.TypeReg && header.Typeflag != 0 {
+ return newIntegrityError("deb package member %s is not a regular file", header.Name)
+ }
+ if header.Size > repositoryBinaryMaxBytes {
+ return newIntegrityError("deb package member %s exceeds %d bytes", header.Name, repositoryBinaryMaxBytes)
+ }
+ outputPath := filepath.Join(destination, filepath.FromSlash(ancPackageBinaryRelativePath))
+ if mkdirErr := os.MkdirAll(filepath.Dir(outputPath), 0o755); mkdirErr != nil {
+ return fmt.Errorf("create extraction directory: %w", mkdirErr)
+ }
+ tmp, createErr := os.CreateTemp(filepath.Dir(outputPath), ".aks-node-controller-extract-*")
+ if createErr != nil {
+ return fmt.Errorf("create extracted binary temp file: %w", createErr)
+ }
+ tmpPath := tmp.Name()
+ success := false
+ defer func() {
+ _ = tmp.Close()
+ if !success {
+ _ = os.Remove(tmpPath)
+ }
+ }()
+ copied, copyErr := io.Copy(tmp, io.LimitReader(reader, repositoryBinaryMaxBytes+1))
+ if copyErr != nil {
+ return fmt.Errorf("copy extracted binary: %w", copyErr)
+ }
+ if copied > repositoryBinaryMaxBytes {
+ return newIntegrityError("deb package member %s exceeds %d bytes", header.Name, repositoryBinaryMaxBytes)
+ }
+ if header.Size >= 0 && copied != header.Size {
+ return fmt.Errorf("short read extracting %s: copied %d of %d bytes", header.Name, copied, header.Size)
+ }
+ if chmodErr := tmp.Chmod(0o755); chmodErr != nil {
+ return fmt.Errorf("chmod extracted binary: %w", chmodErr)
+ }
+ if closeErr := tmp.Close(); closeErr != nil {
+ return fmt.Errorf("close extracted binary: %w", closeErr)
+ }
+ if renameErr := os.Rename(tmpPath, outputPath); renameErr != nil {
+ return fmt.Errorf("rename extracted binary: %w", renameErr)
+ }
+ success = true
+ return nil
+}
+
+func (a *App) extractRPM(ctx context.Context, packagePath, destination string) error {
+ commandCtx, cancel := context.WithTimeout(ctx, repositoryCommandTimeout)
+ defer cancel()
+
+ reader, writer, err := os.Pipe()
+ if err != nil {
+ return fmt.Errorf("create rpm extraction pipe: %w", err)
+ }
+ defer reader.Close()
+ defer writer.Close()
+
+ rpm2cpio := exec.CommandContext(commandCtx, "rpm2cpio", packagePath)
+ rpm2cpio.Stdout = writer
+ rpm2cpio.Stderr = os.Stderr
+ cpio := exec.CommandContext(commandCtx, "cpio", "-idmu", "--quiet", "./usr/bin/aks-node-controller")
+ cpio.Dir = destination
+ cpio.Stdin = reader
+ cpio.Stdout = os.Stdout
+ cpio.Stderr = os.Stderr
+
+ cpioErrCh := make(chan error, 1)
+ go func() {
+ cpioErrCh <- a.cmdRun(cpio)
+ }()
+ rpmErr := a.cmdRun(rpm2cpio)
+ _ = writer.Close()
+ cpioErr := <-cpioErrCh
+ if err := preferredRPMExtractionError(commandCtx.Err(), rpmErr, cpioErr); err != nil {
+ return err
+ }
+ return nil
+}
+
+func preferredRPMExtractionError(ctxErr, rpmErr, cpioErr error) error {
+ var errs []error
+ if ctxErr != nil {
+ errs = append(errs, fmt.Errorf("rpm extraction cancelled: %w", ctxErr))
+ }
+ if rpmErr != nil && !isRepositoryCancellationError(rpmErr) {
+ errs = append(errs, fmt.Errorf("rpm2cpio: %w", rpmErr))
+ }
+ if cpioErr != nil && !isRepositoryCancellationError(cpioErr) {
+ errs = append(errs, fmt.Errorf("cpio: %w", cpioErr))
+ }
+ if len(errs) > 0 {
+ return errors.Join(errs...)
+ }
+ if rpmErr != nil {
+ errs = append(errs, fmt.Errorf("rpm2cpio: %w", rpmErr))
+ }
+ if cpioErr != nil {
+ errs = append(errs, fmt.Errorf("cpio: %w", cpioErr))
+ }
+ return errors.Join(errs...)
+}
+
+type aptRepository struct {
+ URI string
+ Suite string
+ Component string
+ SignedBy []string
+ SourcePath string
+}
+
+func (a *App) ubuntuRepositoryPlan(info platformInfo, hotfixVersion string) (repositoryDownloadPlan, error) {
+ debArch, err := debArchitecture(info.Arch)
+ if err != nil {
+ return repositoryDownloadPlan{}, err
+ }
+ sourcesDir := a.aptSourcesDir
+ if sourcesDir == "" {
+ sourcesDir = defaultAptSourcesDir
+ }
+ sourcePath, err := resolveMicrosoftProdSourceListPath(sourcesDir)
+ if err != nil {
+ return repositoryDownloadPlan{}, newUnsupportedRepositoryError("%v", err)
+ }
+ repository, err := parseAptRepositoryFile(sourcePath, debArch)
+ if err != nil {
+ return repositoryDownloadPlan{}, err
+ }
+ if len(repository.SignedBy) == 0 {
+ keyringsDir := a.aptTrustedKeyringsDir
+ if keyringsDir == "" {
+ keyringsDir = defaultAptTrustedKeyringsDir
+ }
+ repository.SignedBy, err = microsoftAptTrustedKeyrings(keyringsDir)
+ if err != nil {
+ return repositoryDownloadPlan{}, err
+ }
+ }
+ origin, err := validateRepositoryURL(repository.URI)
+ if err != nil {
+ return repositoryDownloadPlan{}, err
+ }
+ origin = asRepositoryBase(origin)
+ if info.VersionID == "" {
+ return repositoryDownloadPlan{}, newUnsupportedRepositoryError("Ubuntu VERSION_ID is empty")
+ }
+
+ fullVersion := hotfixVersion + "-ubuntu" + info.VersionID + "u1"
+ relativePackagePath := fmt.Sprintf(
+ "pool/main/a/%s/%s_%s_%s.deb", ancPackageName, ancPackageName, fullVersion, debArch)
+ packageURL, err := resolveRepositoryURL(origin, relativePackagePath)
+ if err != nil {
+ return repositoryDownloadPlan{}, err
+ }
+
+ return repositoryDownloadPlan{
+ format: "deb",
+ packageURL: packageURL,
+ trustedOrigin: origin,
+ resolveMetadata: func(ctx context.Context) (repositoryPackageMetadata, error) {
+ return a.resolveUbuntuPackageMetadata(
+ ctx, origin, repository, debArch, fullVersion, relativePackagePath)
+ },
+ }, nil
+}
+
+func debArchitecture(goarch string) (string, error) {
+ switch goarch {
+ case archAMD64:
+ return archAMD64, nil
+ case archARM64:
+ return archARM64, nil
+ default:
+ return "", newUnsupportedRepositoryError("unsupported Debian architecture %q", goarch)
+ }
+}
+
+func parseAptRepositoryFile(path, arch string) (aptRepository, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return aptRepository{}, newUnsupportedRepositoryError("read apt source %s: %v", path, err)
+ }
+ if strings.HasSuffix(path, ".sources") {
+ return parseDeb822Repository(string(data), path, arch)
+ }
+ return parseOneLineAptRepository(string(data), path, arch)
+}
+
+// parseAptLineOptions parses the optional bracketed option group of a one-line apt
+// entry (e.g. `[arch=amd64 signed-by=/path/key.gpg]`). It returns the parsed options
+// and the index of the first field after the group. The bool is false when the group is
+// opened but never closed, in which case the caller must skip the line.
+func parseAptLineOptions(fields []string, index int) (map[string]string, int, bool, error) {
+ options := map[string]string{}
+ if !strings.HasPrefix(fields[index], "[") {
+ return options, index, true, nil
+ }
+ end := index
+ for end < len(fields) && !strings.HasSuffix(fields[end], "]") {
+ end++
+ }
+ if end >= len(fields) {
+ return nil, 0, false, nil
+ }
+ optionText := strings.Trim(strings.Join(fields[index:end+1], " "), "[]")
+ for _, option := range strings.Fields(optionText) {
+ keyValue := strings.SplitN(option, "=", 2)
+ if len(keyValue) != 2 {
+ return nil, 0, false, newUnsupportedRepositoryError(
+ "APT source option contains unsupported non-key-value constraint %q", option)
+ }
+ options[strings.ToLower(keyValue[0])] = keyValue[1]
+ }
+ return options, end + 1, true, nil
+}
+
+// parseOneLineAptEntry parses a single non-comment one-line apt entry. The bool is
+// false when the line is not a usable deb entry for arch and must be skipped.
+func parseOneLineAptEntry(line, path, arch string) (aptRepository, bool, error) {
+ if line == "" || !strings.HasPrefix(line, "deb ") {
+ return aptRepository{}, false, nil
+ }
+ fields := strings.Fields(line)
+ if len(fields) < 4 {
+ return aptRepository{}, false, nil
+ }
+ options, index, ok, err := parseAptLineOptions(fields, 1)
+ if err != nil {
+ return aptRepository{}, false, err
+ }
+ if !ok || len(fields) < index+3 {
+ return aptRepository{}, false, nil
+ }
+ if configuredArch := options["arch"]; configuredArch != "" &&
+ !containsString(strings.Split(configuredArch, ","), arch) {
+ return aptRepository{}, false, nil
+ }
+ signedBy, err := splitConfiguredPaths(options["signed-by"])
+ if err != nil {
+ return aptRepository{}, false, err
+ }
+ return aptRepository{
+ URI: fields[index],
+ Suite: fields[index+1],
+ Component: fields[index+2],
+ SignedBy: signedBy,
+ SourcePath: path,
+ }, true, nil
+}
+
+func parseOneLineAptRepository(contents, path, arch string) (aptRepository, error) {
+ for _, rawLine := range strings.Split(contents, "\n") {
+ line := strings.TrimSpace(strings.SplitN(rawLine, "#", 2)[0])
+ repo, ok, err := parseOneLineAptEntry(line, path, arch)
+ if err != nil {
+ return aptRepository{}, err
+ }
+ if ok {
+ return repo, nil
+ }
+ }
+ return aptRepository{}, newUnsupportedRepositoryError("no usable deb entry in %s", path)
+}
+
+func parseDeb822Repository(contents, path, arch string) (aptRepository, error) {
+ for _, paragraph := range splitParagraphs(contents) {
+ fields := parseDeb822Fields(paragraph)
+ if !containsString(strings.Fields(fields["types"]), "deb") ||
+ strings.EqualFold(strings.TrimSpace(fields["enabled"]), "no") {
+ continue
+ }
+ architectures := strings.Fields(fields["architectures"])
+ if len(architectures) > 0 && !containsString(architectures, arch) {
+ continue
+ }
+ uris := strings.Fields(fields["uris"])
+ suites := strings.Fields(fields["suites"])
+ components := strings.Fields(fields["components"])
+ signedBy, err := splitConfiguredPaths(fields["signed-by"])
+ if err != nil {
+ return aptRepository{}, err
+ }
+ if len(uris) == 0 || len(suites) == 0 || len(components) == 0 {
+ continue
+ }
+ return aptRepository{
+ URI: uris[0],
+ Suite: suites[0],
+ Component: components[0],
+ SignedBy: signedBy,
+ SourcePath: path,
+ }, nil
+ }
+ return aptRepository{}, newUnsupportedRepositoryError("no usable deb822 entry in %s", path)
+}
+
+func microsoftAptTrustedKeyrings(dir string) ([]string, error) {
+ paths, err := filepath.Glob(filepath.Join(dir, "microsoft*.gpg"))
+ if err != nil {
+ return nil, newUnsupportedRepositoryError("scan Microsoft APT keyrings: %v", err)
+ }
+ if len(paths) == 0 {
+ return nil, newUnsupportedRepositoryError(
+ "APT source has no Signed-By and %s has no Microsoft keyring", dir)
+ }
+ return paths, nil
+}
+
+func splitParagraphs(contents string) []string {
+ contents = strings.ReplaceAll(contents, "\r\n", "\n")
+ var paragraphs []string
+ var current []string
+ for _, line := range strings.Split(contents, "\n") {
+ if strings.TrimSpace(line) == "" {
+ if len(current) > 0 {
+ paragraphs = append(paragraphs, strings.Join(current, "\n"))
+ current = nil
+ }
+ continue
+ }
+ current = append(current, line)
+ }
+ if len(current) > 0 {
+ paragraphs = append(paragraphs, strings.Join(current, "\n"))
+ }
+ return paragraphs
+}
+
+func parseDeb822Fields(paragraph string) map[string]string {
+ fields := map[string]string{}
+ var currentKey string
+ for _, line := range strings.Split(paragraph, "\n") {
+ if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") {
+ if currentKey != "" {
+ fields[currentKey] += " " + strings.TrimSpace(line)
+ }
+ continue
+ }
+ keyValue := strings.SplitN(line, ":", 2)
+ if len(keyValue) != 2 {
+ currentKey = ""
+ continue
+ }
+ currentKey = strings.ToLower(strings.TrimSpace(keyValue[0]))
+ fields[currentKey] = strings.TrimSpace(keyValue[1])
+ }
+ return fields
+}
+
+func splitConfiguredPaths(value string) ([]string, error) {
+ var paths []string
+ for _, field := range strings.Fields(strings.ReplaceAll(value, ",", " ")) {
+ if !strings.HasPrefix(field, "/") {
+ return nil, newUnsupportedRepositoryError("APT Signed-By contains unsupported non-path constraint %q", field)
+ }
+ paths = append(paths, field)
+ }
+ return paths, nil
+}
+
+func containsString(values []string, target string) bool {
+ for _, value := range values {
+ if strings.EqualFold(strings.TrimSpace(value), target) {
+ return true
+ }
+ }
+ return false
+}
+
+// decompressGzipToTemp expands a gzip file into a new temp file under the staging dir,
+// bounded by repositoryMetadataMaxBytes so a decompression bomb cannot exhaust the disk.
+// The caller owns the returned path.
+func (a *App) decompressGzipToTemp(compressedPath, namePattern string) (string, error) {
+ compressed, err := os.Open(compressedPath)
+ if err != nil {
+ return "", fmt.Errorf("open compressed metadata: %w", err)
+ }
+ defer compressed.Close()
+ gzipReader, err := gzip.NewReader(compressed)
+ if err != nil {
+ return "", newIntegrityError("open metadata gzip: %v", err)
+ }
+ defer gzipReader.Close()
+
+ output, err := os.CreateTemp(a.repositoryStagingDir(), namePattern)
+ if err != nil {
+ return "", fmt.Errorf("create decompressed metadata temp file: %w", err)
+ }
+ outputPath := output.Name()
+ success := false
+ defer func() {
+ _ = output.Close()
+ if !success {
+ _ = os.Remove(outputPath)
+ }
+ }()
+ size, err := io.Copy(output, io.LimitReader(gzipReader, repositoryMetadataMaxBytes+1))
+ if err != nil {
+ return "", newIntegrityError("decompress metadata: %v", err)
+ }
+ if size > repositoryMetadataMaxBytes {
+ return "", newIntegrityError("decompressed metadata exceeds size limit")
+ }
+ if err := output.Close(); err != nil {
+ return "", fmt.Errorf("close decompressed metadata: %w", err)
+ }
+ success = true
+ return outputPath, nil
+}
+
+// packagesVariant is a candidate encoding of the Packages index, named as the InRelease
+// indexes it (suite-relative).
+type packagesVariant struct {
+ suiteRelativePath string
+ gzipped bool
+}
+
+// selectPackagesVariant picks the Packages encoding to fetch. The gzipped index is ~6x
+// smaller (720 KB vs 4.2 MB for jammy/main/binary-amd64), and apt itself fetches the
+// compressed form, so preferring it keeps the fast path from being heavier on the wire than
+// the package-manager path it is meant to beat. Integrity is unaffected: the InRelease
+// signs a SHA-256 for each encoding, and we verify the bytes we actually downloaded.
+// Repositories that publish only the plain index still work.
+func selectPackagesVariant(releasePayload []byte, base string) packagesVariant {
+ gzipped := base + ".gz"
+ if _, _, err := parseReleaseSHA256(releasePayload, gzipped); err == nil {
+ return packagesVariant{suiteRelativePath: gzipped, gzipped: true}
+ }
+ return packagesVariant{suiteRelativePath: base}
+}
+
+func (a *App) resolveUbuntuPackageMetadata(
+ ctx context.Context,
+ origin *url.URL,
+ repository aptRepository,
+ arch string,
+ fullVersion string,
+ expectedPackagePath string,
+) (repositoryPackageMetadata, error) {
+ inReleaseURL, err := resolveRepositoryURL(origin,
+ filepath.ToSlash(filepath.Join("dists", repository.Suite, "InRelease")))
+ if err != nil {
+ return repositoryPackageMetadata{}, err
+ }
+ // An InRelease indexes its checksum entries relative to the suite directory that
+ // contains it (e.g. "main/binary-amd64/Packages"), while the download URL needs the
+ // full repository-root-relative path. Keep the two separate: the suite-relative form
+ // is what parseReleaseSHA256 must match against.
+ packagesBaseSuiteRelative := filepath.ToSlash(filepath.Join(
+ repository.Component, "binary-"+arch, "Packages"))
+
+ inRelease, err := a.downloadRepositoryFile(ctx, inReleaseURL, origin, repositoryMetadataMaxBytes)
+ if err != nil {
+ return repositoryPackageMetadata{}, fmt.Errorf("download InRelease: %w", err)
+ }
+ defer os.Remove(inRelease.path)
+ if err = a.verifyRepoSignature(ctx, inRelease.path, "", repository.SignedBy); err != nil {
+ return repositoryPackageMetadata{}, err
+ }
+
+ releasePayload, err := readClearSignedPayload(inRelease.path)
+ if err != nil {
+ return repositoryPackageMetadata{}, newIntegrityError("parse authenticated InRelease: %v", err)
+ }
+
+ variant := selectPackagesVariant(releasePayload, packagesBaseSuiteRelative)
+ expectedPackagesSHA, expectedPackagesSize, err := parseReleaseSHA256(
+ releasePayload, variant.suiteRelativePath)
+ if err != nil {
+ return repositoryPackageMetadata{}, newIntegrityError("%v", err)
+ }
+ packagesURL, err := resolveRepositoryURL(origin, filepath.ToSlash(filepath.Join(
+ "dists", repository.Suite, variant.suiteRelativePath)))
+ if err != nil {
+ return repositoryPackageMetadata{}, err
+ }
+
+ packages, err := a.downloadRepositoryFile(ctx, packagesURL, origin, repositoryMetadataMaxBytes)
+ if err != nil {
+ return repositoryPackageMetadata{}, fmt.Errorf("download Packages: %w", err)
+ }
+ defer os.Remove(packages.path)
+ // Verify the downloaded encoding against its own signed entry, before decompressing.
+ if packages.size != expectedPackagesSize || !strings.EqualFold(packages.sha256, expectedPackagesSHA) {
+ return repositoryPackageMetadata{}, newIntegrityError(
+ "Packages metadata mismatch for %s: expected size/SHA256 %d/%s, got %d/%s",
+ variant.suiteRelativePath, expectedPackagesSize, expectedPackagesSHA,
+ packages.size, packages.sha256)
+ }
+
+ packagesPath := packages.path
+ if variant.gzipped {
+ decompressed, decErr := a.decompressGzipToTemp(
+ packages.path, ".aks-node-controller-packages-*")
+ if decErr != nil {
+ return repositoryPackageMetadata{}, decErr
+ }
+ defer os.Remove(decompressed)
+ packagesPath = decompressed
+ }
+
+ packageSHA, err := parseDebPackageMetadata(
+ packagesPath, fullVersion, arch, expectedPackagePath)
+ if err != nil {
+ return repositoryPackageMetadata{}, err
+ }
+ return repositoryPackageMetadata{sha256: packageSHA}, nil
+}
+
+func readClearSignedPayload(path string) ([]byte, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+ text := strings.ReplaceAll(string(data), "\r\n", "\n")
+ const begin = "-----BEGIN PGP SIGNED MESSAGE-----"
+ const signature = "-----BEGIN PGP SIGNATURE-----"
+ if !strings.HasPrefix(text, begin) {
+ return nil, fmt.Errorf("missing clear-signed message header")
+ }
+ headerEnd := strings.Index(text, "\n\n")
+ signatureStart := strings.Index(text, "\n"+signature)
+ if headerEnd < 0 || signatureStart < 0 || signatureStart <= headerEnd {
+ return nil, fmt.Errorf("malformed clear-signed message")
+ }
+ payload := text[headerEnd+2 : signatureStart]
+ var unescaped []string
+ for _, line := range strings.Split(payload, "\n") {
+ unescaped = append(unescaped, strings.TrimPrefix(line, "- "))
+ }
+ return []byte(strings.Join(unescaped, "\n")), nil
+}
+
+func parseReleaseSHA256(payload []byte, expectedPath string) (string, int64, error) {
+ scanner := bufio.NewScanner(strings.NewReader(string(payload)))
+ inSHA256 := false
+ for scanner.Scan() {
+ line := scanner.Text()
+ if line == "SHA256:" {
+ inSHA256 = true
+ continue
+ }
+ if inSHA256 && line != "" && line[0] != ' ' && line[0] != '\t' {
+ break
+ }
+ if !inSHA256 {
+ continue
+ }
+ fields := strings.Fields(line)
+ if len(fields) != 3 || filepath.ToSlash(fields[2]) != expectedPath {
+ continue
+ }
+ size, err := strconv.ParseInt(fields[1], 10, 64)
+ if err != nil || size < 0 {
+ return "", 0, fmt.Errorf("invalid size for %s in InRelease", expectedPath)
+ }
+ if !isSHA256Hex(fields[0]) {
+ return "", 0, fmt.Errorf("invalid SHA256 for %s in InRelease", expectedPath)
+ }
+ return strings.ToLower(fields[0]), size, nil
+ }
+ if err := scanner.Err(); err != nil {
+ return "", 0, err
+ }
+ return "", 0, fmt.Errorf("InRelease has no SHA256 entry for %s", expectedPath)
+}
+
+// matchDebPackageStanza reports whether a parsed Packages stanza is the exact ANC
+// package being sought. The bool is true once the identifying fields line up, at which
+// point the stanza is authoritative: a location or checksum problem is an integrity
+// failure rather than a reason to keep scanning.
+func matchDebPackageStanza(stanza map[string]string, fullVersion, arch, expectedLocation string) (string, bool, error) {
+ if stanza["Package"] != ancPackageName ||
+ stanza["Version"] != fullVersion ||
+ stanza["Architecture"] != arch {
+ return "", false, nil
+ }
+ if stanza["Filename"] != expectedLocation {
+ return "", true, newIntegrityError(
+ "authenticated Packages location %q does not match deterministic path %q",
+ stanza["Filename"], expectedLocation)
+ }
+ sum := strings.ToLower(strings.TrimSpace(stanza["SHA256"]))
+ if !isSHA256Hex(sum) {
+ return "", true, newIntegrityError("package stanza has no valid SHA256")
+ }
+ return sum, true, nil
+}
+
+// accumulateDebStanzaLine folds one Packages line into stanza. It returns true when a
+// blank line ends the current stanza; continuation lines (leading space/tab) are ignored.
+func accumulateDebStanzaLine(stanza map[string]string, line string) bool {
+ if strings.TrimSpace(line) == "" {
+ return true
+ }
+ if strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t") {
+ return false
+ }
+ if keyValue := strings.SplitN(line, ":", 2); len(keyValue) == 2 {
+ stanza[keyValue[0]] = strings.TrimSpace(keyValue[1])
+ }
+ return false
+}
+
+func parseDebPackageMetadata(path, fullVersion, arch, expectedLocation string) (string, error) {
+ file, err := os.Open(path)
+ if err != nil {
+ return "", fmt.Errorf("open Packages metadata: %w", err)
+ }
+ defer file.Close()
+
+ scanner := bufio.NewScanner(file)
+ scanner.Buffer(make([]byte, 64*1024), 4<<20)
+ stanza := map[string]string{}
+ for scanner.Scan() {
+ if !accumulateDebStanzaLine(stanza, scanner.Text()) {
+ continue
+ }
+ if sum, matched, matchErr := matchDebPackageStanza(stanza, fullVersion, arch, expectedLocation); matched || matchErr != nil {
+ return sum, matchErr
+ }
+ stanza = map[string]string{}
+ }
+ if err := scanner.Err(); err != nil {
+ return "", fmt.Errorf("scan Packages metadata: %w", err)
+ }
+ // The final stanza may not be terminated by a trailing blank line.
+ if sum, matched, matchErr := matchDebPackageStanza(stanza, fullVersion, arch, expectedLocation); matched || matchErr != nil {
+ return sum, matchErr
+ }
+ return "", newUnsupportedRepositoryError(
+ "authenticated Packages metadata has no exact %s %s %s stanza",
+ ancPackageName, fullVersion, arch)
+}
+
+type rpmRepository struct {
+ BaseURL string
+ GPGKeys []string
+ FilePath string
+ Section string
+}
+
+func (a *App) rpmRepositoryPlan(info platformInfo, hotfixVersion string) (repositoryDownloadPlan, error) {
+ rpmArch, err := rpmArchitecture(info.Arch)
+ if err != nil {
+ return repositoryDownloadPlan{}, err
+ }
+ releaseSuffix, err := rpmReleaseSuffix(info)
+ if err != nil {
+ return repositoryDownloadPlan{}, err
+ }
+ reposDir := a.yumReposDir
+ if reposDir == "" {
+ reposDir = defaultYumReposDir
+ }
+ repository, err := parseMSOSSRepository(reposDir)
+ if err != nil {
+ return repositoryDownloadPlan{}, err
+ }
+ releaseVersion := rpmReleaseVersion(info.VersionID)
+ baseURL := strings.ReplaceAll(repository.BaseURL, "$releasever", releaseVersion)
+ baseURL = strings.ReplaceAll(baseURL, "${releasever}", releaseVersion)
+ baseURL = strings.ReplaceAll(baseURL, "$basearch", rpmArch)
+ baseURL = strings.ReplaceAll(baseURL, "${basearch}", rpmArch)
+ if strings.Contains(baseURL, "$") {
+ return repositoryDownloadPlan{}, newUnsupportedRepositoryError(
+ "unsupported variable in Microsoft repository baseurl %q", repository.BaseURL)
+ }
+ origin, err := validateRepositoryURL(baseURL)
+ if err != nil {
+ return repositoryDownloadPlan{}, err
+ }
+ origin = asRepositoryBase(origin)
+
+ expectedRelease := "1." + releaseSuffix
+ relativePackagePath := fmt.Sprintf(
+ "Packages/a/%s-%s-%s.%s.rpm", ancPackageName, hotfixVersion, expectedRelease, rpmArch)
+ packageURL, err := resolveRepositoryURL(origin, relativePackagePath)
+ if err != nil {
+ return repositoryDownloadPlan{}, err
+ }
+ return repositoryDownloadPlan{
+ format: "rpm",
+ packageURL: packageURL,
+ trustedOrigin: origin,
+ resolveMetadata: func(ctx context.Context) (repositoryPackageMetadata, error) {
+ return a.resolveRPMPackageMetadata(
+ ctx, origin, repository.GPGKeys, hotfixVersion, expectedRelease, rpmArch, relativePackagePath)
+ },
+ }, nil
+}
+
+func rpmArchitecture(goarch string) (string, error) {
+ switch goarch {
+ case archAMD64:
+ return "x86_64", nil
+ case archARM64:
+ return "aarch64", nil
+ default:
+ return "", newUnsupportedRepositoryError("unsupported RPM architecture %q", goarch)
+ }
+}
+
+// rpmReleaseVersion reduces an os-release VERSION_ID to the major.minor form that PMC
+// publishes repositories under. Azure Linux and Mariner nodes can report a three-part
+// VERSION_ID that includes a build date (e.g. "3.0.20260304"), while the repository lives
+// at .../azurelinux/3.0/prod/... -- substituting the raw value into $releasever builds a
+// URL that 404s, silently costing every such node the fast path. Values already in
+// major.minor form, or with no dot at all, are returned unchanged.
+func rpmReleaseVersion(versionID string) string {
+ parts := strings.Split(versionID, ".")
+ if len(parts) < 2 {
+ return versionID
+ }
+ return parts[0] + "." + parts[1]
+}
+
+func rpmReleaseSuffix(info platformInfo) (string, error) {
+ major := strings.SplitN(info.VersionID, ".", 2)[0]
+ switch {
+ case info.ID == osIDAzureLinux && major == "3":
+ return "azl3", nil
+ case info.ID == osIDMariner && major == "2":
+ return "cm2", nil
+ default:
+ return "", newUnsupportedRepositoryError(
+ "cannot establish ANC RPM release suffix for %s %s", info.ID, info.VersionID)
+ }
+}
+
+// matchesMicrosoftRPMRepo reports whether an INI section names the repository that carries
+// the ANC package, for either distro family. AzureLinux publishes it under "ms-oss"
+// (section [azurelinux-official-ms-oss], baseurl .../prod/ms-oss/$basearch), while Mariner
+// 2.0 has no ms-oss repository at all -- that path 404s -- and uses "Microsoft" instead
+// (section [mariner-microsoft], baseurl .../prod/Microsoft/$basearch). Matching only on
+// ms-oss silently excluded every Mariner node from the fast path. Comparisons are
+// lowercased so Mariner's capitalised "/Microsoft/" baseurl matches.
+func matchesMicrosoftRPMRepo(name, baseURL string) bool {
+ matchers := []struct {
+ section string
+ urlPath string
+ }{
+ {section: "ms-oss", urlPath: "/ms-oss/"},
+ {section: "microsoft", urlPath: "/microsoft/"},
+ }
+ lowerName := strings.ToLower(name)
+ lowerURL := strings.ToLower(baseURL)
+ for _, matcher := range matchers {
+ if strings.Contains(lowerName, matcher.section) || strings.Contains(lowerURL, matcher.urlPath) {
+ return true
+ }
+ }
+ return false
+}
+
+func parseMSOSSRepository(reposDir string) (rpmRepository, error) {
+ paths, err := filepath.Glob(filepath.Join(reposDir, "*.repo"))
+ if err != nil {
+ return rpmRepository{}, newUnsupportedRepositoryError("scan RPM repos: %v", err)
+ }
+ for _, path := range paths {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ continue
+ }
+ sections := parseINISections(string(data))
+ for name, values := range sections {
+ if !matchesMicrosoftRPMRepo(name, values["baseurl"]) {
+ continue
+ }
+ if strings.TrimSpace(values["enabled"]) == "0" {
+ continue
+ }
+ keys, err := localGPGKeyPaths(values["gpgkey"])
+ if err != nil {
+ return rpmRepository{}, err
+ }
+ baseURL := strings.Fields(values["baseurl"])
+ if len(baseURL) == 0 {
+ continue
+ }
+ return rpmRepository{
+ BaseURL: baseURL[0],
+ GPGKeys: keys,
+ FilePath: path,
+ Section: name,
+ }, nil
+ }
+ }
+ return rpmRepository{}, newUnsupportedRepositoryError(
+ "no enabled Microsoft-published RPM repository in %s", reposDir)
+}
+
+func parseINISections(contents string) map[string]map[string]string {
+ sections := map[string]map[string]string{}
+ var current map[string]string
+ for _, rawLine := range strings.Split(strings.ReplaceAll(contents, "\r\n", "\n"), "\n") {
+ line := strings.TrimSpace(rawLine)
+ if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") {
+ continue
+ }
+ if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
+ name := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(line, "["), "]"))
+ current = map[string]string{}
+ sections[name] = current
+ continue
+ }
+ if current == nil {
+ continue
+ }
+ keyValue := strings.SplitN(line, "=", 2)
+ if len(keyValue) == 2 {
+ current[strings.ToLower(strings.TrimSpace(keyValue[0]))] = strings.TrimSpace(keyValue[1])
+ }
+ }
+ return sections
+}
+
+func localGPGKeyPaths(value string) ([]string, error) {
+ var paths []string
+ for _, field := range strings.Fields(value) {
+ u, err := url.Parse(field)
+ if err != nil {
+ return nil, newUnsupportedRepositoryError("parse RPM gpgkey %q: %v", field, err)
+ }
+ switch {
+ case u.Scheme == "file" && u.Path != "":
+ paths = append(paths, u.Path)
+ case u.Scheme == "" && strings.HasPrefix(field, "/"):
+ paths = append(paths, field)
+ default:
+ return nil, newUnsupportedRepositoryError(
+ "RPM gpgkey %q is not an installed local key", field)
+ }
+ }
+ if len(paths) == 0 {
+ return nil, newUnsupportedRepositoryError("Microsoft repository has no local gpgkey")
+ }
+ return paths, nil
+}
+
+type rpmRepoMD struct {
+ Data []struct {
+ Type string `xml:"type,attr"`
+ Checksum rpmChecksum `xml:"checksum"`
+ OpenChecksum rpmChecksum `xml:"open-checksum"`
+ Location struct {
+ Href string `xml:"href,attr"`
+ } `xml:"location"`
+ Size int64 `xml:"size"`
+ OpenSize int64 `xml:"open-size"`
+ } `xml:"data"`
+}
+
+type rpmChecksum struct {
+ Type string `xml:"type,attr"`
+ Value string `xml:",chardata"`
+}
+
+type rpmPrimaryPackage struct {
+ Name string `xml:"name"`
+ Arch string `xml:"arch"`
+ Version struct {
+ Ver string `xml:"ver,attr"`
+ Rel string `xml:"rel,attr"`
+ } `xml:"version"`
+ Checksum rpmChecksum `xml:"checksum"`
+ Location struct {
+ Href string `xml:"href,attr"`
+ } `xml:"location"`
+}
+
+// verifiedPrimaryReference downloads repomd.xml and its detached signature, verifies the
+// signature against keyrings, and returns the authenticated reference to primary metadata.
+// Both downloaded files are removed before returning; only the parsed reference escapes.
+func (a *App) verifiedPrimaryReference(
+ ctx context.Context,
+ origin *url.URL,
+ keyrings []string,
+) (primaryMetadataReference, error) {
+ repomdURL, err := resolveRepositoryURL(origin, "repodata/repomd.xml")
+ if err != nil {
+ return primaryMetadataReference{}, err
+ }
+ signatureURL, err := resolveRepositoryURL(origin, "repodata/repomd.xml.asc")
+ if err != nil {
+ return primaryMetadataReference{}, err
+ }
+ repomd, err := a.downloadRepositoryFile(ctx, repomdURL, origin, repositoryMetadataMaxBytes)
+ if err != nil {
+ return primaryMetadataReference{}, fmt.Errorf("download repomd.xml: %w", err)
+ }
+ defer os.Remove(repomd.path)
+ signature, err := a.downloadRepositoryFile(ctx, signatureURL, origin, 1<<20)
+ if err != nil {
+ return primaryMetadataReference{}, fmt.Errorf("download repomd.xml.asc: %w", err)
+ }
+ defer os.Remove(signature.path)
+ if err = a.verifyRepoSignature(ctx, repomd.path, signature.path, keyrings); err != nil {
+ return primaryMetadataReference{}, err
+ }
+ return parsePrimaryReference(repomd.path)
+}
+
+// verifyPrimaryPayload checks the downloaded primary metadata against the authenticated
+// reference and returns the path to its uncompressed XML. The returned cleanup func must
+// be called by the caller; it removes any file this function created or downloaded.
+func (a *App) verifyPrimaryPayload(
+ primary primaryMetadataReference,
+ primaryFile downloadedRepositoryFile,
+) (string, func(), error) {
+ cleanup := func() { os.Remove(primaryFile.path) }
+ if primary.size > 0 && primary.size != primaryFile.size {
+ return "", cleanup, newIntegrityError(
+ "primary metadata size mismatch: expected %d, got %d", primary.size, primaryFile.size)
+ }
+ if !strings.EqualFold(primary.checksum, primaryFile.sha256) {
+ return "", cleanup, newIntegrityError(
+ "primary metadata SHA-256 mismatch: expected %s, got %s",
+ primary.checksum, primaryFile.sha256)
+ }
+
+ switch {
+ case strings.HasSuffix(primary.location, ".gz"):
+ decompressed, decErr := a.decompressPrimaryMetadata(primaryFile.path, primary)
+ if decErr != nil {
+ return "", cleanup, decErr
+ }
+ return decompressed, func() {
+ os.Remove(decompressed)
+ os.Remove(primaryFile.path)
+ }, nil
+ case strings.HasSuffix(primary.location, ".xml"):
+ if primary.openSize > 0 && primary.openSize != primaryFile.size {
+ return "", cleanup, newIntegrityError(
+ "primary metadata open-size mismatch: expected %d, got %d",
+ primary.openSize, primaryFile.size)
+ }
+ if primary.openChecksum != "" &&
+ !strings.EqualFold(primary.openChecksum, primaryFile.sha256) {
+ return "", cleanup, newIntegrityError(
+ "primary metadata open-checksum mismatch: expected %s, got %s",
+ primary.openChecksum, primaryFile.sha256)
+ }
+ return primaryFile.path, cleanup, nil
+ default:
+ return "", cleanup, newUnsupportedRepositoryError(
+ "unsupported primary metadata compression for %q", primary.location)
+ }
+}
+
+func (a *App) resolveRPMPackageMetadata(
+ ctx context.Context,
+ origin *url.URL,
+ keyrings []string,
+ version, release, arch, expectedLocation string,
+) (repositoryPackageMetadata, error) {
+ primary, err := a.verifiedPrimaryReference(ctx, origin, keyrings)
+ if err != nil {
+ return repositoryPackageMetadata{}, err
+ }
+ primaryURL, err := resolveRepositoryURL(origin, primary.location)
+ if err != nil {
+ return repositoryPackageMetadata{}, err
+ }
+ primaryFile, err := a.downloadRepositoryFile(ctx, primaryURL, origin, repositoryMetadataMaxBytes)
+ if err != nil {
+ return repositoryPackageMetadata{}, fmt.Errorf("download primary metadata: %w", err)
+ }
+ xmlPath, cleanup, err := a.verifyPrimaryPayload(primary, primaryFile)
+ defer cleanup()
+ if err != nil {
+ return repositoryPackageMetadata{}, err
+ }
+ packageSHA, err := parseRPMPrimaryMetadata(
+ xmlPath, version, release, arch, expectedLocation)
+ if err != nil {
+ return repositoryPackageMetadata{}, err
+ }
+ return repositoryPackageMetadata{sha256: packageSHA}, nil
+}
+
+type primaryMetadataReference struct {
+ location string
+ checksum string
+ size int64
+ openChecksum string
+ openSize int64
+}
+
+func parsePrimaryReference(path string) (primaryMetadataReference, error) {
+ file, err := os.Open(path)
+ if err != nil {
+ return primaryMetadataReference{}, fmt.Errorf("open repomd.xml: %w", err)
+ }
+ defer file.Close()
+ var metadata rpmRepoMD
+ if err := xml.NewDecoder(file).Decode(&metadata); err != nil {
+ return primaryMetadataReference{}, newIntegrityError("parse authenticated repomd.xml: %v", err)
+ }
+ for _, data := range metadata.Data {
+ if data.Type != "primary" {
+ continue
+ }
+ if !strings.EqualFold(data.Checksum.Type, "sha256") ||
+ !isSHA256Hex(strings.TrimSpace(data.Checksum.Value)) {
+ return primaryMetadataReference{}, newUnsupportedRepositoryError(
+ "primary metadata does not provide a SHA-256 checksum")
+ }
+ ref := primaryMetadataReference{
+ location: strings.TrimSpace(data.Location.Href),
+ checksum: strings.ToLower(strings.TrimSpace(data.Checksum.Value)),
+ size: data.Size,
+ openSize: data.OpenSize,
+ }
+ if strings.TrimSpace(data.OpenChecksum.Value) != "" {
+ if !strings.EqualFold(data.OpenChecksum.Type, "sha256") ||
+ !isSHA256Hex(strings.TrimSpace(data.OpenChecksum.Value)) {
+ return primaryMetadataReference{}, newUnsupportedRepositoryError(
+ "primary metadata open-checksum is not SHA-256")
+ }
+ ref.openChecksum = strings.ToLower(strings.TrimSpace(data.OpenChecksum.Value))
+ }
+ if ref.location == "" {
+ return primaryMetadataReference{}, newIntegrityError("primary metadata location is empty")
+ }
+ return ref, nil
+ }
+ return primaryMetadataReference{}, newUnsupportedRepositoryError("repomd.xml has no primary metadata")
+}
+
+func (a *App) decompressPrimaryMetadata(
+ compressedPath string,
+ primary primaryMetadataReference,
+) (string, error) {
+ compressed, err := os.Open(compressedPath)
+ if err != nil {
+ return "", fmt.Errorf("open compressed primary metadata: %w", err)
+ }
+ defer compressed.Close()
+ gzipReader, err := gzip.NewReader(compressed)
+ if err != nil {
+ return "", newIntegrityError("open primary metadata gzip: %v", err)
+ }
+ defer gzipReader.Close()
+ output, err := os.CreateTemp(a.repositoryStagingDir(), ".aks-node-controller-primary-*")
+ if err != nil {
+ return "", fmt.Errorf("create primary metadata temp file: %w", err)
+ }
+ outputPath := output.Name()
+ success := false
+ defer func() {
+ _ = output.Close()
+ if !success {
+ _ = os.Remove(outputPath)
+ }
+ }()
+ hasher := sha256.New()
+ size, err := io.Copy(
+ io.MultiWriter(output, hasher),
+ io.LimitReader(gzipReader, repositoryMetadataMaxBytes+1))
+ if err != nil {
+ return "", newIntegrityError("decompress primary metadata: %v", err)
+ }
+ if size > repositoryMetadataMaxBytes {
+ return "", newIntegrityError("decompressed primary metadata exceeds size limit")
+ }
+ if err := output.Close(); err != nil {
+ return "", fmt.Errorf("close decompressed primary metadata: %w", err)
+ }
+ actualChecksum := hex.EncodeToString(hasher.Sum(nil))
+ if primary.openSize > 0 && primary.openSize != size {
+ return "", newIntegrityError(
+ "primary metadata open-size mismatch: expected %d, got %d", primary.openSize, size)
+ }
+ if primary.openChecksum != "" && !strings.EqualFold(primary.openChecksum, actualChecksum) {
+ return "", newIntegrityError(
+ "primary metadata open-checksum mismatch: expected %s, got %s",
+ primary.openChecksum, actualChecksum)
+ }
+ success = true
+ return outputPath, nil
+}
+
+func parseRPMPrimaryMetadata(path, version, release, arch, expectedLocation string) (string, error) {
+ file, err := os.Open(path)
+ if err != nil {
+ return "", fmt.Errorf("open primary metadata: %w", err)
+ }
+ defer file.Close()
+ decoder := xml.NewDecoder(file)
+ for {
+ token, err := decoder.Token()
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ return "", newIntegrityError("parse primary metadata: %v", err)
+ }
+ start, ok := token.(xml.StartElement)
+ if !ok || start.Name.Local != "package" {
+ continue
+ }
+ var pkg rpmPrimaryPackage
+ if err := decoder.DecodeElement(&pkg, &start); err != nil {
+ return "", newIntegrityError("parse primary package: %v", err)
+ }
+ if pkg.Name != ancPackageName || pkg.Arch != arch ||
+ pkg.Version.Ver != version || pkg.Version.Rel != release {
+ continue
+ }
+ if pkg.Location.Href != expectedLocation {
+ return "", newIntegrityError(
+ "authenticated RPM location %q does not match deterministic path %q",
+ pkg.Location.Href, expectedLocation)
+ }
+ sum := strings.ToLower(strings.TrimSpace(pkg.Checksum.Value))
+ if !strings.EqualFold(pkg.Checksum.Type, "sha256") || !isSHA256Hex(sum) {
+ return "", newIntegrityError("RPM package metadata has no valid SHA-256")
+ }
+ return sum, nil
+ }
+ return "", newUnsupportedRepositoryError(
+ "primary metadata has no exact %s %s-%s.%s package",
+ ancPackageName, version, release, arch)
+}
diff --git a/aks-node-controller/repository_hotfix_test.go b/aks-node-controller/repository_hotfix_test.go
new file mode 100644
index 00000000000..9979c032460
--- /dev/null
+++ b/aks-node-controller/repository_hotfix_test.go
@@ -0,0 +1,1140 @@
+package main
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestDownloadBinaryHotfixGatesRepositoryWork(t *testing.T) {
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ requests.Add(1)
+ http.Error(w, "unexpected request", http.StatusInternalServerError)
+ }))
+ defer server.Close()
+
+ dir := t.TempDir()
+ aptDir := filepath.Join(dir, "sources")
+ require.NoError(t, os.MkdirAll(aptDir, 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(aptDir, "microsoft-prod.list"), []byte(fmt.Sprintf(
+ "deb [arch=amd64 signed-by=/keys/microsoft.gpg] %s/ubuntu/22.04/prod jammy main\n",
+ server.URL)), 0o644))
+ osRelease := filepath.Join(dir, "os-release")
+ require.NoError(t, os.WriteFile(osRelease, []byte("ID=ubuntu\nVERSION_ID=\"22.04\"\n"), 0o644))
+
+ var commands atomic.Int32
+ app := NewTestApp(t, TestAppConfig{RunFunc: func(*exec.Cmd) error {
+ commands.Add(1)
+ return nil
+ }}).App
+ app.aptSourcesDir = aptDir
+ app.osReleasePath = osRelease
+ app.goArch = "amd64"
+ app.repositoryTempDir = dir
+
+ originalVersion := Version
+ t.Cleanup(func() { Version = originalVersion })
+
+ Version = "202608.21.0"
+ require.NoError(t, app.downloadBinaryHotfixIfNeeded(context.Background(), &hotfixConfig{}))
+ Version = "202608.21.2"
+ require.NoError(t, app.downloadBinaryHotfixIfNeeded(context.Background(), &hotfixConfig{
+ Hotfixes: map[string]string{"202608.21": "202608.21.1"},
+ }))
+ Version = "202608.21.0"
+ require.NoError(t, app.downloadBinaryHotfixIfNeeded(context.Background(), &hotfixConfig{
+ Hotfixes: map[string]string{"202608.21": "202607.20.2"},
+ }))
+
+ assert.Zero(t, requests.Load(), "repository must not be contacted before version gating passes")
+ assert.Zero(t, commands.Load(), "package manager must not run before version gating passes")
+}
+
+func TestUbuntuRepositoryFastPathParallelSuccessExtractsBinary(t *testing.T) {
+ const (
+ hotfixVersion = "202608.21.1"
+ fullVersion = hotfixVersion + "-ubuntu22.04u1"
+ )
+ packageBytes := []byte("authenticated-deb-package-bytes")
+ packageSHA := sha256Hex(packageBytes)
+ packageLocation := "pool/main/a/aks-node-controller/aks-node-controller_" +
+ fullVersion + "_amd64.deb"
+ packages := []byte(fmt.Sprintf(
+ "Package: aks-node-controller\nVersion: %s\nArchitecture: amd64\nFilename: %s\nSHA256: %s\n\n",
+ fullVersion, packageLocation, packageSHA))
+ packagesSHA := sha256Hex(packages)
+ // The InRelease indexes the Packages file suite-relative; the HTTP path is rooted at
+ // the repository. Keeping these distinct is what proves we look each one up correctly.
+ packagesSuiteRelative := "main/binary-amd64/Packages"
+ packagesLocation := "dists/jammy/" + packagesSuiteRelative
+ inRelease := clearSignedRelease(packagesSuiteRelative, packagesSHA, int64(len(packages)))
+
+ packageStarted := make(chan struct{})
+ metadataStarted := make(chan struct{})
+ var packageOnce, metadataOnce sync.Once
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/ubuntu/22.04/prod/" + packageLocation:
+ packageOnce.Do(func() { close(packageStarted) })
+ select {
+ case <-metadataStarted:
+ case <-time.After(2 * time.Second):
+ http.Error(w, "metadata did not start concurrently", http.StatusInternalServerError)
+ return
+ }
+ _, _ = w.Write(packageBytes)
+ case "/ubuntu/22.04/prod/dists/jammy/InRelease":
+ metadataOnce.Do(func() { close(metadataStarted) })
+ select {
+ case <-packageStarted:
+ case <-time.After(2 * time.Second):
+ http.Error(w, "package did not start concurrently", http.StatusInternalServerError)
+ return
+ }
+ _, _ = w.Write(inRelease)
+ case "/ubuntu/22.04/prod/" + packagesLocation:
+ _, _ = w.Write(packages)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ dir := t.TempDir()
+ app := configuredUbuntuRepositoryApp(t, dir, server.URL, func(*exec.Cmd) error {
+ return fmt.Errorf("package-manager fallback was not expected")
+ })
+ vhdPath := filepath.Join(dir, "aks-node-controller")
+ hotfixPath := filepath.Join(dir, "aks-node-controller-hotfix")
+ require.NoError(t, os.WriteFile(vhdPath, []byte("vhd-binary"), 0o755))
+ app.vhdBinaryPath = vhdPath
+ app.hotfixBinaryPath = hotfixPath
+ app.verifyRepositorySignature = func(
+ _ context.Context, signedPath, signaturePath string, keyrings []string,
+ ) error {
+ assert.Empty(t, signaturePath)
+ assert.Equal(t, []string{"/keys/microsoft.gpg"}, keyrings)
+ data, err := os.ReadFile(signedPath)
+ require.NoError(t, err)
+ assert.Contains(t, string(data), "BEGIN PGP SIGNED MESSAGE")
+ return nil
+ }
+ app.extractRepositoryPackage = func(
+ _ context.Context, format, packagePath, destination string,
+ ) error {
+ assert.Equal(t, "deb", format)
+ data, err := os.ReadFile(packagePath)
+ require.NoError(t, err)
+ assert.Equal(t, packageBytes, data)
+ extracted := filepath.Join(destination, filepath.FromSlash(ancPackageBinaryRelativePath))
+ require.NoError(t, os.MkdirAll(filepath.Dir(extracted), 0o755))
+ return os.WriteFile(extracted, []byte("extracted-anc-binary"), 0o644)
+ }
+
+ originalVersion := Version
+ Version = "202608.21.0"
+ t.Cleanup(func() { Version = originalVersion })
+ err := app.downloadBinaryHotfixIfNeeded(context.Background(), &hotfixConfig{
+ Hotfixes: map[string]string{"202608.21": hotfixVersion},
+ })
+ require.NoError(t, err)
+
+ staged, err := os.ReadFile(hotfixPath)
+ require.NoError(t, err)
+ assert.Equal(t, []byte("extracted-anc-binary"), staged)
+ assert.NotEqual(t, packageBytes, staged, "the .deb bytes must never be staged as the executable")
+}
+
+func TestUbuntuRepositoryPackageChecksumMismatchIsHardFailure(t *testing.T) {
+ const (
+ hotfixVersion = "202608.21.1"
+ fullVersion = hotfixVersion + "-ubuntu22.04u1"
+ )
+ packageBytes := []byte("tampered-package")
+ packageLocation := "pool/main/a/aks-node-controller/aks-node-controller_" +
+ fullVersion + "_amd64.deb"
+ packages := []byte(fmt.Sprintf(
+ "Package: aks-node-controller\nVersion: %s\nArchitecture: amd64\nFilename: %s\nSHA256: %s\n\n",
+ fullVersion, packageLocation, strings.Repeat("a", 64)))
+ packagesSuiteRelative := "main/binary-amd64/Packages"
+ packagesLocation := "dists/jammy/" + packagesSuiteRelative
+ inRelease := clearSignedRelease(packagesSuiteRelative, sha256Hex(packages), int64(len(packages)))
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/ubuntu/22.04/prod/" + packageLocation:
+ _, _ = w.Write(packageBytes)
+ case "/ubuntu/22.04/prod/dists/jammy/InRelease":
+ _, _ = w.Write(inRelease)
+ case "/ubuntu/22.04/prod/" + packagesLocation:
+ _, _ = w.Write(packages)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ dir := t.TempDir()
+ var packageManagerCalled atomic.Bool
+ app := configuredUbuntuRepositoryApp(t, dir, server.URL, func(*exec.Cmd) error {
+ packageManagerCalled.Store(true)
+ return nil
+ })
+ app.verifyRepositorySignature = func(context.Context, string, string, []string) error { return nil }
+ hotfixPath := filepath.Join(dir, "aks-node-controller-hotfix")
+ app.hotfixBinaryPath = hotfixPath
+ require.NoError(t, os.WriteFile(hotfixPath, []byte("stale-hotfix"), 0o755))
+
+ originalVersion := Version
+ Version = "202608.21.0"
+ t.Cleanup(func() { Version = originalVersion })
+ err := app.downloadBinaryHotfixIfNeeded(context.Background(), &hotfixConfig{
+ Hotfixes: map[string]string{"202608.21": hotfixVersion},
+ })
+ require.Error(t, err)
+ assert.True(t, isIntegrityError(err))
+ assert.False(t, packageManagerCalled.Load(), "integrity failures must not use apt fallback")
+ _, statErr := os.Stat(hotfixPath)
+ assert.True(t, os.IsNotExist(statErr), "stale hotfix must be removed after integrity failure")
+}
+
+func TestUbuntuRepositoryHTTPErrorFallsBackToApt(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ http.Error(w, "transient repository failure", http.StatusServiceUnavailable)
+ }))
+ defer server.Close()
+
+ dir := t.TempDir()
+ var commands []string
+ var mu sync.Mutex
+ app := configuredUbuntuRepositoryApp(t, dir, server.URL, func(cmd *exec.Cmd) error {
+ mu.Lock()
+ commands = append(commands, strings.Join(cmd.Args, " "))
+ mu.Unlock()
+ return nil
+ })
+
+ originalVersion := Version
+ Version = "202608.21.0"
+ t.Cleanup(func() { Version = originalVersion })
+ err := app.downloadBinaryHotfixIfNeeded(context.Background(), &hotfixConfig{
+ Hotfixes: map[string]string{"202608.21": "202608.21.1"},
+ })
+ require.Error(t, err) // fallback reaches staging, where /usr/bin is absent in the unit test
+ assert.False(t, isIntegrityError(err))
+ assert.Condition(t, func() bool {
+ mu.Lock()
+ defer mu.Unlock()
+ for _, command := range commands {
+ if strings.Contains(command, "apt-get install") {
+ return true
+ }
+ }
+ return false
+ }, "an operational direct-path failure must invoke apt fallback")
+}
+
+func TestUbuntuRepositoryFallbackDurationIncludesRepositoryAttempt(t *testing.T) {
+ const repositoryDelay = 50 * time.Millisecond
+ logs := installLogCapturer(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ time.Sleep(repositoryDelay)
+ http.Error(w, "transient repository failure", http.StatusServiceUnavailable)
+ }))
+ defer server.Close()
+
+ dir := t.TempDir()
+ app := configuredUbuntuRepositoryApp(t, dir, server.URL, func(*exec.Cmd) error {
+ return nil
+ })
+ app.vhdBinaryPath = filepath.Join(dir, "aks-node-controller")
+ app.pkgBinaryPath = filepath.Join(dir, "usr-bin-aks-node-controller")
+ app.hotfixBinaryPath = filepath.Join(dir, "aks-node-controller-hotfix")
+ require.NoError(t, os.WriteFile(app.vhdBinaryPath, []byte("vhd-binary"), 0o755))
+ require.NoError(t, os.WriteFile(app.pkgBinaryPath, []byte("package-manager-binary"), 0o755))
+
+ originalVersion := Version
+ Version = "202608.21.0"
+ t.Cleanup(func() { Version = originalVersion })
+ err := app.downloadBinaryHotfixIfNeeded(context.Background(), &hotfixConfig{
+ Hotfixes: map[string]string{"202608.21": "202608.21.1"},
+ })
+ require.NoError(t, err)
+
+ var durationMs int64 = -1
+ for _, record := range logs.getRecords() {
+ if record.Message != "downloaded ANC hotfix" {
+ continue
+ }
+ duration, parseErr := strconv.ParseInt(record.Attrs["durationMs"], 10, 64)
+ require.NoError(t, parseErr)
+ durationMs = duration
+ }
+ require.GreaterOrEqual(t, durationMs, repositoryDelay.Milliseconds())
+ staged, err := os.ReadFile(app.hotfixBinaryPath)
+ require.NoError(t, err)
+ assert.Equal(t, []byte("package-manager-binary"), staged)
+}
+
+func TestParseAptRepositoryFormats(t *testing.T) {
+ t.Run("one-line deb", func(t *testing.T) {
+ repository, err := parseOneLineAptRepository(
+ "deb [arch=amd64,arm64 signed-by=/usr/share/keyrings/microsoft-prod.gpg] https://packages.microsoft.com/ubuntu/22.04/prod jammy main\n",
+ "microsoft-prod.list", "amd64")
+ require.NoError(t, err)
+ assert.Equal(t, "https://packages.microsoft.com/ubuntu/22.04/prod", repository.URI)
+ assert.Equal(t, "jammy", repository.Suite)
+ assert.Equal(t, "main", repository.Component)
+ assert.Equal(t, []string{"/usr/share/keyrings/microsoft-prod.gpg"}, repository.SignedBy)
+ })
+
+ t.Run("one-line deb using global Microsoft keyring", func(t *testing.T) {
+ repository, err := parseOneLineAptRepository(
+ "deb [arch=amd64,arm64] https://packages.microsoft.com/ubuntu/22.04/prod jammy main\n",
+ "microsoft-prod.list", "amd64")
+ require.NoError(t, err)
+ assert.Empty(t, repository.SignedBy)
+
+ keyringsDir := t.TempDir()
+ expected := filepath.Join(keyringsDir, "microsoft.gpg")
+ require.NoError(t, os.WriteFile(expected, []byte("keyring"), 0o644))
+ keyrings, err := microsoftAptTrustedKeyrings(keyringsDir)
+ require.NoError(t, err)
+ assert.Equal(t, []string{expected}, keyrings)
+ })
+
+ t.Run("one-line deb rejects signed-by fingerprint constraints", func(t *testing.T) {
+ _, err := parseOneLineAptRepository(
+ "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft-prod.gpg ABCD1234] https://packages.microsoft.com/ubuntu/22.04/prod jammy main\n",
+ "microsoft-prod.list", "amd64")
+ require.Error(t, err)
+ var unsupported *unsupportedRepositoryError
+ assert.True(t, errors.As(err, &unsupported))
+ })
+
+ t.Run("deb822", func(t *testing.T) {
+ repository, err := parseDeb822Repository(`
+Types: deb
+URIs: https://repodepot.example/microsoft/ubuntu/22.04/prod
+Suites: jammy
+Components: main
+Architectures: amd64 arm64
+Signed-By: /usr/share/keyrings/microsoft-prod.gpg /usr/share/keyrings/microsoft-2025.gpg
+`, "microsoft-prod.sources", "arm64")
+ require.NoError(t, err)
+ assert.Equal(t, "https://repodepot.example/microsoft/ubuntu/22.04/prod", repository.URI)
+ assert.Equal(t, []string{
+ "/usr/share/keyrings/microsoft-prod.gpg",
+ "/usr/share/keyrings/microsoft-2025.gpg",
+ }, repository.SignedBy)
+ })
+
+ t.Run("deb822 rejects embedded signed-by key", func(t *testing.T) {
+ _, err := parseDeb822Repository(`
+Types: deb
+URIs: https://repodepot.example/microsoft/ubuntu/22.04/prod
+Suites: jammy
+Components: main
+Architectures: amd64
+Signed-By:
+ -----BEGIN PGP PUBLIC KEY BLOCK-----
+ fake
+ -----END PGP PUBLIC KEY BLOCK-----
+`, "microsoft-prod.sources", "amd64")
+ require.Error(t, err)
+ var unsupported *unsupportedRepositoryError
+ assert.True(t, errors.As(err, &unsupported))
+ })
+}
+
+func TestPrepareGPGVKeyringsDearmorsArmoredKeys(t *testing.T) {
+ dir := t.TempDir()
+ armoredPath := filepath.Join(dir, "MICROSOFT-RPM-GPG-KEY")
+ require.NoError(t, os.WriteFile(armoredPath, []byte(
+ "-----BEGIN PGP PUBLIC KEY BLOCK-----\nfake\n-----END PGP PUBLIC KEY BLOCK-----\n"), 0o644))
+
+ app := NewTestApp(t, TestAppConfig{
+ RunFunc: func(cmd *exec.Cmd) error {
+ assert.Equal(t, "gpg", filepath.Base(cmd.Path))
+ outputIndex := -1
+ for i, arg := range cmd.Args {
+ if arg == "--output" {
+ outputIndex = i + 1
+ break
+ }
+ }
+ require.Greater(t, outputIndex, 0)
+ return os.WriteFile(cmd.Args[outputIndex], []byte("binary-keyring"), 0o600)
+ },
+ }).App
+ app.repositoryTempDir = dir
+
+ keyrings, cleanup, err := app.prepareGPGVKeyrings(context.Background(), []string{armoredPath})
+ require.NoError(t, err)
+ require.Len(t, keyrings, 1)
+ assert.NotEqual(t, armoredPath, keyrings[0])
+ assert.FileExists(t, keyrings[0])
+ cleanup()
+ assert.NoFileExists(t, keyrings[0])
+}
+
+func TestExtractRepositoryTarMemberOnlyWritesANCBinary(t *testing.T) {
+ var archive bytes.Buffer
+ writer := tar.NewWriter(&archive)
+ require.NoError(t, writer.WriteHeader(&tar.Header{
+ Name: "etc/not-anc",
+ Mode: 0o644,
+ Size: int64(len("not-anc")),
+ }))
+ _, err := writer.Write([]byte("not-anc"))
+ require.NoError(t, err)
+ require.NoError(t, writer.WriteHeader(&tar.Header{
+ Name: "./" + ancPackageBinaryRelativePath,
+ Mode: 0o755,
+ Size: int64(len("anc-binary")),
+ }))
+ _, err = writer.Write([]byte("anc-binary"))
+ require.NoError(t, err)
+ require.NoError(t, writer.WriteHeader(&tar.Header{
+ Name: "usr/bin/other",
+ Mode: 0o755,
+ Size: int64(len("other")),
+ }))
+ _, err = writer.Write([]byte("other"))
+ require.NoError(t, err)
+ require.NoError(t, writer.Close())
+
+ dir := t.TempDir()
+ found, err := extractRepositoryTarMember(bytes.NewReader(archive.Bytes()), dir)
+ require.NoError(t, err)
+ assert.True(t, found)
+ extracted, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(ancPackageBinaryRelativePath)))
+ require.NoError(t, err)
+ assert.Equal(t, []byte("anc-binary"), extracted)
+ assert.NoFileExists(t, filepath.Join(dir, "etc/not-anc"))
+ assert.NoFileExists(t, filepath.Join(dir, "usr/bin/other"))
+}
+
+func TestExtractRepositoryTarMemberRejectsOversizedANCBinary(t *testing.T) {
+ var archive bytes.Buffer
+ writer := tar.NewWriter(&archive)
+ require.NoError(t, writer.WriteHeader(&tar.Header{
+ Name: "./" + ancPackageBinaryRelativePath,
+ Mode: 0o755,
+ Size: repositoryBinaryMaxBytes + 1,
+ }))
+
+ dir := t.TempDir()
+ found, err := extractRepositoryTarMember(bytes.NewReader(archive.Bytes()), dir)
+ require.Error(t, err)
+ assert.True(t, found)
+ assert.True(t, isIntegrityError(err))
+ assert.NoFileExists(t, filepath.Join(dir, filepath.FromSlash(ancPackageBinaryRelativePath)))
+}
+
+func TestParseMSOSSRepository(t *testing.T) {
+ dir := t.TempDir()
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "azurelinux-ms-oss.repo"), []byte(`
+[azurelinux-official-ms-oss]
+name=Azure Linux Microsoft Open Source
+baseurl=https://repodepot.example/azurelinux/$releasever/prod/ms-oss/$basearch
+gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
+enabled=1
+`), 0o644))
+
+ repository, err := parseMSOSSRepository(dir)
+ require.NoError(t, err)
+ assert.Equal(t, "https://repodepot.example/azurelinux/$releasever/prod/ms-oss/$basearch", repository.BaseURL)
+ assert.Equal(t, []string{"/etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY"}, repository.GPGKeys)
+ assert.Equal(t, "azurelinux-official-ms-oss", repository.Section)
+
+ app := NewTestApp(t, TestAppConfig{}).App
+ app.yumReposDir = dir
+ plan, err := app.rpmRepositoryPlan(platformInfo{
+ OS: "linux", ID: "azurelinux", VersionID: "3.0", Arch: "amd64",
+ }, "202607.20.2")
+ require.NoError(t, err)
+ assert.Equal(t,
+ "https://repodepot.example/azurelinux/3.0/prod/ms-oss/x86_64/Packages/a/aks-node-controller-202607.20.2-1.azl3.x86_64.rpm",
+ plan.packageURL)
+}
+
+func TestRepositoryArchitectureAndReleaseMappings(t *testing.T) {
+ debAMD64, err := debArchitecture("amd64")
+ require.NoError(t, err)
+ assert.Equal(t, "amd64", debAMD64)
+ debARM64, err := debArchitecture("arm64")
+ require.NoError(t, err)
+ assert.Equal(t, "arm64", debARM64)
+
+ rpmAMD64, err := rpmArchitecture("amd64")
+ require.NoError(t, err)
+ assert.Equal(t, "x86_64", rpmAMD64)
+ rpmARM64, err := rpmArchitecture("arm64")
+ require.NoError(t, err)
+ assert.Equal(t, "aarch64", rpmARM64)
+
+ azlSuffix, err := rpmReleaseSuffix(platformInfo{ID: "azurelinux", VersionID: "3.0"})
+ require.NoError(t, err)
+ assert.Equal(t, "azl3", azlSuffix)
+ marinerSuffix, err := rpmReleaseSuffix(platformInfo{ID: "mariner", VersionID: "2.0"})
+ require.NoError(t, err)
+ assert.Equal(t, "cm2", marinerSuffix)
+ _, err = rpmReleaseSuffix(platformInfo{ID: "azurelinux", VersionID: "2.0"})
+ assert.Error(t, err)
+}
+
+func TestRPMMetadataParsing(t *testing.T) {
+ const (
+ primaryLocation = "repodata/abc-primary.xml.gz"
+ packageLocation = "Packages/a/aks-node-controller-202607.20.2-1.azl3.x86_64.rpm"
+ )
+ dir := t.TempDir()
+ repomdPath := filepath.Join(dir, "repomd.xml")
+ primaryPath := filepath.Join(dir, "primary.xml")
+ require.NoError(t, os.WriteFile(repomdPath, []byte(fmt.Sprintf(`
+
+
+ %s
+ %s
+
+ 123
+ 456
+
+`, strings.Repeat("b", 64), strings.Repeat("c", 64), primaryLocation)), 0o644))
+ require.NoError(t, os.WriteFile(primaryPath, []byte(fmt.Sprintf(`
+
+
+ aks-node-controller
+ x86_64
+
+ %s
+
+
+`, strings.Repeat("d", 64), packageLocation)), 0o644))
+
+ reference, err := parsePrimaryReference(repomdPath)
+ require.NoError(t, err)
+ assert.Equal(t, primaryLocation, reference.location)
+ assert.Equal(t, strings.Repeat("b", 64), reference.checksum)
+ assert.Equal(t, strings.Repeat("c", 64), reference.openChecksum)
+ assert.Equal(t, int64(123), reference.size)
+ assert.Equal(t, int64(456), reference.openSize)
+
+ sum, err := parseRPMPrimaryMetadata(
+ primaryPath, "202607.20.2", "1.azl3", "x86_64", packageLocation)
+ require.NoError(t, err)
+ assert.Equal(t, strings.Repeat("d", 64), sum)
+
+ _, err = parseRPMPrimaryMetadata(
+ primaryPath, "202607.20.2", "1.azl3", "x86_64", "Packages/a/wrong.rpm")
+ require.Error(t, err)
+ assert.True(t, isIntegrityError(err))
+}
+
+func TestResolveRPMPackageMetadata(t *testing.T) {
+ const packageLocation = "Packages/a/aks-node-controller-202607.20.2-1.azl3.x86_64.rpm"
+ packageSHA := strings.Repeat("e", 64)
+ primaryXML := []byte(fmt.Sprintf(`
+
+
+ aks-node-controller
+ x86_64
+
+ %s
+
+
+`, packageSHA, packageLocation))
+ var compressed bytes.Buffer
+ gzipWriter := gzip.NewWriter(&compressed)
+ _, err := gzipWriter.Write(primaryXML)
+ require.NoError(t, err)
+ require.NoError(t, gzipWriter.Close())
+ primaryBytes := compressed.Bytes()
+ primaryLocation := "repodata/test-primary.xml.gz"
+ repomd := []byte(fmt.Sprintf(`
+
+
+ %s
+ %s
+
+ %d
+ %d
+
+`,
+ sha256Hex(primaryBytes), sha256Hex(primaryXML), primaryLocation,
+ len(primaryBytes), len(primaryXML)))
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/azurelinux/3.0/prod/ms-oss/x86_64/repodata/repomd.xml":
+ _, _ = w.Write(repomd)
+ case "/azurelinux/3.0/prod/ms-oss/x86_64/repodata/repomd.xml.asc":
+ _, _ = w.Write([]byte("detached-signature"))
+ case "/azurelinux/3.0/prod/ms-oss/x86_64/" + primaryLocation:
+ _, _ = w.Write(primaryBytes)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ origin, err := validateRepositoryURL(
+ server.URL + "/azurelinux/3.0/prod/ms-oss/x86_64")
+ require.NoError(t, err)
+ origin = asRepositoryBase(origin)
+ app := NewTestApp(t, TestAppConfig{}).App
+ app.repositoryTempDir = t.TempDir()
+ app.verifyRepositorySignature = func(
+ _ context.Context, signedPath, signaturePath string, keyrings []string,
+ ) error {
+ assert.Equal(t, []string{"/keys/ms-rpm.gpg"}, keyrings)
+ assert.NotEmpty(t, signedPath)
+ assert.NotEmpty(t, signaturePath)
+ return nil
+ }
+
+ metadata, err := app.resolveRPMPackageMetadata(
+ context.Background(),
+ origin,
+ []string{"/keys/ms-rpm.gpg"},
+ "202607.20.2",
+ "1.azl3",
+ "x86_64",
+ packageLocation,
+ )
+ require.NoError(t, err)
+ assert.Equal(t, packageSHA, metadata.sha256)
+}
+
+func configuredUbuntuRepositoryApp(
+ t *testing.T,
+ dir, serverURL string,
+ runFunc func(*exec.Cmd) error,
+) *App {
+ t.Helper()
+ aptDir := filepath.Join(dir, "sources")
+ require.NoError(t, os.MkdirAll(aptDir, 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(aptDir, "microsoft-prod.list"), []byte(fmt.Sprintf(
+ "deb [arch=amd64 signed-by=/keys/microsoft.gpg] %s/ubuntu/22.04/prod jammy main\n",
+ serverURL)), 0o644))
+ osRelease := filepath.Join(dir, "os-release")
+ require.NoError(t, os.WriteFile(osRelease, []byte("ID=ubuntu\nVERSION_ID=\"22.04\"\n"), 0o644))
+ app := NewTestApp(t, TestAppConfig{RunFunc: runFunc}).App
+ app.aptSourcesDir = aptDir
+ app.osReleasePath = osRelease
+ app.goArch = "amd64"
+ app.repositoryTempDir = dir
+ return app
+}
+
+// clearSignedRelease builds a minimal InRelease. Per the Debian repository format,
+// suiteRelativePath must be relative to the suite directory holding the InRelease (e.g.
+// "main/binary-amd64/Packages"), matching what a real Ubuntu/PMC InRelease publishes --
+// not the repository-root-relative path ("dists//...") used to fetch the file.
+func clearSignedRelease(suiteRelativePath, sum string, size int64) []byte {
+ return []byte(fmt.Sprintf(`-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA256
+
+Origin: Microsoft
+SHA256:
+ %s %d %s
+-----BEGIN PGP SIGNATURE-----
+fake-signature
+-----END PGP SIGNATURE-----
+`, sum, size, suiteRelativePath))
+}
+
+func sha256Hex(data []byte) string {
+ sum := sha256.Sum256(data)
+ return hex.EncodeToString(sum[:])
+}
+
+// A fast package failure must cancel the in-flight metadata branch instead of waiting it
+// out, and the induced cancellation must not be reported as an integrity failure -- that
+// classification would disarm the staged hotfix and skip the package-manager fallback.
+func TestRepositoryFastPathCancelsPeerBranchOnFailure(t *testing.T) {
+ const (
+ hotfixVersion = "202608.21.1"
+ fullVersion = hotfixVersion + "-ubuntu22.04u1"
+ )
+ packageLocation := "pool/main/a/aks-node-controller/aks-node-controller_" +
+ fullVersion + "_amd64.deb"
+
+ metadataCtxDone := make(chan struct{})
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, ".deb") {
+ http.NotFound(w, r) // fails fast, cancelling the metadata branch
+ return
+ }
+ // Stand in for a slow InRelease fetch: block until cancelled, or give up well
+ // before the 30s request timeout so a regression fails loudly instead of hanging.
+ select {
+ case <-r.Context().Done():
+ close(metadataCtxDone)
+ case <-time.After(10 * time.Second):
+ }
+ }))
+ defer server.Close()
+
+ dir := t.TempDir()
+ app := configuredUbuntuRepositoryApp(t, dir, server.URL, func(*exec.Cmd) error {
+ return fmt.Errorf("package manager should not run in this test")
+ })
+ app.vhdBinaryPath = filepath.Join(dir, "aks-node-controller")
+ app.hotfixBinaryPath = filepath.Join(dir, "aks-node-controller-hotfix")
+ require.NoError(t, os.WriteFile(app.vhdBinaryPath, []byte("vhd-binary"), 0o755))
+
+ start := time.Now()
+ err := app.tryRepositoryDownload(context.Background(), hotfixVersion)
+ elapsed := time.Since(start)
+
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), packageLocation[strings.LastIndex(packageLocation, "/")+1:],
+ "the package failure should be reported, not the cancelled peer")
+ assert.False(t, isIntegrityError(err),
+ "a cancelled peer must not be reported as an integrity failure: that would disarm "+
+ "the staged hotfix and skip the package-manager fallback")
+
+ select {
+ case <-metadataCtxDone:
+ default:
+ t.Fatal("metadata branch was not cancelled when the package download failed")
+ }
+ assert.Less(t, elapsed, 5*time.Second,
+ "failure should return promptly rather than waiting out the peer branch")
+}
+
+func TestRepositoryFastPathPrefersIntegrityErrorFromEitherBranch(t *testing.T) {
+ packageRequested := make(chan struct{})
+ var closePackageRequested sync.Once
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, ".deb") {
+ closePackageRequested.Do(func() { close(packageRequested) })
+ http.NotFound(w, r)
+ return
+ }
+ http.NotFound(w, r)
+ }))
+ defer server.Close()
+
+ origin, err := validateRepositoryURL(server.URL)
+ require.NoError(t, err)
+ app := NewTestApp(t, TestAppConfig{}).App
+ app.repositoryTempDir = t.TempDir()
+ _, _, err = app.fetchPackageAndMetadata(context.Background(), repositoryDownloadPlan{
+ packageURL: server.URL + "/aks-node-controller.deb",
+ trustedOrigin: origin,
+ resolveMetadata: func(context.Context) (repositoryPackageMetadata, error) {
+ <-packageRequested
+ return repositoryPackageMetadata{}, newIntegrityError("authenticated metadata checksum mismatch")
+ },
+ })
+ require.Error(t, err)
+ assert.True(t, isIntegrityError(err), "integrity errors must outrank operational package failures")
+ assert.Contains(t, err.Error(), "resolve authenticated repository metadata")
+ assert.Contains(t, err.Error(), "authenticated metadata checksum mismatch")
+}
+
+func TestPreferredRPMExtractionErrorPreservesBothCommandFailures(t *testing.T) {
+ err := preferredRPMExtractionError(nil, errors.New("bad rpm payload"), errors.New("cpio read failed"))
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "rpm2cpio: bad rpm payload")
+ assert.Contains(t, err.Error(), "cpio: cpio read failed")
+}
+
+// Mariner 2.0 has no ms-oss repository -- that path 404s on packages.microsoft.com. Its
+// Microsoft-published packages live in [mariner-microsoft] at .../prod/Microsoft/$basearch
+// (see mariner-package-update.sh, which lists mariner-microsoft.repo). Discovery keyed only
+// on "ms-oss" therefore excluded every Mariner node from the repository fast path.
+func TestParseMicrosoftRepositoryMariner(t *testing.T) {
+ dir := t.TempDir()
+ // Sibling repos that must not be selected, mirroring a real Mariner node.
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "mariner-official-base.repo"), []byte(`
+[mariner-official-base]
+name=CBL-Mariner Official Base
+baseurl=https://packages.microsoft.com/cbl-mariner/$releasever/prod/base/$basearch
+gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
+enabled=1
+`), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "mariner-microsoft.repo"), []byte(`
+[mariner-microsoft]
+name=CBL-Mariner Microsoft
+baseurl=https://packages.microsoft.com/cbl-mariner/$releasever/prod/Microsoft/$basearch
+gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
+enabled=1
+`), 0o644))
+
+ repository, err := parseMSOSSRepository(dir)
+ require.NoError(t, err)
+ assert.Equal(t, "mariner-microsoft", repository.Section)
+ assert.Equal(t,
+ "https://packages.microsoft.com/cbl-mariner/$releasever/prod/Microsoft/$basearch",
+ repository.BaseURL)
+
+ app := NewTestApp(t, TestAppConfig{}).App
+ app.yumReposDir = dir
+ plan, err := app.rpmRepositoryPlan(platformInfo{
+ OS: "linux", ID: "mariner", VersionID: "2.0", Arch: "amd64",
+ }, "202607.20.2")
+ require.NoError(t, err)
+ assert.Equal(t,
+ "https://packages.microsoft.com/cbl-mariner/2.0/prod/Microsoft/x86_64/"+
+ "Packages/a/aks-node-controller-202607.20.2-1.cm2.x86_64.rpm",
+ plan.packageURL)
+ assert.Equal(t, "rpm", plan.format)
+}
+
+// A disabled Microsoft repo must be skipped rather than selected.
+func TestParseMicrosoftRepositorySkipsDisabled(t *testing.T) {
+ dir := t.TempDir()
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "mariner-microsoft.repo"), []byte(`
+[mariner-microsoft]
+baseurl=https://packages.microsoft.com/cbl-mariner/$releasever/prod/Microsoft/$basearch
+gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
+enabled=0
+`), 0o644))
+
+ _, err := parseMSOSSRepository(dir)
+ require.Error(t, err)
+ assert.False(t, isIntegrityError(err), "an absent repository is unsupported, not tampering")
+}
+
+// gzipBytes compresses data the way a repository publishes Packages.gz.
+func gzipBytes(t *testing.T, data []byte) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ writer := gzip.NewWriter(&buf)
+ _, err := writer.Write(data)
+ require.NoError(t, err)
+ require.NoError(t, writer.Close())
+ return buf.Bytes()
+}
+
+// clearSignedReleaseEntries builds an InRelease listing several checksum entries, as a real
+// InRelease does (both Packages and Packages.gz).
+func clearSignedReleaseEntries(entries ...[3]string) []byte {
+ var lines strings.Builder
+ for _, e := range entries {
+ // sum, size, suite-relative path
+ fmt.Fprintf(&lines, " %s %s %s\n", e[0], e[1], e[2])
+ }
+ return []byte(fmt.Sprintf(`-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA256
+
+Origin: Microsoft
+SHA256:
+%s-----BEGIN PGP SIGNATURE-----
+fake-signature
+-----END PGP SIGNATURE-----
+`, lines.String()))
+}
+
+// When the InRelease publishes Packages.gz, the fast path must fetch the compressed index
+// rather than the plain one: for jammy/main/binary-amd64 that is ~720 KB instead of ~4.2 MB,
+// which is what apt itself fetches. Downloading the plain index made the fast path heavier
+// on the wire than the package-manager path it exists to beat.
+func TestUbuntuFastPathPrefersCompressedPackagesIndex(t *testing.T) {
+ const (
+ hotfixVersion = "202608.21.1"
+ fullVersion = hotfixVersion + "-ubuntu22.04u1"
+ )
+ packageBytes := []byte("authenticated-deb-package-bytes")
+ packageLocation := "pool/main/a/aks-node-controller/aks-node-controller_" +
+ fullVersion + "_amd64.deb"
+ packages := []byte(fmt.Sprintf(
+ "Package: aks-node-controller\nVersion: %s\nArchitecture: amd64\nFilename: %s\nSHA256: %s\n\n",
+ fullVersion, packageLocation, sha256Hex(packageBytes)))
+ packagesGz := gzipBytes(t, packages)
+
+ suiteRelative := "main/binary-amd64/Packages"
+ inRelease := clearSignedReleaseEntries(
+ [3]string{sha256Hex(packages), fmt.Sprint(len(packages)), suiteRelative},
+ [3]string{sha256Hex(packagesGz), fmt.Sprint(len(packagesGz)), suiteRelative + ".gz"},
+ )
+
+ var plainFetched, gzFetched atomic.Bool
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/ubuntu/22.04/prod/" + packageLocation:
+ _, _ = w.Write(packageBytes)
+ case "/ubuntu/22.04/prod/dists/jammy/InRelease":
+ _, _ = w.Write(inRelease)
+ case "/ubuntu/22.04/prod/dists/jammy/" + suiteRelative + ".gz":
+ gzFetched.Store(true)
+ _, _ = w.Write(packagesGz)
+ case "/ubuntu/22.04/prod/dists/jammy/" + suiteRelative:
+ plainFetched.Store(true)
+ _, _ = w.Write(packages)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ dir := t.TempDir()
+ app := configuredUbuntuRepositoryApp(t, dir, server.URL, func(*exec.Cmd) error {
+ return fmt.Errorf("package-manager fallback was not expected")
+ })
+ app.vhdBinaryPath = filepath.Join(dir, "aks-node-controller")
+ app.hotfixBinaryPath = filepath.Join(dir, "aks-node-controller-hotfix")
+ require.NoError(t, os.WriteFile(app.vhdBinaryPath, []byte("vhd-binary"), 0o755))
+ app.verifyRepositorySignature = func(context.Context, string, string, []string) error { return nil }
+ app.extractRepositoryPackage = func(
+ _ context.Context, _, _, destination string,
+ ) error {
+ extracted := filepath.Join(destination, filepath.FromSlash(ancPackageBinaryRelativePath))
+ require.NoError(t, os.MkdirAll(filepath.Dir(extracted), 0o755))
+ return os.WriteFile(extracted, []byte("extracted-anc-binary"), 0o644)
+ }
+
+ require.NoError(t, app.tryRepositoryDownload(context.Background(), hotfixVersion))
+
+ assert.True(t, gzFetched.Load(), "the compressed Packages index should have been fetched")
+ assert.False(t, plainFetched.Load(),
+ "the plain Packages index must not be fetched when Packages.gz is published")
+ assert.FileExists(t, app.hotfixBinaryPath)
+}
+
+// Repositories that publish only the plain index must still work.
+func TestUbuntuFastPathFallsBackToPlainPackagesIndex(t *testing.T) {
+ const (
+ hotfixVersion = "202608.21.1"
+ fullVersion = hotfixVersion + "-ubuntu22.04u1"
+ )
+ packageBytes := []byte("authenticated-deb-package-bytes")
+ packageLocation := "pool/main/a/aks-node-controller/aks-node-controller_" +
+ fullVersion + "_amd64.deb"
+ packages := []byte(fmt.Sprintf(
+ "Package: aks-node-controller\nVersion: %s\nArchitecture: amd64\nFilename: %s\nSHA256: %s\n\n",
+ fullVersion, packageLocation, sha256Hex(packageBytes)))
+
+ suiteRelative := "main/binary-amd64/Packages"
+ inRelease := clearSignedRelease(suiteRelative, sha256Hex(packages), int64(len(packages)))
+
+ var plainFetched atomic.Bool
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/ubuntu/22.04/prod/" + packageLocation:
+ _, _ = w.Write(packageBytes)
+ case "/ubuntu/22.04/prod/dists/jammy/InRelease":
+ _, _ = w.Write(inRelease)
+ case "/ubuntu/22.04/prod/dists/jammy/" + suiteRelative:
+ plainFetched.Store(true)
+ _, _ = w.Write(packages)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ dir := t.TempDir()
+ app := configuredUbuntuRepositoryApp(t, dir, server.URL, func(*exec.Cmd) error {
+ return fmt.Errorf("package-manager fallback was not expected")
+ })
+ app.vhdBinaryPath = filepath.Join(dir, "aks-node-controller")
+ app.hotfixBinaryPath = filepath.Join(dir, "aks-node-controller-hotfix")
+ require.NoError(t, os.WriteFile(app.vhdBinaryPath, []byte("vhd-binary"), 0o755))
+ app.verifyRepositorySignature = func(context.Context, string, string, []string) error { return nil }
+ app.extractRepositoryPackage = func(
+ _ context.Context, _, _, destination string,
+ ) error {
+ extracted := filepath.Join(destination, filepath.FromSlash(ancPackageBinaryRelativePath))
+ require.NoError(t, os.MkdirAll(filepath.Dir(extracted), 0o755))
+ return os.WriteFile(extracted, []byte("extracted-anc-binary"), 0o644)
+ }
+
+ require.NoError(t, app.tryRepositoryDownload(context.Background(), hotfixVersion))
+ assert.True(t, plainFetched.Load(), "plain index should be used when no .gz is published")
+}
+
+// A tampered compressed index must be rejected before it is decompressed.
+func TestUbuntuFastPathRejectsTamperedCompressedIndex(t *testing.T) {
+ const (
+ hotfixVersion = "202608.21.1"
+ fullVersion = hotfixVersion + "-ubuntu22.04u1"
+ )
+ packages := []byte("Package: aks-node-controller\nVersion: " + fullVersion + "\n\n")
+ packagesGz := gzipBytes(t, packages)
+ suiteRelative := "main/binary-amd64/Packages"
+ packageLocation := "pool/main/a/aks-node-controller/aks-node-controller_" +
+ fullVersion + "_amd64.deb"
+ // Advertise a checksum that the served bytes will not match.
+ inRelease := clearSignedReleaseEntries(
+ [3]string{strings.Repeat("b", 64), fmt.Sprint(len(packagesGz)), suiteRelative + ".gz"},
+ )
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ // The package must be served successfully: if it 404s, that failure wins the race
+ // and cancels the metadata branch, and the integrity error under test is correctly
+ // discarded as induced noise.
+ case "/ubuntu/22.04/prod/" + packageLocation:
+ _, _ = w.Write([]byte("authenticated-deb-package-bytes"))
+ case "/ubuntu/22.04/prod/dists/jammy/InRelease":
+ _, _ = w.Write(inRelease)
+ case "/ubuntu/22.04/prod/dists/jammy/" + suiteRelative + ".gz":
+ _, _ = w.Write(packagesGz)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ dir := t.TempDir()
+ app := configuredUbuntuRepositoryApp(t, dir, server.URL, func(*exec.Cmd) error {
+ return fmt.Errorf("package-manager fallback was not expected")
+ })
+ app.vhdBinaryPath = filepath.Join(dir, "aks-node-controller")
+ app.hotfixBinaryPath = filepath.Join(dir, "aks-node-controller-hotfix")
+ require.NoError(t, os.WriteFile(app.vhdBinaryPath, []byte("vhd-binary"), 0o755))
+ app.verifyRepositorySignature = func(context.Context, string, string, []string) error { return nil }
+
+ err := app.tryRepositoryDownload(context.Background(), hotfixVersion)
+ require.Error(t, err)
+ assert.True(t, isIntegrityError(err), "a checksum mismatch on the index is an integrity failure")
+ assert.NoFileExists(t, app.hotfixBinaryPath, "no binary should be staged from a tampered index")
+}
+
+// The fast path must request an exact, closed set of URLs: the InRelease, one Packages
+// index, and the single .deb whose path is derived from name+version+arch+codename. This
+// asserts the whole request log, not just that the expected ones appear -- so any extra
+// fetch (a directory listing, a second architecture, a dbgsym or source package, a
+// Release/Release.gpg probe) fails the test rather than passing unnoticed.
+func TestUbuntuFastPathRequestsExactlyTheExpectedURLs(t *testing.T) {
+ const (
+ hotfixVersion = "202608.21.1"
+ fullVersion = hotfixVersion + "-ubuntu22.04u1"
+ )
+ packageBytes := []byte("authenticated-deb-package-bytes")
+ packageLocation := "pool/main/a/aks-node-controller/aks-node-controller_" +
+ fullVersion + "_amd64.deb"
+ packages := []byte(fmt.Sprintf(
+ "Package: aks-node-controller\nVersion: %s\nArchitecture: amd64\nFilename: %s\nSHA256: %s\n\n",
+ fullVersion, packageLocation, sha256Hex(packageBytes)))
+ packagesGz := gzipBytes(t, packages)
+ suiteRelative := "main/binary-amd64/Packages"
+ inRelease := clearSignedReleaseEntries(
+ [3]string{sha256Hex(packages), fmt.Sprint(len(packages)), suiteRelative},
+ [3]string{sha256Hex(packagesGz), fmt.Sprint(len(packagesGz)), suiteRelative + ".gz"},
+ )
+
+ var mu sync.Mutex
+ var requested []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ requested = append(requested, r.Method+" "+r.URL.Path)
+ mu.Unlock()
+ switch r.URL.Path {
+ case "/ubuntu/22.04/prod/" + packageLocation:
+ _, _ = w.Write(packageBytes)
+ case "/ubuntu/22.04/prod/dists/jammy/InRelease":
+ _, _ = w.Write(inRelease)
+ case "/ubuntu/22.04/prod/dists/jammy/" + suiteRelative + ".gz":
+ _, _ = w.Write(packagesGz)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ dir := t.TempDir()
+ app := configuredUbuntuRepositoryApp(t, dir, server.URL, func(*exec.Cmd) error {
+ return fmt.Errorf("package-manager fallback was not expected")
+ })
+ app.vhdBinaryPath = filepath.Join(dir, "aks-node-controller")
+ app.hotfixBinaryPath = filepath.Join(dir, "aks-node-controller-hotfix")
+ require.NoError(t, os.WriteFile(app.vhdBinaryPath, []byte("vhd-binary"), 0o755))
+ app.verifyRepositorySignature = func(context.Context, string, string, []string) error { return nil }
+ app.extractRepositoryPackage = func(_ context.Context, _, _, destination string) error {
+ extracted := filepath.Join(destination, filepath.FromSlash(ancPackageBinaryRelativePath))
+ require.NoError(t, os.MkdirAll(filepath.Dir(extracted), 0o755))
+ return os.WriteFile(extracted, []byte("extracted-anc-binary"), 0o644)
+ }
+
+ require.NoError(t, app.tryRepositoryDownload(context.Background(), hotfixVersion))
+
+ mu.Lock()
+ got := append([]string(nil), requested...)
+ mu.Unlock()
+ sort.Strings(got)
+
+ assert.Equal(t, []string{
+ "GET /ubuntu/22.04/prod/dists/jammy/InRelease",
+ "GET /ubuntu/22.04/prod/dists/jammy/main/binary-amd64/Packages.gz",
+ "GET /ubuntu/22.04/prod/" + packageLocation,
+ }, got, "the fast path must fetch exactly these three URLs and nothing else")
+}
+
+// Azure Linux and Mariner can report a three-part VERSION_ID carrying a build date, while
+// PMC publishes repositories under major.minor only: azurelinux/3.0/prod/... is HTTP 200,
+// azurelinux/3.0.20260304/prod/... is 404. Substituting the raw value silently costs every
+// such node the fast path.
+func TestRPMReleaseVersion(t *testing.T) {
+ tests := []struct {
+ versionID string
+ want string
+ }{
+ {"3.0", "3.0"},
+ {"2.0", "2.0"},
+ // The case that motivated this: a dated VERSION_ID must reduce to the repo path.
+ {"3.0.20260304", "3.0"},
+ {"2.0.20240808", "2.0"},
+ // Must preserve minor rather than forcing ".0": a future 3.1 has to resolve to the
+ // 3.1 repository, not silently to 3.0's.
+ {"3.1", "3.1"},
+ {"3.1.20260101", "3.1"},
+ // Degenerate inputs pass through; rpmReleaseSuffix rejects unsupported majors.
+ {"3", "3"},
+ {"", ""},
+ }
+ for _, tc := range tests {
+ t.Run(tc.versionID, func(t *testing.T) {
+ assert.Equal(t, tc.want, rpmReleaseVersion(tc.versionID))
+ })
+ }
+}
+
+// The plan must build a major.minor repository URL even when the node reports a dated
+// VERSION_ID.
+func TestRPMRepositoryPlanUsesMajorMinorRepoPath(t *testing.T) {
+ dir := t.TempDir()
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "azurelinux-ms-oss.repo"), []byte(`
+[azurelinux-official-ms-oss]
+baseurl=https://packages.microsoft.com/azurelinux/$releasever/prod/ms-oss/$basearch
+gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
+enabled=1
+`), 0o644))
+
+ app := NewTestApp(t, TestAppConfig{}).App
+ app.yumReposDir = dir
+ plan, err := app.rpmRepositoryPlan(platformInfo{
+ OS: "linux", ID: osIDAzureLinux, VersionID: "3.0.20260304", Arch: archAMD64,
+ }, "202607.20.2")
+ require.NoError(t, err)
+ assert.Equal(t,
+ "https://packages.microsoft.com/azurelinux/3.0/prod/ms-oss/x86_64/"+
+ "Packages/a/aks-node-controller-202607.20.2-1.azl3.x86_64.rpm",
+ plan.packageURL)
+}
diff --git a/e2e/scenario_hotfix_bootstrap_perf.go b/e2e/scenario_hotfix_bootstrap_perf.go
new file mode 100644
index 00000000000..66e228dd2c0
--- /dev/null
+++ b/e2e/scenario_hotfix_bootstrap_perf.go
@@ -0,0 +1,155 @@
+package e2e
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "github.com/Azure/agentbaker/e2e/config"
+)
+
+// ancOutputLog is written by the launcher during bootstrap, before the node joins the
+// cluster. Reading timings out of it is what makes this a bootstrap-time measurement:
+// the numbers were produced while cloud-init was still running, contending with image
+// pulls and service startup on a cold apt cache. Timing the same commands over SSH on a
+// Ready node measures a warm, idle machine instead -- a different, much friendlier
+// environment that reads several seconds faster.
+const ancOutputLog = "/var/log/azure/aks-node-controller.output"
+
+// Both hotfix paths emit durationMs on completion, but with distinct messages, so a run
+// cannot be silently misattributed to the wrong path. Note fastPathMsg has pmcMsg as a
+// prefix; check for the fast path first.
+const (
+ hotfixFastPathMsg = "downloaded ANC hotfix through authenticated repository fast path"
+ hotfixPMCMsg = "downloaded ANC hotfix"
+)
+
+var durationMsRe = regexp.MustCompile(`durationMs=(\d+)`)
+
+var errNoHotfixCompletionLine = errors.New("no hotfix completion line found")
+
+// hotfixPathTiming is one observation of a hotfix install during bootstrap.
+type hotfixPathTiming struct {
+ // FastPath is true when the repository fast path completed, false when the run fell
+ // through to apt/dnf.
+ FastPath bool
+ Duration int64
+ Line string
+}
+
+// parseHotfixTiming extracts the hotfix timing from the ANC bootstrap log.
+func parseHotfixTiming(log string) (*hotfixPathTiming, error) {
+ var found *hotfixPathTiming
+ for _, line := range strings.Split(log, "\n") {
+ isFast := strings.Contains(line, hotfixFastPathMsg)
+ isPMC := !isFast && strings.Contains(line, hotfixPMCMsg)
+ if !isFast && !isPMC {
+ continue
+ }
+ match := durationMsRe.FindStringSubmatch(line)
+ if match == nil {
+ return nil, fmt.Errorf("hotfix completion line carries no durationMs: %q", line)
+ }
+ ms, err := strconv.ParseInt(match[1], 10, 64)
+ if err != nil {
+ return nil, fmt.Errorf("parse durationMs from %q: %w", line, err)
+ }
+ found = &hotfixPathTiming{FastPath: isFast, Duration: ms, Line: strings.TrimSpace(line)}
+ }
+ if found == nil {
+ return nil, errNoHotfixCompletionLine
+ }
+ return found, nil
+}
+
+// validateHotfixBootstrapTiming reads the timing the node recorded for itself during
+// bootstrap and reports it. It reads past events from the log rather than re-running any
+// command, so the result is unaffected by the node now being warm and idle.
+//
+// It reports rather than asserting a threshold: a single sample is not a budget, and a
+// flaky perf gate is worse than no gate. Run a scenario a few times and read the values.
+func validateHotfixBootstrapTiming(ctx context.Context, s *Scenario) error {
+ result, err := execScriptOnVMForScenarioValidateExitCode(
+ ctx, s, "sudo cat "+ancOutputLog, 0,
+ "could not read the ANC bootstrap log",
+ )
+ if err != nil {
+ return err
+ }
+
+ timing, err := parseHotfixTiming(result.stdout + "\n" + result.stderr)
+ if errors.Is(err, errNoHotfixCompletionLine) {
+ // Not a failure of the code under test: if no hotfix was configured for this run
+ // there is nothing to time. Log it plainly and pass, rather than failing on absent
+ // data or reporting a misleading zero. Validators no longer control test outcome
+ // (see 70d6199c3e), so this cannot skip the test from here.
+ s.Logger.Logf("NO BOOTSTRAP HOTFIX TIMING: %v (no hotfix ran on this node)", err)
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("parse hotfix bootstrap timing: %w", err)
+ }
+
+ path := "package-manager (apt/dnf)"
+ if timing.FastPath {
+ path = "repository fast path"
+ }
+ s.Logger.Logf("BOOTSTRAP HOTFIX TIMING: distro=%s path=%s durationMs=%d",
+ s.VHD.Name, path, timing.Duration)
+ s.Logger.Logf(" source line: %s", timing.Line)
+ s.Logger.Logf(" measured during provisioning, on a node that had not yet joined")
+
+ // Surface the fallback reason when the fast path did not win, so a slow run can be
+ // explained rather than guessed at.
+ if !timing.FastPath {
+ fallback, ferr := execScriptOnVMForScenario(ctx, s,
+ "sudo grep -F 'falling back to package manager' "+ancOutputLog+" || true")
+ if ferr == nil && strings.TrimSpace(fallback.stdout) != "" {
+ s.Logger.Logf(" fell back because: %s", strings.TrimSpace(fallback.stdout))
+ }
+ }
+ return nil
+}
+
+// Ubuntu2204_HotfixBootstrapTiming records how long the ANC hotfix install takes during
+// real provisioning, on a node that has not yet joined the cluster.
+//
+// Why this exists: timings collected over SSH on an already-Ready node measure a warm apt
+// cache on an idle machine. That produced ~1.7s for scoped apt against ~0.2s for a serial
+// approximation of the fast path, well under the ~10.2s and 22s+ previously reported from
+// AgentBaker e2e. The gap is the environment, not the code. During bootstrap
+// /var/lib/apt/lists may be cold, CPU and IO contend with image pulls and service startup,
+// dpkg may be mid-operation, and networking has just come up. This scenario reads the
+// timing the node recorded while all of that was true, so it is a genuine bootstrap-time
+// measurement rather than an approximation of one.
+//
+// It reports rather than asserting a threshold: a single sample is not a budget, and a
+// flaky perf gate is worse than no gate.
+var _ = Register(&Scenario{
+ Name: "Ubuntu2204_HotfixBootstrapTiming",
+ Description: "records ANC hotfix install duration during bootstrap, before the node joins",
+ Config: Config{
+ Cluster: ClusterKubenet,
+ VHD: config.VHDUbuntu2204Gen2Containerd,
+ Validator: validateHotfixBootstrapTiming,
+ },
+})
+
+// AzureLinuxV3_HotfixBootstrapTiming is the dnf counterpart. Azure Linux is measured
+// separately because it was the slowest observed path: 21.23s on a standalone node against
+// 13.20s for Ubuntu 22.04, both including binary staging. dnf, its metadata handling, and
+// the RPM fast path (repomd.xml plus primary.xml.gz, rather than InRelease plus
+// Packages.gz) are a different code path with a different cost, so an Ubuntu number does
+// not stand in for it.
+var _ = Register(&Scenario{
+ Name: "AzureLinuxV3_HotfixBootstrapTiming",
+ Description: "records ANC hotfix install duration during bootstrap on Azure Linux 3 (dnf)",
+ Config: Config{
+ Cluster: ClusterKubenet,
+ VHD: config.VHDAzureLinuxV3Gen2,
+ Validator: validateHotfixBootstrapTiming,
+ },
+})
diff --git a/e2e/scenario_hotfix_bootstrap_perf_test.go b/e2e/scenario_hotfix_bootstrap_perf_test.go
new file mode 100644
index 00000000000..9c340735034
--- /dev/null
+++ b/e2e/scenario_hotfix_bootstrap_perf_test.go
@@ -0,0 +1,52 @@
+package e2e
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestParseHotfixTiming(t *testing.T) {
+ t.Run("fast path", func(t *testing.T) {
+ log := `time=2026-09-03T01:00:00Z level=INFO msg="downloading ANC hotfix" current=202608.21.0 target=202608.21.1
+time=2026-09-03T01:00:01Z level=INFO msg="downloaded ANC hotfix through authenticated repository fast path" target=202608.21.1 format=deb path=/opt/azure/containers/aks-node-controller-hotfix durationMs=1843`
+ timing, err := parseHotfixTiming(log)
+ require.NoError(t, err)
+ assert.True(t, timing.FastPath)
+ assert.Equal(t, int64(1843), timing.Duration)
+ })
+
+ // The fast-path message contains the package-manager message as a prefix, so a naive
+ // substring check would report a fast-path run as a package-manager run.
+ t.Run("package manager path is not confused with fast path", func(t *testing.T) {
+ log := `time=2026-09-03T01:00:00Z level=WARN msg="safe repository download unavailable, falling back to package manager" version=202608.21.1
+time=2026-09-03T01:00:13Z level=INFO msg="downloaded ANC hotfix" target=202608.21.1 path=/opt/azure/containers/aks-node-controller-hotfix durationMs=13204`
+ timing, err := parseHotfixTiming(log)
+ require.NoError(t, err)
+ assert.False(t, timing.FastPath, "this run fell back and must not be reported as the fast path")
+ assert.Equal(t, int64(13204), timing.Duration)
+ })
+
+ t.Run("no hotfix ran", func(t *testing.T) {
+ _, err := parseHotfixTiming("time=... msg=\"ANC version not targeted by hotfix, skipping download\"")
+ assert.ErrorIs(t, err, errNoHotfixCompletionLine)
+ })
+
+ // A completion line without durationMs means the build predates the instrumentation;
+ // reporting zero would look like an impossibly fast run.
+ t.Run("completion without durationMs is an error, not a zero", func(t *testing.T) {
+ _, err := parseHotfixTiming(`msg="downloaded ANC hotfix" target=202608.21.1`)
+ require.Error(t, err)
+ assert.False(t, errors.Is(err, errNoHotfixCompletionLine))
+ assert.Contains(t, err.Error(), "no durationMs")
+ })
+
+ t.Run("malformed durationMs is an error, not no-completion", func(t *testing.T) {
+ _, err := parseHotfixTiming(`msg="downloaded ANC hotfix" target=202608.21.1 durationMs=abc`)
+ require.Error(t, err)
+ assert.False(t, errors.Is(err, errNoHotfixCompletionLine))
+ assert.Contains(t, err.Error(), "no durationMs")
+ })
+}