diff --git a/.github/workflows/bm_maintenance.yml b/.github/workflows/bm_maintenance.yml index 485bba58437..7cfd22b2203 100644 --- a/.github/workflows/bm_maintenance.yml +++ b/.github/workflows/bm_maintenance.yml @@ -16,9 +16,16 @@ concurrency: cancel-in-progress: true jobs: + nix-build: + uses: ./.github/workflows/nix_build.yml + secrets: + GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }} + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + build-image: name: "Build cleanup-bare-metal image" runs-on: ubuntu-24.04 + needs: nix-build permissions: contents: read packages: write diff --git a/.github/workflows/check_patches.yml b/.github/workflows/check_patches.yml index 79058991d2e..c1bc7d0e030 100644 --- a/.github/workflows/check_patches.yml +++ b/.github/workflows/check_patches.yml @@ -9,8 +9,15 @@ on: - packages/by-name/kata/kata-runtime/0*.patch jobs: + nix-build: + uses: ./.github/workflows/nix_build.yml + secrets: + GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }} + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + check-patches: runs-on: ubuntu-24.04 + needs: nix-build timeout-minutes: 15 permissions: contents: read diff --git a/.github/workflows/docs_publish.yml b/.github/workflows/docs_publish.yml index 21aef525042..9719a897088 100644 --- a/.github/workflows/docs_publish.yml +++ b/.github/workflows/docs_publish.yml @@ -27,9 +27,16 @@ permissions: pull-requests: write jobs: + nix-build: + uses: ./.github/workflows/nix_build.yml + secrets: + GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }} + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + deploy: name: Publish docs to GitHub Pages runs-on: ubuntu-24.04 + needs: nix-build # Prevent the job from running on forked PRs, for security reasons. if: | (github.event_name == 'push' && github.ref_name == 'main') || diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 31623339344..f413cba984b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -38,9 +38,16 @@ env: DO_NOT_TRACK: 1 jobs: + nix-build: + uses: ./.github/workflows/nix_build.yml + secrets: + GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }} + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + test: name: "${{ inputs.test-name }}${{ inputs.debug-shell && ' (with debug shell)' || '' }}" runs-on: ${{ inputs.runner }} + needs: nix-build permissions: contents: read packages: write diff --git a/.github/workflows/imagepuller-benchmark.yml b/.github/workflows/imagepuller-benchmark.yml index 81004fb05ae..12e996812ab 100644 --- a/.github/workflows/imagepuller-benchmark.yml +++ b/.github/workflows/imagepuller-benchmark.yml @@ -12,8 +12,15 @@ on: - main jobs: + nix-build: + uses: ./.github/workflows/nix_build.yml + secrets: + GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }} + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + run-benchmark: runs-on: ubuntu-24.04 + needs: nix-build permissions: contents: read packages: read diff --git a/.github/workflows/k3s_compatibility.yml b/.github/workflows/k3s_compatibility.yml index 219828ecf93..294df17656f 100644 --- a/.github/workflows/k3s_compatibility.yml +++ b/.github/workflows/k3s_compatibility.yml @@ -12,8 +12,15 @@ env: container_registry: ghcr.io/edgelesssys jobs: + nix-build: + uses: ./.github/workflows/nix_build.yml + secrets: + GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }} + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + test: runs-on: ubuntu-24.04 + needs: nix-build permissions: contents: read packages: write diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index ccafd8bc69b..61b1b4b6e96 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -9,9 +9,16 @@ on: - ".github/workflows/links.yml" jobs: + nix-build: + uses: ./.github/workflows/nix_build.yml + secrets: + GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }} + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + check-links: name: check external links runs-on: ubuntu-24.04 + needs: nix-build timeout-minutes: 15 permissions: contents: read diff --git a/.github/workflows/nix_build.yml b/.github/workflows/nix_build.yml new file mode 100644 index 00000000000..24cab01689c --- /dev/null +++ b/.github/workflows/nix_build.yml @@ -0,0 +1,31 @@ +name: build and cache all flake outputs + +on: + workflow_call: + secrets: + GITHUB_TOKEN_IN: + required: true + CACHIX_AUTH_TOKEN: + required: true + +jobs: + nix-build: + runs-on: ${{ matrix.runner }} + permissions: + contents: read + strategy: + matrix: + include: + - runner: yet-to-be-set-up-nix-build-host + system: x86_64-linux + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/setup_nix + with: + githubToken: ${{ secrets.GITHUB_TOKEN_IN || secrets.GITHUB_TOKEN }} + cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + - name: pre-build all flake outputs + run: | + nix build .#matrix.${{ matrix.system }} diff --git a/.github/workflows/pr_release_artifacts.yml b/.github/workflows/pr_release_artifacts.yml index 38c6d85da8b..d22897a8ebd 100644 --- a/.github/workflows/pr_release_artifacts.yml +++ b/.github/workflows/pr_release_artifacts.yml @@ -12,10 +12,17 @@ on: default: false jobs: + nix-build: + uses: ./.github/workflows/nix_build.yml + secrets: + GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }} + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + create-release-artifacts: name: Create release artifacts for PR if: github.event_name == 'workflow_dispatch' && github.event.inputs.cleanup == 'false' runs-on: ubuntu-24.04 + needs: nix-build permissions: pull-requests: write issues: write diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 732be4aa440..bf8f037dc18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,10 +119,16 @@ jobs: exit 1 fi + nix-build: + uses: ./.github/workflows/nix_build.yml + secrets: + GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }} + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + update-main: name: Update main branch runs-on: ubuntu-24.04 - needs: [process-inputs, release] + needs: [process-inputs, release, nix-build] permissions: contents: write env: @@ -220,7 +226,7 @@ jobs: release: name: Build and push artifacts, create release runs-on: ubuntu-24.04 - needs: process-inputs + needs: [process-inputs, nix-build] permissions: contents: write packages: write @@ -311,7 +317,7 @@ jobs: # Job needs content:write to see draft releases. contents: write packages: read - needs: [process-inputs, release] + needs: [process-inputs, release, nix-build] env: VERSION: ${{ inputs.version }} steps: diff --git a/.github/workflows/rim_updates.yml b/.github/workflows/rim_updates.yml index 8130b3797e7..0f2efb19ff3 100644 --- a/.github/workflows/rim_updates.yml +++ b/.github/workflows/rim_updates.yml @@ -6,9 +6,16 @@ on: - cron: "0 18 * * 0" jobs: + nix-build: + uses: ./.github/workflows/nix_build.yml + secrets: + GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }} + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + update-rim-ids: name: update NVIDIA RIM IDs runs-on: ubuntu-24.04 + needs: nix-build timeout-minutes: 20 permissions: contents: read diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml index 907b1bb9bf4..d70951fc7a4 100644 --- a/.github/workflows/static.yml +++ b/.github/workflows/static.yml @@ -8,8 +8,15 @@ on: pull_request: jobs: + nix-build: + uses: ./.github/workflows/nix_build.yml + secrets: + GITHUB_TOKEN_IN: ${{ secrets.GITHUB_TOKEN_IN }} + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + flake-check: runs-on: ubuntu-24.04 + needs: nix-build timeout-minutes: 15 permissions: contents: read @@ -27,6 +34,7 @@ jobs: generate: runs-on: ubuntu-24.04 + needs: nix-build timeout-minutes: 60 permissions: contents: write @@ -52,6 +60,7 @@ jobs: govulncheck: runs-on: ubuntu-24.04 + needs: nix-build timeout-minutes: 15 permissions: contents: read @@ -69,6 +78,7 @@ jobs: golangci-lint: runs-on: ubuntu-24.04 + needs: nix-build timeout-minutes: 15 permissions: contents: read @@ -86,6 +96,7 @@ jobs: go-licenses: runs-on: ubuntu-24.04 + needs: nix-build timeout-minutes: 15 permissions: contents: read diff --git a/flake.nix b/flake.nix index 52662d61784..9e4185b0559 100644 --- a/flake.nix +++ b/flake.nix @@ -36,12 +36,13 @@ overlays = [ (final: _prev: { fenix = self.inputs.fenix.packages.${final.stdenv.hostPlatform.system}; }) (import ./overlays/nixpkgs.nix) - (import ./overlays/contrast.nix) + (import ./overlays/contrast.nix { inherit (self) inputs; }) ]; config.allowUnfree = true; config.nvidia.acceptLicense = true; }; treefmtEval = treefmt-nix.lib.evalModule pkgs ./treefmt.nix; + inherit (pkgs) lib; in { @@ -57,6 +58,8 @@ "contrastPkgs" ]; }; + + matrix = pkgs.writeText "output-matrix.json" (builtins.toJSON (lib.ci.allOutputs self system)); } ); diff --git a/lib/ci.nix b/lib/ci.nix new file mode 100644 index 00000000000..51d4b083076 --- /dev/null +++ b/lib/ci.nix @@ -0,0 +1,22 @@ +{ lib, ... }: +rec { + fromPackageOutputs = + flake: system: + lib.concatMap (kind: lib.attrValues (lib.attrByPath [ kind system ] { } flake)) [ + "legacyPackages" + "packages" + "checks" + "devShells" + "formatters" + ]; + + allOutputs = + flake: system: + lib.filter lib.isDerivation ( + lib.unique ( + lib.concatMap (from: from flake system) [ + fromPackageOutputs + ] + ) + ); +} diff --git a/lib/default.nix b/lib/default.nix new file mode 100644 index 00000000000..3f4a597388f --- /dev/null +++ b/lib/default.nix @@ -0,0 +1,12 @@ +{ inputs }: + +let + mkLib = + nixpkgs: + nixpkgs.lib.extend ( + self: _: { + ci = import ./ci.nix { lib = self; }; + } + ); +in +mkLib inputs.nixpkgs diff --git a/overlays/contrast.nix b/overlays/contrast.nix index e57d1ceb0f6..d45c6e59abd 100644 --- a/overlays/contrast.nix +++ b/overlays/contrast.nix @@ -1,8 +1,11 @@ # Copyright 2024 Edgeless Systems GmbH # SPDX-License-Identifier: BUSL-1.1 +{ inputs }: + final: _prev: { contrastPkgs = import ../packages { pkgs = final; }; + lib = import ../lib { inherit inputs; }; } diff --git a/packages/contrast-releases.json b/packages/contrast-releases.json index c3925350a26..b57528ccd90 100644 --- a/packages/contrast-releases.json +++ b/packages/contrast-releases.json @@ -1,9 +1,5 @@ { "contrast": [ - { - "version": "v0.2.0", - "hash": "sha256-cGyWvUmL/rbjX3Bu1OPGTZlk5PgVW/O0qdo9KmZPo+U=" - }, { "version": "v0.3.0", "hash": "sha256-8yVZLhwf2bsWIMxSAqXUKDNCjltCSGA6bwrXTi+cJB0=" diff --git a/packages/scripts.nix b/packages/scripts.nix index 4f9e4e12ea6..831c5d82cd4 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -809,4 +809,46 @@ lib.makeScope pkgs.newScope (scripts: { echo "$timestamp $digest" ''; }; + + # Shows which derivations' dependency changes cause rebuilds + show-input-diff = writeShellApplication { + name = "show-input-diff"; + runtimeInputs = with pkgs; [ + nix-diff + jq + ]; + text = '' + maxDepth=999 + new_args=() + while [[ $# -gt 0 ]]; do + case $1 in + --max-depth) + maxDepth="$2" + shift 2 + ;; + --max-depth=*) + maxDepth="''${1#*=}" + shift + ;; + *) + new_args+=("$1") + shift + ;; + esac + done + set -- "''${new_args[@]}" + + left=$(nix eval --raw "''${1:-"github:edgelesssys/contrast#matrix.x86_64-linux"}.drvPath" | tr -d '\n') + right=$(nix eval --raw "''${2:-".#matrix.x86_64-linux"}.drvPath" | tr -d '\n') + nix-diff "$left" "$right" --json | jq -r --argjson maxDepth "$maxDepth" ' + def printTree(level): + ( + select(.drvName != null and .drvName != "") | (" " * level) + .drvName, + (.drvNames // [] | .[] | (" " * (level + 1)) + .), + (if level + 1 < $maxDepth then (.drvDiff.inputsDiff.inputDerivationDiffs // [] | .[] | printTree(level + 1)) else empty end) + ); + .inputsDiff.inputDerivationDiffs[] | printTree(0) + ' + ''; + }; }) diff --git a/packages/static.nix b/packages/static.nix new file mode 100644 index 00000000000..ebeb3a7ef74 --- /dev/null +++ b/packages/static.nix @@ -0,0 +1,16 @@ +# Copyright 2025 Edgeless Systems GmbH +# SPDX-License-Identifier: BUSL-1.1 + +{ pkgs }: + +let + inherit (pkgs.lib) makeScope; +in + +makeScope pkgs.pkgsStatic.newScope ( + self: + pkgs.lib.packagesFromDirectoryRecursive { + inherit (self) callPackage newScope; + directory = ./by-name; + } +)