TEMPORARY, do not merge: measure the Hermes image's push two ways (FIN-360 follow-up)

.github/actions/free-disk/action.yml

@@ -0,0 +1,29 @@
+# The runner's disk freed before a job builds the Hermes image. Some
+# ubuntu-24.04 runners (image 20261018.96.1) have a 72 GB root, about
+# 12 GB free, which the image (held twice where BuildKit's container
+# builds it and `--load` copies it to Docker) and its build cache fill:
+# the Docker lane's tests ran out (runs 41249649091, 41253027860; PR
+# #890), and so did the merge queue's preview image, its runner's own log
+# "No space left on device" (runs 41265350002, 41266642000; FIN-360).
+# Most have 145 GB. None of these jobs uses the image's Android (12 GB),
+# .NET (5.8), Haskell (3.7), Swift (3.5) or CodeQL (1.4), nor its cached
+# Docker images (2.5): they go first, 28 GB in about 90 seconds. The
+# disk's layout is said, as it differs between runners, and the root's use
+# and the memory's are sampled every 5 seconds into
+# `$RUNNER_TEMP/disk-used` (MiB, one line each: disk, then memory), whose
+# peaks the job's `disk, after` says.
+name: free the disk
+description: the runner's unused toolchains and cached Docker images deleted, its disk and memory then sampled
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      run: |
+        df -h /
+        free -h
+        lsblk -o NAME,SIZE,TYPE,MOUNTPOINT
+        sudo dmsetup status 2>/dev/null || true
+        sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /usr/share/swift /opt/hostedtoolcache/CodeQL
+        docker image prune -af > /dev/null
+        df -h /
+        nohup sh -c 'while :; do echo "$(df -B1M --output=used / | tail -1 | tr -d " ") $(free -m | awk "/^Mem:/ { print \$3 }")"; sleep 5; done' > "$RUNNER_TEMP/disk-used" 2>&1 &

.github/workflows/ci.yml

@@ -394,6 +394,10 @@ jobs:
         run: |
           cargo xtask image preview --output "$RUNNER_TEMP/preview-image.json"
           echo "build=$(jq -r .build "$RUNNER_TEMP/preview-image.json")" >> "$GITHUB_OUTPUT"
+      # its disk freed before it builds, as the merge queue's build's is
+      # (the action says why)
+      - if: steps.probe.outputs.build == 'true'
+        uses: ./.github/actions/free-disk
       - id: buildx
         if: steps.probe.outputs.build == 'true'
         uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
@@ -420,6 +424,11 @@ jobs:
           cargo xtask image preview --build --output "$RUNNER_TEMP/preview-image.json" --summary "$GITHUB_STEP_SUMMARY"
           echo "digest=$(jq -er .digest "$RUNNER_TEMP/preview-image.json")" >> "$GITHUB_OUTPUT"
           echo "key=$(jq -er .key "$RUNNER_TEMP/preview-image.json")" >> "$GITHUB_OUTPUT"
+      - if: always() && steps.probe.outputs.build == 'true'
+        name: disk, after
+        run: |
+          docker system df; df -h /; free -h
+          echo "peak: $(awk '$1 > d { d = $1 } END { print d }' "$RUNNER_TEMP/disk-used") MiB of the disk used, $(awk '$2 > m { m = $2 } END { print m }' "$RUNNER_TEMP/disk-used") MiB of the memory"
 
   # Continuous deployment, on a green push: master's to the preview
   # (`--branch preview`), production's to production, after `check` and

.github/workflows/images.yml

@@ -167,25 +167,8 @@ jobs:
         working-directory: images
     steps:
       - uses: actions/checkout@v4
-      # Some ubuntu-24.04 runners (image 20261018.96.1) have a 72 GB root,
-      # 12 GB free once the lane's test binary is built, which its two
-      # images (6.4 GB) and their build cache (2.7 GB) all but fill: runs
-      # 41249649091 and 41253027860 ran out of disk. Most have 145 GB, 86 GB
-      # free. The lane uses none of the image's Android (12 GB), .NET (5.8),
-      # Haskell (3.7), Swift (3.5) or CodeQL (1.4), nor its cached Docker
-      # images (2.5): they go first, 28 GB in about 90 seconds (PR #890).
-      # Its disk's layout is said, as it differs between runners.
-      - name: free the disk
-        working-directory: .
-        run: |
-          df -h /
-          lsblk -o NAME,SIZE,TYPE,MOUNTPOINT
-          sudo dmsetup status 2>/dev/null || true
-          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /usr/share/swift /opt/hostedtoolcache/CodeQL
-          docker image prune -af > /dev/null
-          df -h /
-          # the root's use every 5 seconds, whose peak `disk, after` says
-          nohup sh -c 'while :; do df -B1M --output=used / | tail -1; sleep 5; done' > "$RUNNER_TEMP/disk-used" 2>&1 &
+      # its disk freed first (the action says why: PR #890), and sampled
+      - uses: ./.github/actions/free-disk
       - uses: dtolnay/rust-toolchain@stable
       # the check job's cache, read only
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
@@ -217,8 +200,8 @@ jobs:
       - if: always()
         name: disk, after
         run: |
-          docker system df; df -h /
-          echo "peak: $(sort -n "$RUNNER_TEMP/disk-used" | tail -1) MiB used"
+          docker system df; df -h /; free -h
+          echo "peak: $(awk '$1 > d { d = $1 } END { print d }' "$RUNNER_TEMP/disk-used") MiB of the disk used, $(awk '$2 > m { m = $2 } END { print m }' "$RUNNER_TEMP/disk-used") MiB of the memory"
       # each container's log (Container's drop) and the tests' own
       - if: failure()
         uses: actions/upload-artifact@v4

.github/workflows/measure-image-push.yml

@@ -0,0 +1,79 @@
+# TEMPORARY (FIN-360's follow-up; a draft PR, never merged): the Hermes
+# image's push measured two ways, on the runner the preview image's
+# prebuild gets, against a registry on the runner itself.
+# A, as `--queue` does now: `docker build --load` into Docker's store, then
+#   `docker push` from there.
+# B, the cut: `docker buildx build --push` straight from BuildKit's builder.
+name: measure-image-push
+
+on:
+  pull_request:
+    paths: [".github/workflows/measure-image-push.yml"]
+
+jobs:
+  measure:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/free-disk
+      - run: docker run -d --name registry --restart always -p 5000:5000 registry:2
+      - id: buildx
+        uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
+        with:
+          driver-opts: network=host
+      - name: warm the builder, as the prebuild's build-push step does
+        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+        env:
+          DOCKER_BUILD_SUMMARY: false
+          DOCKER_BUILD_RECORD_UPLOAD: false
+        with:
+          context: .
+          file: images/hermes/Dockerfile
+          platforms: linux/amd64
+          provenance: false
+          cache-from: type=gha,scope=preview-hermes
+      - name: before
+        run: |
+          df -B1M --output=used / | tail -1 | tr -d ' ' > "$RUNNER_TEMP/before"
+          docker system df
+          echo "builder: $(docker buildx du --builder ${{ steps.buildx.outputs.name }} 2>/dev/null | tail -1)"
+      - name: "A: build --load, then push from Docker"
+        env:
+          BUILDX_BUILDER: ${{ steps.buildx.outputs.name }}
+        run: |
+          t0=$(date +%s)
+          docker build --load --platform linux/amd64 --provenance=false -t localhost:5000/hermes:a -f images/hermes/Dockerfile .
+          t1=$(date +%s)
+          docker push localhost:5000/hermes:a > /dev/null
+          t2=$(date +%s)
+          used=$(df -B1M --output=used / | tail -1 | tr -d ' ')
+          echo "A: build --load $((t1 - t0)) s, push $((t2 - t1)) s, total $((t2 - t0)) s; disk +$((used - $(cat "$RUNNER_TEMP/before"))) MiB (Docker's store and the registry)"
+          docker system df
+          docker image inspect localhost:5000/hermes:a --format 'image: {{.Size}} bytes'
+          echo "$used" > "$RUNNER_TEMP/after-a"
+      - name: "Docker's copy and the registry's, gone"
+        run: |
+          docker image rm localhost:5000/hermes:a > /dev/null
+          docker rm -f registry > /dev/null && docker volume prune -af > /dev/null
+          docker run -d --name registry --restart always -p 5000:5000 registry:2
+          df -B1M --output=used / | tail -1 | tr -d ' ' > "$RUNNER_TEMP/before-b"
+          echo "back to $(cat "$RUNNER_TEMP/before-b") MiB used (before A: $(cat "$RUNNER_TEMP/before"))"
+      - name: "B: push straight from BuildKit"
+        env:
+          BUILDX_BUILDER: ${{ steps.buildx.outputs.name }}
+        run: |
+          t0=$(date +%s)
+          docker buildx build --push --platform linux/amd64 --provenance=false -t localhost:5000/hermes:b --metadata-file "$RUNNER_TEMP/meta.json" -f images/hermes/Dockerfile .
+          t1=$(date +%s)
+          used=$(df -B1M --output=used / | tail -1 | tr -d ' ')
+          echo "B: build --push $((t1 - t0)) s; disk +$((used - $(cat "$RUNNER_TEMP/before-b"))) MiB (the registry alone)"
+          echo "digest: $(jq -r '."containerimage.digest"' "$RUNNER_TEMP/meta.json")"
+          docker system df
+      - if: always()
+        name: disk, after
+        run: |
+          df -h /; free -h
+          echo "peak: $(awk '$1 > d { d = $1 } END { print d }' "$RUNNER_TEMP/disk-used") MiB of the disk used, $(awk '$2 > m { m = $2 } END { print m }' "$RUNNER_TEMP/disk-used") MiB of the memory"

.github/workflows/preview-image.yml

@@ -62,6 +62,11 @@ jobs:
         run: |
           cargo xtask image preview --output "$RUNNER_TEMP/preview-image.json"
           echo "build=$(jq -r .build "$RUNNER_TEMP/preview-image.json")" >> "$GITHUB_OUTPUT"
+      # its disk freed before it builds: the image, held by BuildKit's
+      # builder and copied to Docker by `--queue`, filled a 72 GB runner (the
+      # action says)
+      - if: steps.probe.outputs.build == 'true'
+        uses: ./.github/actions/free-disk
       - id: buildx
         if: steps.probe.outputs.build == 'true'
         uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
@@ -80,3 +85,8 @@ jobs:
         env:
           BUILDX_BUILDER: ${{ steps.buildx.outputs.name }}
         run: cargo xtask image preview --queue --output "$RUNNER_TEMP/preview-image.json"
+      - if: always() && steps.probe.outputs.build == 'true'
+        name: disk, after
+        run: |
+          docker system df; df -h /; free -h
+          echo "peak: $(awk '$1 > d { d = $1 } END { print d }' "$RUNNER_TEMP/disk-used") MiB of the disk used, $(awk '$2 > m { m = $2 } END { print m }' "$RUNNER_TEMP/disk-used") MiB of the memory"