dev: build using plain-ish build.rs

.devin/wiki.json

@@ -86,7 +86,7 @@
     },
     {
       "title": "Crate Structure",
-      "purpose": "Document the Rust workspace layout: each crate (cadmus-core, cadmus, emulator, importer, fetcher, xtask), their roles, the binaries they produce, and the inter-crate dependency graph. Cover shared workspace dependencies such as rustls and the release-minsized build profile.",
+      "purpose": "Document the Rust workspace layout: each crate (cadmus-core, cadmus, emulator, importer, fetcher, xtask, build-deps), their roles, the binaries they produce, and the inter-crate dependency graph. Cover shared workspace dependencies such as rustls and the release-minsized build profile. The build-deps crate is shared between cadmus-core's build.rs and xtask, and owns all thirdparty C/C++ build orchestration (native and Kobo recipes, MuPDF source preparation, mupdf_wrapper compilation).",
       "parent": "Contributor Guide",
       "page_notes": [
         {
@@ -96,7 +96,7 @@
     },
     {
       "title": "Development Environment",
-      "purpose": "Document the devenv and Nix-based reproducible development environment: entering the shell, available cargo xtask commands (setup-native, run-emulator, build-kobo, dist, bundle, test, docs), devenv tasks (deps:native, docs:build, build:kobo), the integrated observability stack (OTel Collector, Prometheus, Tempo, Loki, Grafana), platform support (Linux full, macOS native-only), and local override via devenv.local.nix.",
+      "purpose": "Document the devenv and Nix-based reproducible development environment: entering the shell, available cargo xtask commands (run-emulator, build-kobo, dist, bundle, test, docs, download-assets), devenv tasks (docs:build, build:kobo), the integrated observability stack (OTel Collector, Prometheus, Tempo, Loki, Grafana), platform support (Linux full, macOS full including Kobo cross-compilation), and local override via devenv.local.nix. Thirdparty native dependencies are built lazily by the cadmus-core build.rs (via the build-deps crate) the first time `cargo build` runs, so no explicit setup step is required.",
       "parent": "Contributor Guide",
       "page_notes": [
         {
@@ -176,7 +176,7 @@
     },
     {
       "title": "Building for Device",
-      "purpose": "Explain the end-to-end process for cross-compiling and packaging Cadmus for a target device: the ARM cross-compilation toolchain, cargo xtask build-kobo (fast vs slow modes, mupdf_wrapper compilation), cargo xtask dist assembly, cargo xtask bundle packaging, NickelMenu integration, and the device management shell scripts (wifi-enable.sh, wifi-disable.sh, usb-enable.sh, usb-disable.sh). Currently Kobo-specific; the build system is expected to generalise as additional devices are supported.",
+      "purpose": "Explain the end-to-end process for cross-compiling and packaging Cadmus for a target device: the ARM cross-compilation toolchain, cargo xtask build-kobo (a thin wrapper over `cargo build --release --target arm-unknown-linux-gnueabihf` whose thirdparty native artifacts are produced lazily by the cadmus-core build.rs via the build-deps crate), cargo xtask dist assembly, cargo xtask bundle packaging, NickelMenu integration, and the device management shell scripts (wifi-enable.sh, wifi-disable.sh, usb-enable.sh, usb-disable.sh). Currently Kobo-specific; the build system is expected to generalise as additional devices are supported.",
       "parent": "Contributor Guide",
       "page_notes": [
         {
@@ -186,7 +186,7 @@
     },
     {
       "title": "Third-Party Dependencies",
-      "purpose": "Document the thirdparty/ directory: its role in the cross-compilation pipeline, how library sources are downloaded at build time (not vendored), the dependency-ordered build process via build-kobo.sh scripts, which libraries carry Kobo-specific patches and the rationale behind each patch (e.g. bypassing pkg-config for cross-compilation, stripping unnecessary build targets, adding a device-specific cross-compile target), the relationship between version constants in xtask and Renovate-managed automated updates, and guidance on adding or upgrading a library.",
+      "purpose": "Document the thirdparty/ directory: its role in the cross-compilation pipeline, how library sources are vendored as git submodules and built from source by the build-deps crate (invoked from the cadmus-core build.rs and from xtask), the dependency-ordered build process via per-library recipes in build-deps (build/kobo/recipes.rs for ARM, build/native.rs for the host), which libraries carry Kobo-specific patches under build-scripts/ and the rationale behind each patch (e.g. bypassing pkg-config for cross-compilation, stripping unnecessary build targets, adding a device-specific cross-compile target), the relationship between version constants in build-deps (versions.rs) and Renovate-managed automated updates (including the git-submodules manager for submodule SHAs), and guidance on adding or upgrading a library.",
       "parent": "Contributor Guide",
       "page_notes": [
         {

.github/actions/build-kobo/action.yml

@@ -23,18 +23,6 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Cache Cargo registry
-      id: rust-toolchain-cache
-      uses: actions/cache@v5
-      with:
-        path: |
-          ~/.cargo/bin/
-          ~/.cargo/registry/index/
-          ~/.cargo/registry/cache/
-          ~/.cargo/git/db/
-          target/
-        key: ${{ runner.os }}-cargo-build-kobo-${{ hashFiles('**/Cargo.lock') }}
-
     - name: Cache Linaro toolchain
       id: cache-linaro
       uses: actions/cache@v5
@@ -52,11 +40,6 @@ runs:
         tar xf gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz --strip-components=1
         rm gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz
 
-    - name: Cache thirdparty libraries
-      uses: ./.github/actions/cache-thirdparty
-      with:
-        kind: kobo-build
-
     - name: Cache NickelMenu downloads
       uses: actions/cache@v5
       with:
@@ -88,9 +71,9 @@ runs:
         GH_OAUTH_CLIENT_ID: ${{ inputs.gh-oauth-client-id }}
       run: |
         if [ "${{ inputs.test }}" = "true" ]; then
-          cargo xtask build-kobo --slow --features test
+          cargo xtask build-kobo --features test
         else
-          cargo xtask build-kobo --slow
+          cargo xtask build-kobo
         fi
 
     - name: Create distribution

.github/workflows/build-release-candidate.yml

@@ -43,9 +43,24 @@ jobs:
           ref: ${{ steps.pr.outputs.branch }}
 
       - uses: dtolnay/rust-toolchain@stable
+        id: rust-toolchain
         with:
           targets: arm-unknown-linux-gnueabihf
 
+      - name: Restore shared cargo cache
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+            libs/
+          key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-
+
       - name: Build Kobo artifacts
         uses: ./.github/actions/build-kobo
         with:

.github/workflows/cadmus-docs.yml

@@ -47,6 +47,7 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
       - uses: dtolnay/rust-toolchain@stable
+        id: rust-toolchain
 
       - name: Setup Python
         uses: actions/setup-python@v6
@@ -71,17 +72,24 @@ jobs:
         with:
           node-version: ${{ env.NODE_VERSION }}
 
-      - name: Cache cargo doc
-        uses: actions/cache@v5
+      - name: Restore shared cargo artifacts
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae
         with:
-          path: target/doc
-          key: ${{ runner.os }}-cargo-doc-${{ hashFiles('**/Cargo.lock') }}
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+            libs/
+          key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }}
           restore-keys: |
-            ${{ runner.os }}-cargo-doc-
+            ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-
 
       - name: Build documentation portal
         env:
           RUSTDOCFLAGS: -D warnings
+          CADMUS_SKIP_THIRDPARTY_DEPS: "1"
         run: cargo xtask docs --base-url "$ZOLA_BASE_URL"
 
       - name: Upload build artifact

.github/workflows/cargo.yml

@@ -1,6 +1,7 @@
 name: Cargo
 
 env:
+  NEXTEST_VERSION: "0.9.94"
   NODE_VERSION: "24"
   SQLX_OFFLINE: true
 
@@ -16,8 +17,7 @@ on:
       - ".github/actions/build-kobo/**"
       - ".github/actions/build-docs-epub/**"
       - ".github/actions/install-doc-tools/**"
-      - "thirdparty/**/*.patch"
-      - "thirdparty/**/*.patch"
+      - "build-scripts/**"
 
 concurrency:
   group: "cargo-${{ github.event_name }}-${{ github.event.number || github.ref }}"
@@ -36,15 +36,16 @@ jobs:
           components: rustfmt
       - &restore-shared-cache
         name: Restore shared cargo cache
-        uses: actions/cache/restore@v5
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae
         with:
           path: |
             ~/.cargo/bin/
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock') }}
+            libs/
+          key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }}
           restore-keys: |
             ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-
       - name: Check formatting
@@ -89,47 +90,85 @@ jobs:
       - uses: actions/checkout@v6
       - uses: dtolnay/rust-toolchain@stable
         id: rust-toolchain
-      - name: Set build metadata
+        with:
+          targets: arm-unknown-linux-gnueabihf
+
+      - &set-build-metadata
+        name: Set build metadata
         run: |
           if [ "${{ github.event_name }}" = "pull_request" ]; then
             echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> "$GITHUB_ENV"
             echo "PR_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV"
           fi
           echo "GIT_VERSION=$(git describe --tags --dirty --always)" >> "$GITHUB_ENV"
+
       - name: Install apt packages
-        uses: awalsh128/cache-apt-pkgs-action@latest
+        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05
         with:
           packages: git wget curl pkg-config gcc g++ make cmake meson ninja-build autoconf automake libtool gperf python3 zlib1g-dev libbz2-dev libpng-dev libjpeg-dev libopenjp2-7-dev libjbig2dec0-dev libgumbo-dev libfreetype6-dev libharfbuzz-dev libdjvulibre-dev libsdl2-dev
           version: 1.0
-      - name: Cache thirdparty libraries
-        uses: ./.github/actions/cache-thirdparty
+      - name: Cache Linaro toolchain
+        id: cache-linaro
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae
         with:
-          kind: native
+          path: ~/linaro-toolchain
+          key: linaro-gcc-4.9.4-2017.01
+      - name: Download Linaro toolchain
+        if: steps.cache-linaro.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p ~/linaro-toolchain
+          cd ~/linaro-toolchain
+          wget -q https://releases.linaro.org/components/toolchain/binaries/4.9-2017.01/arm-linux-gnueabihf/gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz
+          tar xf gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz --strip-components=1
+          rm gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz
+      - name: Setup Linaro toolchain
+        run: echo "$HOME/linaro-toolchain/bin" >> "$GITHUB_PATH"
       - name: Cache shared cargo artifacts
         id: cache
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae
         with:
           path: |
             ~/.cargo/bin/
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock') }}
+            libs/
+          key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }}
           restore-keys: |
             ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-
-      - name: Setup native dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: cargo xtask setup-native
+
+      - name: Cache cargo-nextest
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae
+        id: cargo-nextest-cache
+        with:
+          path: ~/.cargo/bin/cargo-nextest
+          key: ${{ runner.os }}-nextest-${{ env.NEXTEST_VERSION }}
+
+      - name: Install cargo-nextest
+        if: steps.cargo-nextest-cache.outputs.cache-hit != 'true'
+        run: cargo install cargo-nextest --version "$NEXTEST_VERSION" --locked
+
       - name: Download documentation EPUB
         if: steps.cache.outputs.cache-hit != 'true'
         uses: actions/download-artifact@v8
         with:
           name: docs-epub
           path: docs/book/epub
-      - name: Warmup build
+      - name: Cache Plato static assets
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: ./.github/actions/cache-assets
+      - name: Download static assets
+        if: steps.cache.outputs.cache-hit != 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: cargo xtask download-assets
+      - name: Warmup build (native)
         if: steps.cache.outputs.cache-hit != 'true'
         run: cargo check --workspace --all-features --all-targets
+      - name: Warmup build (Kobo release)
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: cargo build --release --target arm-unknown-linux-gnueabihf -p cadmus
 
   clippy:
     if: github.event_name == 'pull_request'
@@ -151,14 +190,7 @@ jobs:
         id: rust-toolchain
         with:
           components: clippy
-      - &set-build-metadata
-        name: Set build metadata
-        run: |
-          if [ "${{ github.event_name }}" = "pull_request" ]; then
-            echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> "$GITHUB_ENV"
-            echo "PR_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV"
-          fi
-          echo "GIT_VERSION=$(git describe --tags --dirty --always)" >> "$GITHUB_ENV"
+      - *set-build-metadata
       - *restore-shared-cache
       - name: Download documentation EPUB
         uses: actions/download-artifact@v8
@@ -246,13 +278,13 @@ jobs:
 
       - name: Install apt packages (Linux)
         if: runner.os == 'Linux'
-        uses: awalsh128/cache-apt-pkgs-action@latest
+        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05
         with:
           packages: git wget curl pkg-config gcc g++ make cmake meson ninja-build autoconf automake libtool gperf python3 zlib1g-dev libbz2-dev libpng-dev libjpeg-dev libopenjp2-7-dev libjbig2dec0-dev libgumbo-dev libfreetype6-dev libharfbuzz-dev libdjvulibre-dev libsdl2-dev
           version: 1.0
       - name: Cache Homebrew packages (macOS)
         if: runner.os == 'macOS'
-        uses: actions/cache@v5
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae
         with:
           path: |
             ~/Library/Caches/Homebrew/downloads
@@ -264,29 +296,17 @@ jobs:
         run: brew install sdl2 freetype harfbuzz openjpeg jpeg-turbo libpng jbig2dec gumbo-parser djvulibre pkg-config
 
       - *restore-shared-cache
-      - name: Cache thirdparty libraries
-        uses: ./.github/actions/cache-thirdparty
-        with:
-          kind: native
-
-      - name: Set nextest version
-        shell: bash
-        run: echo "NEXTEST_VERSION=0.9.94" >> "$GITHUB_ENV"
 
       - name: Cache cargo-nextest
-        uses: actions/cache@v5
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae
+        id: cargo-nextest-cache
         with:
           path: ~/.cargo/bin/cargo-nextest
           key: ${{ runner.os }}-nextest-${{ env.NEXTEST_VERSION }}
 
       - name: Install cargo-nextest
-        run: |
-          if ! cargo nextest --version 2>/dev/null | grep -q "$NEXTEST_VERSION"; then
-            cargo install cargo-nextest --version "$NEXTEST_VERSION" --locked
-          fi
-
-      - name: Setup native dependencies
-        run: cargo xtask setup-native
+        if: steps.cargo-nextest-cache.outputs.cache-hit != 'true'
+        run: cargo install cargo-nextest --version "$NEXTEST_VERSION" --locked
 
       - name: Run cargo test (${{ matrix.label }})
         run: cargo xtask test --features "${{ matrix.label }}"
@@ -309,6 +329,7 @@ jobs:
       - *set-build-metadata
 
       - uses: dtolnay/rust-toolchain@stable
+        id: rust-toolchain
         with:
           targets: arm-unknown-linux-gnueabihf
 
@@ -375,6 +396,7 @@ jobs:
       - *set-build-metadata
 
       - uses: dtolnay/rust-toolchain@stable
+        id: rust-toolchain
         with:
           targets: arm-unknown-linux-gnueabihf