From 56568acf79a0cd771acd8155409b7b11bae4a9d5 Mon Sep 17 00:00:00 2001 From: Kevin Hellemun <17928966+OGKevin@users.noreply.github.com> Date: Mon, 1 Jun 2026 20:33:07 +0200 Subject: [PATCH 1/2] dev: build using build.rs Change-Id: b03e8f80c2eab7a3d9fedc4c8892c1cb Change-Id-Short: ozwlrkrznxlp --- .devin/wiki.json | 8 +- .github/actions/build-kobo/action.yml | 21 +- .github/actions/cache-thirdparty/action.yml | 32 - .github/workflows/build-release-candidate.yml | 15 + .github/workflows/cadmus-docs.yml | 18 +- .github/workflows/cargo.yml | 104 ++-- .github/workflows/copilot-setup-steps.yml | 23 +- .github/workflows/release.yml | 15 + .gitmodules | 48 ++ Cargo.lock | 22 +- Cargo.toml | 5 + .../djvulibre/kobo.patch | 0 .../harfbuzz/kobo-options.txt | 0 .../harfbuzz/kobo.patch | 0 .../mupdf/README.md | 0 .../mupdf/kobo.patch | 0 .../mupdf/webp-image-h-kobo.patch | 0 .../webp-load-webp-deviations-kobo.patch | 0 .../mupdf/webp-upstream-697749-kobo.patch | 0 crates/build-deps/Cargo.toml | 17 + crates/build-deps/src/build/kobo.rs | 340 ++++++++++ crates/build-deps/src/build/kobo/mupdf.rs | 221 +++++++ crates/build-deps/src/build/kobo/recipes.rs | 373 +++++++++++ crates/build-deps/src/build/kobo/source.rs | 90 +++ crates/build-deps/src/build/mod.rs | 22 + crates/build-deps/src/build/mupdf.rs | 58 ++ crates/build-deps/src/build/mupdf_wrapper.rs | 119 ++++ crates/build-deps/src/build/native.rs | 579 ++++++++++++++++++ crates/build-deps/src/cmd.rs | 67 ++ crates/build-deps/src/lib.rs | 67 ++ crates/build-deps/src/markers.rs | 56 ++ crates/build-deps/src/versions.rs | 117 ++++ crates/cadmus/Cargo.toml | 3 + crates/core/Cargo.toml | 7 +- crates/core/build.rs | 295 +++++---- crates/emulator/Cargo.toml | 3 + crates/fetcher/Cargo.toml | 3 + crates/importer/Cargo.toml | 3 + renovate.json | 152 +---- thirdparty/bzip2 | 1 + thirdparty/bzip2/Makefile-kobo | 65 -- thirdparty/bzip2/build-kobo.sh | 3 - thirdparty/djvulibre | 1 + thirdparty/djvulibre/build-kobo.sh | 11 - thirdparty/freetype2 | 1 + thirdparty/freetype2/build-kobo.sh | 17 - thirdparty/gumbo | 1 + thirdparty/gumbo/build-kobo.sh | 8 - thirdparty/harfbuzz | 1 + thirdparty/harfbuzz/build-kobo.sh | 4 - thirdparty/jbig2dec | 1 + thirdparty/jbig2dec/build-kobo.sh | 10 - thirdparty/libjpeg | 1 + thirdparty/libjpeg/build-kobo.sh | 11 - thirdparty/libpng | 1 + thirdparty/libpng/build-kobo.sh | 12 - thirdparty/libwebp | 1 + thirdparty/libwebp/build-kobo.sh | 21 - thirdparty/mupdf | 1 + thirdparty/mupdf/build-kobo.sh | 11 - thirdparty/openjpeg | 1 + thirdparty/openjpeg/build-kobo.sh | 15 - thirdparty/zlib | 1 + thirdparty/zlib/build-kobo.sh | 9 - xtask/Cargo.toml | 5 +- xtask/src/lib.rs | 13 +- xtask/src/tasks/build_kobo.rs | 228 +------ xtask/src/tasks/dist.rs | 8 +- xtask/src/tasks/docs.rs | 2 +- xtask/src/tasks/install_importer.rs | 16 +- xtask/src/tasks/mod.rs | 1 - xtask/src/tasks/run_emulator.rs | 19 +- xtask/src/tasks/setup_native.rs | 469 -------------- xtask/src/tasks/util/mod.rs | 2 - xtask/src/tasks/util/mupdf_wrapper.rs | 115 ---- xtask/src/tasks/util/thirdparty.rs | 573 ----------------- 76 files changed, 2581 insertions(+), 1982 deletions(-) delete mode 100644 .github/actions/cache-thirdparty/action.yml create mode 100644 .gitmodules rename {thirdparty => build-scripts}/djvulibre/kobo.patch (100%) rename {thirdparty => build-scripts}/harfbuzz/kobo-options.txt (100%) rename {thirdparty => build-scripts}/harfbuzz/kobo.patch (100%) rename thirdparty/mupdf/README-kobo.md => build-scripts/mupdf/README.md (100%) rename {thirdparty => build-scripts}/mupdf/kobo.patch (100%) rename {thirdparty => build-scripts}/mupdf/webp-image-h-kobo.patch (100%) rename {thirdparty => build-scripts}/mupdf/webp-load-webp-deviations-kobo.patch (100%) rename {thirdparty => build-scripts}/mupdf/webp-upstream-697749-kobo.patch (100%) create mode 100644 crates/build-deps/Cargo.toml create mode 100644 crates/build-deps/src/build/kobo.rs create mode 100644 crates/build-deps/src/build/kobo/mupdf.rs create mode 100644 crates/build-deps/src/build/kobo/recipes.rs create mode 100644 crates/build-deps/src/build/kobo/source.rs create mode 100644 crates/build-deps/src/build/mod.rs create mode 100644 crates/build-deps/src/build/mupdf.rs create mode 100644 crates/build-deps/src/build/mupdf_wrapper.rs create mode 100644 crates/build-deps/src/build/native.rs create mode 100644 crates/build-deps/src/cmd.rs create mode 100644 crates/build-deps/src/lib.rs create mode 100644 crates/build-deps/src/markers.rs create mode 100644 crates/build-deps/src/versions.rs create mode 160000 thirdparty/bzip2 delete mode 100644 thirdparty/bzip2/Makefile-kobo delete mode 100755 thirdparty/bzip2/build-kobo.sh create mode 160000 thirdparty/djvulibre delete mode 100755 thirdparty/djvulibre/build-kobo.sh create mode 160000 thirdparty/freetype2 delete mode 100755 thirdparty/freetype2/build-kobo.sh create mode 160000 thirdparty/gumbo delete mode 100755 thirdparty/gumbo/build-kobo.sh create mode 160000 thirdparty/harfbuzz delete mode 100755 thirdparty/harfbuzz/build-kobo.sh create mode 160000 thirdparty/jbig2dec delete mode 100755 thirdparty/jbig2dec/build-kobo.sh create mode 160000 thirdparty/libjpeg delete mode 100755 thirdparty/libjpeg/build-kobo.sh create mode 160000 thirdparty/libpng delete mode 100755 thirdparty/libpng/build-kobo.sh create mode 160000 thirdparty/libwebp delete mode 100755 thirdparty/libwebp/build-kobo.sh create mode 160000 thirdparty/mupdf delete mode 100755 thirdparty/mupdf/build-kobo.sh create mode 160000 thirdparty/openjpeg delete mode 100755 thirdparty/openjpeg/build-kobo.sh create mode 160000 thirdparty/zlib delete mode 100755 thirdparty/zlib/build-kobo.sh delete mode 100644 xtask/src/tasks/setup_native.rs delete mode 100644 xtask/src/tasks/util/mupdf_wrapper.rs delete mode 100644 xtask/src/tasks/util/thirdparty.rs diff --git a/.devin/wiki.json b/.devin/wiki.json index b5c45838..76cc17d2 100644 --- a/.devin/wiki.json +++ b/.devin/wiki.json @@ -86,7 +86,7 @@ }, { "title": "Crate Structure", - "purpose": "Document the Rust workspace layout: each crate (cadmus-core, cadmus, emulator, importer, fetcher, xtask), their roles, the binaries they produce, and the inter-crate dependency graph. Cover shared workspace dependencies such as rustls and the release-minsized build profile.", + "purpose": "Document the Rust workspace layout: each crate (cadmus-core, cadmus, emulator, importer, fetcher, xtask, build-deps), their roles, the binaries they produce, and the inter-crate dependency graph. Cover shared workspace dependencies such as rustls and the release-minsized build profile. The build-deps crate is shared between cadmus-core's build.rs and xtask, and owns all thirdparty C/C++ build orchestration (native and Kobo recipes, MuPDF source preparation, mupdf_wrapper compilation).", "parent": "Contributor Guide", "page_notes": [ { @@ -96,7 +96,7 @@ }, { "title": "Development Environment", - "purpose": "Document the devenv and Nix-based reproducible development environment: entering the shell, available cargo xtask commands (setup-native, run-emulator, build-kobo, dist, bundle, test, docs), devenv tasks (deps:native, docs:build, build:kobo), the integrated observability stack (OTel Collector, Prometheus, Tempo, Loki, Grafana), platform support (Linux full, macOS native-only), and local override via devenv.local.nix.", + "purpose": "Document the devenv and Nix-based reproducible development environment: entering the shell, available cargo xtask commands (run-emulator, build-kobo, dist, bundle, test, docs, download-assets), devenv tasks (docs:build, build:kobo), the integrated observability stack (OTel Collector, Prometheus, Tempo, Loki, Grafana), platform support (Linux full, macOS full including Kobo cross-compilation), and local override via devenv.local.nix. Thirdparty native dependencies are built lazily by the cadmus-core build.rs (via the build-deps crate) the first time `cargo build` runs, so no explicit setup step is required.", "parent": "Contributor Guide", "page_notes": [ { @@ -176,7 +176,7 @@ }, { "title": "Building for Device", - "purpose": "Explain the end-to-end process for cross-compiling and packaging Cadmus for a target device: the ARM cross-compilation toolchain, cargo xtask build-kobo (fast vs slow modes, mupdf_wrapper compilation), cargo xtask dist assembly, cargo xtask bundle packaging, NickelMenu integration, and the device management shell scripts (wifi-enable.sh, wifi-disable.sh, usb-enable.sh, usb-disable.sh). Currently Kobo-specific; the build system is expected to generalise as additional devices are supported.", + "purpose": "Explain the end-to-end process for cross-compiling and packaging Cadmus for a target device: the ARM cross-compilation toolchain, cargo xtask build-kobo (a thin wrapper over `cargo build --release --target arm-unknown-linux-gnueabihf` whose thirdparty native artifacts are produced lazily by the cadmus-core build.rs via the build-deps crate), cargo xtask dist assembly, cargo xtask bundle packaging, NickelMenu integration, and the device management shell scripts (wifi-enable.sh, wifi-disable.sh, usb-enable.sh, usb-disable.sh). Currently Kobo-specific; the build system is expected to generalise as additional devices are supported.", "parent": "Contributor Guide", "page_notes": [ { @@ -186,7 +186,7 @@ }, { "title": "Third-Party Dependencies", - "purpose": "Document the thirdparty/ directory: its role in the cross-compilation pipeline, how library sources are downloaded at build time (not vendored), the dependency-ordered build process via build-kobo.sh scripts, which libraries carry Kobo-specific patches and the rationale behind each patch (e.g. bypassing pkg-config for cross-compilation, stripping unnecessary build targets, adding a device-specific cross-compile target), the relationship between version constants in xtask and Renovate-managed automated updates, and guidance on adding or upgrading a library.", + "purpose": "Document the thirdparty/ directory: its role in the cross-compilation pipeline, how library sources are vendored as git submodules and built from source by the build-deps crate (invoked from the cadmus-core build.rs and from xtask), the dependency-ordered build process via per-library recipes in build-deps (build/kobo/recipes.rs for ARM, build/native.rs for the host), which libraries carry Kobo-specific patches under build-scripts/ and the rationale behind each patch (e.g. bypassing pkg-config for cross-compilation, stripping unnecessary build targets, adding a device-specific cross-compile target), the relationship between version constants in build-deps (versions.rs) and Renovate-managed automated updates (including the git-submodules manager for submodule SHAs), and guidance on adding or upgrading a library.", "parent": "Contributor Guide", "page_notes": [ { diff --git a/.github/actions/build-kobo/action.yml b/.github/actions/build-kobo/action.yml index d7332a8b..57852277 100644 --- a/.github/actions/build-kobo/action.yml +++ b/.github/actions/build-kobo/action.yml @@ -23,18 +23,6 @@ inputs: runs: using: composite steps: - - name: Cache Cargo registry - id: rust-toolchain-cache - uses: actions/cache@v5 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-build-kobo-${{ hashFiles('**/Cargo.lock') }} - - name: Cache Linaro toolchain id: cache-linaro uses: actions/cache@v5 @@ -52,11 +40,6 @@ runs: tar xf gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz --strip-components=1 rm gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz - - name: Cache thirdparty libraries - uses: ./.github/actions/cache-thirdparty - with: - kind: kobo-build - - name: Cache NickelMenu downloads uses: actions/cache@v5 with: @@ -88,9 +71,9 @@ runs: GH_OAUTH_CLIENT_ID: ${{ inputs.gh-oauth-client-id }} run: | if [ "${{ inputs.test }}" = "true" ]; then - cargo xtask build-kobo --slow --features test + cargo xtask build-kobo --features test else - cargo xtask build-kobo --slow + cargo xtask build-kobo fi - name: Create distribution diff --git a/.github/actions/cache-thirdparty/action.yml b/.github/actions/cache-thirdparty/action.yml deleted file mode 100644 index 29edc822..00000000 --- a/.github/actions/cache-thirdparty/action.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Cache thirdparty libraries -description: > - Restore or save the thirdparty library cache. The cache key covers all files - that determine which library versions are downloaded and how they are built, - including the xtask source that encodes download URLs. - -inputs: - kind: - description: "'kobo-build' for built ARM libs, 'kobo-download' for source-only ARM cache, 'native' for host libs used in tests" - required: true - -runs: - using: composite - steps: - - name: Cache thirdparty libraries (kobo) - if: startsWith(inputs.kind, 'kobo-') - uses: actions/cache@v5 - with: - path: | - libs/ - thirdparty/ - mupdf_wrapper/ - key: ${{ runner.os }}-thirdparty-libs-${{ inputs.kind }}-${{ hashFiles('thirdparty/**/build-kobo.sh', 'thirdparty/**/*.patch', 'xtask/src/tasks/util/thirdparty.rs', 'mupdf_wrapper/mupdf_wrapper.c') }} - - - name: Cache thirdparty libraries (native) - if: inputs.kind == 'native' - uses: actions/cache@v5 - with: - path: | - thirdparty/mupdf - mupdf_wrapper/ - key: ${{ runner.os }}-thirdparty-libs-native-${{ hashFiles('xtask/src/tasks/util/thirdparty.rs', 'xtask/src/tasks/setup_native.rs', 'mupdf_wrapper/mupdf_wrapper.c') }} diff --git a/.github/workflows/build-release-candidate.yml b/.github/workflows/build-release-candidate.yml index 753f3762..cbe5b28e 100644 --- a/.github/workflows/build-release-candidate.yml +++ b/.github/workflows/build-release-candidate.yml @@ -43,9 +43,24 @@ jobs: ref: ${{ steps.pr.outputs.branch }} - uses: dtolnay/rust-toolchain@stable + id: rust-toolchain with: targets: arm-unknown-linux-gnueabihf + - name: Restore shared cargo cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + libs/ + key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }} + restore-keys: | + ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}- + - name: Build Kobo artifacts uses: ./.github/actions/build-kobo with: diff --git a/.github/workflows/cadmus-docs.yml b/.github/workflows/cadmus-docs.yml index da0b1ec3..fe127509 100644 --- a/.github/workflows/cadmus-docs.yml +++ b/.github/workflows/cadmus-docs.yml @@ -47,6 +47,7 @@ jobs: fetch-depth: 0 fetch-tags: true - uses: dtolnay/rust-toolchain@stable + id: rust-toolchain - name: Setup Python uses: actions/setup-python@v6 @@ -71,17 +72,24 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} - - name: Cache cargo doc - uses: actions/cache@v5 + - name: Restore shared cargo artifacts + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae with: - path: target/doc - key: ${{ runner.os }}-cargo-doc-${{ hashFiles('**/Cargo.lock') }} + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + libs/ + key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }} restore-keys: | - ${{ runner.os }}-cargo-doc- + ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}- - name: Build documentation portal env: RUSTDOCFLAGS: -D warnings + CADMUS_SKIP_THIRDPARTY_DEPS: "1" run: cargo xtask docs --base-url "$ZOLA_BASE_URL" - name: Upload build artifact diff --git a/.github/workflows/cargo.yml b/.github/workflows/cargo.yml index fb7d0393..23f00f7c 100644 --- a/.github/workflows/cargo.yml +++ b/.github/workflows/cargo.yml @@ -1,6 +1,7 @@ name: Cargo env: + NEXTEST_VERSION: "0.9.94" NODE_VERSION: "24" SQLX_OFFLINE: true @@ -16,8 +17,7 @@ on: - ".github/actions/build-kobo/**" - ".github/actions/build-docs-epub/**" - ".github/actions/install-doc-tools/**" - - "thirdparty/**/*.patch" - - "thirdparty/**/*.patch" + - "build-scripts/**" concurrency: group: "cargo-${{ github.event_name }}-${{ github.event.number || github.ref }}" @@ -36,7 +36,7 @@ jobs: components: rustfmt - &restore-shared-cache name: Restore shared cargo cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae with: path: | ~/.cargo/bin/ @@ -44,7 +44,8 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock') }} + libs/ + key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }} restore-keys: | ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}- - name: Check formatting @@ -89,25 +90,42 @@ jobs: - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@stable id: rust-toolchain - - name: Set build metadata + with: + targets: arm-unknown-linux-gnueabihf + + - &set-build-metadata + name: Set build metadata run: | if [ "${{ github.event_name }}" = "pull_request" ]; then echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> "$GITHUB_ENV" echo "PR_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" fi echo "GIT_VERSION=$(git describe --tags --dirty --always)" >> "$GITHUB_ENV" + - name: Install apt packages - uses: awalsh128/cache-apt-pkgs-action@latest + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 with: packages: git wget curl pkg-config gcc g++ make cmake meson ninja-build autoconf automake libtool gperf python3 zlib1g-dev libbz2-dev libpng-dev libjpeg-dev libopenjp2-7-dev libjbig2dec0-dev libgumbo-dev libfreetype6-dev libharfbuzz-dev libdjvulibre-dev libsdl2-dev version: 1.0 - - name: Cache thirdparty libraries - uses: ./.github/actions/cache-thirdparty + - name: Cache Linaro toolchain + id: cache-linaro + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae with: - kind: native + path: ~/linaro-toolchain + key: linaro-gcc-4.9.4-2017.01 + - name: Download Linaro toolchain + if: steps.cache-linaro.outputs.cache-hit != 'true' + run: | + mkdir -p ~/linaro-toolchain + cd ~/linaro-toolchain + wget -q https://releases.linaro.org/components/toolchain/binaries/4.9-2017.01/arm-linux-gnueabihf/gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz + tar xf gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz --strip-components=1 + rm gcc-linaro-4.9.4-2017.01-x86_64_arm-linux-gnueabihf.tar.xz + - name: Setup Linaro toolchain + run: echo "$HOME/linaro-toolchain/bin" >> "$GITHUB_PATH" - name: Cache shared cargo artifacts id: cache - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae with: path: | ~/.cargo/bin/ @@ -115,21 +133,42 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock') }} + libs/ + key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }} restore-keys: | ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}- - - name: Setup native dependencies - if: steps.cache.outputs.cache-hit != 'true' - run: cargo xtask setup-native + + - name: Cache cargo-nextest + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae + id: cargo-nextest-cache + with: + path: ~/.cargo/bin/cargo-nextest + key: ${{ runner.os }}-nextest-${{ env.NEXTEST_VERSION }} + + - name: Install cargo-nextest + if: steps.cargo-nextest-cache.outputs.cache-hit != 'true' + run: cargo install cargo-nextest --version "$NEXTEST_VERSION" --locked + - name: Download documentation EPUB if: steps.cache.outputs.cache-hit != 'true' uses: actions/download-artifact@v8 with: name: docs-epub path: docs/book/epub - - name: Warmup build + - name: Cache Plato static assets + if: steps.cache.outputs.cache-hit != 'true' + uses: ./.github/actions/cache-assets + - name: Download static assets + if: steps.cache.outputs.cache-hit != 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: cargo xtask download-assets + - name: Warmup build (native) if: steps.cache.outputs.cache-hit != 'true' run: cargo check --workspace --all-features --all-targets + - name: Warmup build (Kobo release) + if: steps.cache.outputs.cache-hit != 'true' + run: cargo build --release --target arm-unknown-linux-gnueabihf -p cadmus clippy: if: github.event_name == 'pull_request' @@ -151,14 +190,7 @@ jobs: id: rust-toolchain with: components: clippy - - &set-build-metadata - name: Set build metadata - run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> "$GITHUB_ENV" - echo "PR_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" - fi - echo "GIT_VERSION=$(git describe --tags --dirty --always)" >> "$GITHUB_ENV" + - *set-build-metadata - *restore-shared-cache - name: Download documentation EPUB uses: actions/download-artifact@v8 @@ -246,13 +278,13 @@ jobs: - name: Install apt packages (Linux) if: runner.os == 'Linux' - uses: awalsh128/cache-apt-pkgs-action@latest + uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 with: packages: git wget curl pkg-config gcc g++ make cmake meson ninja-build autoconf automake libtool gperf python3 zlib1g-dev libbz2-dev libpng-dev libjpeg-dev libopenjp2-7-dev libjbig2dec0-dev libgumbo-dev libfreetype6-dev libharfbuzz-dev libdjvulibre-dev libsdl2-dev version: 1.0 - name: Cache Homebrew packages (macOS) if: runner.os == 'macOS' - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae with: path: | ~/Library/Caches/Homebrew/downloads @@ -264,29 +296,17 @@ jobs: run: brew install sdl2 freetype harfbuzz openjpeg jpeg-turbo libpng jbig2dec gumbo-parser djvulibre pkg-config - *restore-shared-cache - - name: Cache thirdparty libraries - uses: ./.github/actions/cache-thirdparty - with: - kind: native - - - name: Set nextest version - shell: bash - run: echo "NEXTEST_VERSION=0.9.94" >> "$GITHUB_ENV" - name: Cache cargo-nextest - uses: actions/cache@v5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + id: cargo-nextest-cache with: path: ~/.cargo/bin/cargo-nextest key: ${{ runner.os }}-nextest-${{ env.NEXTEST_VERSION }} - name: Install cargo-nextest - run: | - if ! cargo nextest --version 2>/dev/null | grep -q "$NEXTEST_VERSION"; then - cargo install cargo-nextest --version "$NEXTEST_VERSION" --locked - fi - - - name: Setup native dependencies - run: cargo xtask setup-native + if: steps.cargo-nextest-cache.outputs.cache-hit != 'true' + run: cargo install cargo-nextest --version "$NEXTEST_VERSION" --locked - name: Run cargo test (${{ matrix.label }}) run: cargo xtask test --features "${{ matrix.label }}" @@ -309,6 +329,7 @@ jobs: - *set-build-metadata - uses: dtolnay/rust-toolchain@stable + id: rust-toolchain with: targets: arm-unknown-linux-gnueabihf @@ -375,6 +396,7 @@ jobs: - *set-build-metadata - uses: dtolnay/rust-toolchain@stable + id: rust-toolchain with: targets: arm-unknown-linux-gnueabihf diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index f53719bd..bbbb2380 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -27,7 +27,7 @@ jobs: targets: arm-unknown-linux-gnueabihf - name: Restore shared cargo cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae with: path: | ~/.cargo/bin/ @@ -35,21 +35,12 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock') }} + libs/ + key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }} restore-keys: | ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}- - - name: Cache thirdparty libraries (Kobo cross-compile) - uses: ./.github/actions/cache-thirdparty - with: - kind: kobo-download - - - name: Cache thirdparty libraries (native testing) - uses: ./.github/actions/cache-thirdparty - with: - kind: native - - - name: Cache Linaro toolchain + - name: Restore Linaro toolchain id: cache-linaro uses: actions/cache@v5 with: @@ -98,12 +89,6 @@ jobs: - name: Build documentation run: cargo xtask docs --mdbook-only - - name: Download thirdparty dependencies - run: cargo xtask build-kobo --slow --download-only - - - name: Setup native dependencies for testing - run: cargo xtask setup-native - - name: Install reviewdog uses: reviewdog/action-setup@v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e1cb7600..d8d302aa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -70,9 +70,24 @@ jobs: fetch-tags: true - uses: dtolnay/rust-toolchain@stable + id: rust-toolchain with: targets: arm-unknown-linux-gnueabihf + - name: Restore shared cargo cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + libs/ + key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }} + restore-keys: | + ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}- + - name: Build documentation EPUB uses: ./.github/actions/build-docs-epub with: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..bccf5eae --- /dev/null +++ b/.gitmodules @@ -0,0 +1,48 @@ +[submodule "thirdparty/zlib"] + path = thirdparty/zlib + url = https://github.com/madler/zlib + branch = v1.3.2 +[submodule "thirdparty/bzip2"] + path = thirdparty/bzip2 + url = https://gitlab.com/bzip2/bzip2 + branch = bzip2-1.0.8 +[submodule "thirdparty/libpng"] + path = thirdparty/libpng + url = https://github.com/pnggroup/libpng + branch = v1.6.53 +[submodule "thirdparty/libjpeg"] + path = thirdparty/libjpeg + url = https://github.com/libjpeg-turbo/libjpeg-turbo + branch = jpeg-10 +[submodule "thirdparty/openjpeg"] + path = thirdparty/openjpeg + url = https://github.com/uclouvain/openjpeg + branch = v2.5.4 +[submodule "thirdparty/jbig2dec"] + path = thirdparty/jbig2dec + url = https://github.com/ArtifexSoftware/jbig2dec + branch = 0.20 +[submodule "thirdparty/libwebp"] + path = thirdparty/libwebp + url = https://github.com/webmproject/libwebp + branch = v1.2.3 +[submodule "thirdparty/freetype2"] + path = thirdparty/freetype2 + url = https://github.com/freetype/freetype + branch = VER-2-14-1 +[submodule "thirdparty/harfbuzz"] + path = thirdparty/harfbuzz + url = https://github.com/harfbuzz/harfbuzz + branch = 14.2.0 +[submodule "thirdparty/gumbo"] + path = thirdparty/gumbo + url = https://github.com/google/gumbo-parser + branch = v0.10.1 +[submodule "thirdparty/djvulibre"] + path = thirdparty/djvulibre + url = https://github.com/barak/djvulibre + branch = release.3.5.30 +[submodule "thirdparty/mupdf"] + path = thirdparty/mupdf + url = https://github.com/ArtifexSoftware/mupdf + branch = 1.27.2 diff --git a/Cargo.lock b/Cargo.lock index 4b9a3f07..d1df3900 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -352,6 +352,15 @@ dependencies = [ "serde", ] +[[package]] +name = "build-deps" +version = "0.10.1" +dependencies = [ + "anyhow", + "cc", + "tempfile", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -396,7 +405,9 @@ dependencies = [ "arc-swap", "bitflags 2.11.1", "blake3", + "build-deps", "byteorder", + "cc", "chrono", "criterion", "ctor", @@ -1020,7 +1031,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3345,7 +3356,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3404,7 +3415,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4107,7 +4118,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4958,7 +4969,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -5364,6 +5375,7 @@ name = "xtask" version = "0.10.1" dependencies = [ "anyhow", + "build-deps", "clap", "flate2", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index fe39ed7a..1e01b997 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "2" members = [ + "crates/build-deps", "crates/core", "crates/cadmus", "crates/emulator", @@ -16,6 +17,10 @@ rustls = { version = "0.23", default-features = false, features = [ "std", "tls12", ] } +cc = "1" + +[workspace.lints] +rustdoc.private_intra_doc_links = "allow" [profile.release-minsized] inherits = "release" diff --git a/thirdparty/djvulibre/kobo.patch b/build-scripts/djvulibre/kobo.patch similarity index 100% rename from thirdparty/djvulibre/kobo.patch rename to build-scripts/djvulibre/kobo.patch diff --git a/thirdparty/harfbuzz/kobo-options.txt b/build-scripts/harfbuzz/kobo-options.txt similarity index 100% rename from thirdparty/harfbuzz/kobo-options.txt rename to build-scripts/harfbuzz/kobo-options.txt diff --git a/thirdparty/harfbuzz/kobo.patch b/build-scripts/harfbuzz/kobo.patch similarity index 100% rename from thirdparty/harfbuzz/kobo.patch rename to build-scripts/harfbuzz/kobo.patch diff --git a/thirdparty/mupdf/README-kobo.md b/build-scripts/mupdf/README.md similarity index 100% rename from thirdparty/mupdf/README-kobo.md rename to build-scripts/mupdf/README.md diff --git a/thirdparty/mupdf/kobo.patch b/build-scripts/mupdf/kobo.patch similarity index 100% rename from thirdparty/mupdf/kobo.patch rename to build-scripts/mupdf/kobo.patch diff --git a/thirdparty/mupdf/webp-image-h-kobo.patch b/build-scripts/mupdf/webp-image-h-kobo.patch similarity index 100% rename from thirdparty/mupdf/webp-image-h-kobo.patch rename to build-scripts/mupdf/webp-image-h-kobo.patch diff --git a/thirdparty/mupdf/webp-load-webp-deviations-kobo.patch b/build-scripts/mupdf/webp-load-webp-deviations-kobo.patch similarity index 100% rename from thirdparty/mupdf/webp-load-webp-deviations-kobo.patch rename to build-scripts/mupdf/webp-load-webp-deviations-kobo.patch diff --git a/thirdparty/mupdf/webp-upstream-697749-kobo.patch b/build-scripts/mupdf/webp-upstream-697749-kobo.patch similarity index 100% rename from thirdparty/mupdf/webp-upstream-697749-kobo.patch rename to build-scripts/mupdf/webp-upstream-697749-kobo.patch diff --git a/crates/build-deps/Cargo.toml b/crates/build-deps/Cargo.toml new file mode 100644 index 00000000..b0291f1d --- /dev/null +++ b/crates/build-deps/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "build-deps" +version = "0.10.1" +edition = "2024" +description = "Thirdparty dependency build helpers shared between build.rs and xtask" + +[lib] +name = "build_deps" +path = "src/lib.rs" + +[dependencies] +anyhow = "1" +cc = { workspace = true } +tempfile = "3" + +[lints] +workspace = true diff --git a/crates/build-deps/src/build/kobo.rs b/crates/build-deps/src/build/kobo.rs new file mode 100644 index 00000000..b49bad70 --- /dev/null +++ b/crates/build-deps/src/build/kobo.rs @@ -0,0 +1,340 @@ +//! Cross-compilation helpers for the Kobo (ARM) target. +//! +//! The flow has three layers, each in its own submodule: +//! +//! * [`source`] prepares a clean, patched copy of each thirdparty +//! library's source tree under +//! `target/cadmus-build-deps///`. +//! * [`recipes`] builds the individual libraries (configure / make / +//! meson / cmake invocations tailored to each upstream build +//! system). +//! * [`mupdf`] links the per-library outputs into the final +//! `libmupdf.so` shared object loaded by the Kobo runtime. +//! +//! The top-level entry points ([`build_libraries`], [`copy_built_libs`], +//! [`create_symlinks`]) compose these layers. Callers that just want +//! "the Kobo target is ready to link" should use the single +//! [`ensure_kobo_artifacts`] helper, which owns the full flow +//! (submodule init, library build/copy/symlink, and the +//! `mupdf_wrapper` archive). + +pub(crate) mod mupdf; +pub(crate) mod recipes; +pub(crate) mod source; + +use std::path::Path; + +use anyhow::{Context, Result, bail}; + +use crate::cmd; +use crate::markers; +use crate::versions::{self, SONAMES}; + +/// Default Cargo target triple when `TARGET` is unset. +fn target_triple() -> String { + std::env::var("TARGET").unwrap_or_else(|_| "arm-unknown-linux-gnueabihf".to_string()) +} + +/// Build-root directory used for Kobo cross-builds. +pub fn build_root(root: &Path) -> std::path::PathBuf { + root.join("target/cadmus-build-deps").join(target_triple()) +} + +/// Build every thirdparty library in [`LIBRARY_NAMES`][versions::LIBRARY_NAMES] +/// from source, in dependency order. +/// +/// Each library is copied from its submodule into the per-target build +/// directory, patched (see [`source::copy_source`]) and built with the +/// toolchain-appropriate recipe from [`recipes`]. A `.built` marker is +/// written on success so re-runs are idempotent. +/// +/// # Errors +/// +/// Returns an error if a submodule is missing, the source cannot be +/// copied or patched, or any of the underlying build commands fail. +fn build_libraries(thirdparty_dir: &Path) -> Result<()> { + let targets = versions::LIBRARY_NAMES.to_vec(); + + let root = thirdparty_dir + .parent() + .context("thirdparty dir has no parent")?; + let build_root = build_root(root); + std::fs::create_dir_all(&build_root)?; + + for name in targets { + let src_dir = thirdparty_dir.join(name); + if !src_dir.exists() { + anyhow::bail!( + "thirdparty/{name} not found - run `git submodule update --init --recursive` first" + ); + } + + let build_dir = build_root.join(name); + if markers::is_built(&build_dir) { + println!("Skipping {name} (already built)..."); + continue; + } + + if build_dir.exists() { + std::fs::remove_dir_all(&build_dir) + .with_context(|| format!("failed to remove old build dir for {name}"))?; + } + + source::copy_source(&src_dir, &build_dir, name, root)?; + source::apply_patches(&build_dir, name, root)?; + recipes::build_library(name, &build_dir)?; + + markers::mark_built(&build_dir, name)?; + } + + Ok(()) +} + +/// Copy the per-library outputs produced by [`build_libraries`] into +/// `libs/`, naming each file according to +/// [`BUILT_LIBRARY_COPIES`][versions::BUILT_LIBRARY_COPIES]. +/// +/// Source paths in the constant use the `thirdparty//...` form +/// and are rewritten against the per-target build root. +pub(crate) fn copy_built_libs(root: &Path, libs_dir: &Path) -> Result<()> { + let build_root = build_root(root); + + for &(src_rel, dest_name) in versions::BUILT_LIBRARY_COPIES { + let rel = src_rel.strip_prefix("thirdparty/").unwrap_or(src_rel); + let src = build_root.join(rel); + let dest = libs_dir.join(dest_name); + std::fs::copy(&src, &dest).map_err(|e| { + anyhow::anyhow!( + "failed to copy {} to {}: {e}", + src.display(), + dest.display() + ) + })?; + } + Ok(()) +} + +/// Build every artefact the Kobo (ARM) target needs — the thirdparty +/// shared libraries staged under `libs/` and the +/// `target/mupdf_wrapper/Kobo/libmupdf_wrapper.a` C glue archive — if +/// it's not already cached. +/// +/// On a warm cache (every [`SONAMES`] entry present and the wrapper +/// archive on disk) this returns `Ok(())` without touching git or the +/// cross toolchain. Otherwise it initialises submodules, builds the +/// libraries, copies/symlinks them, and finally compiles the +/// `mupdf_wrapper` archive against the Kobo MuPDF headers. +/// +/// # Errors +/// +/// Returns an error if submodules cannot be initialised, `thirdparty/` +/// is missing, any of the library build steps fail, or the wrapper +/// compile fails. +pub fn ensure_kobo_artifacts(root: &Path) -> Result<()> { + if kobo_artifacts_present(root) { + return Ok(()); + } + + crate::ensure_submodules(root).context("failed to initialize git submodules")?; + + let thirdparty_dir = root.join("thirdparty"); + if !thirdparty_dir.exists() { + bail!("thirdparty/ directory not found. Run: git submodule update --init --recursive"); + } + + let libs_dir = root.join("libs"); + build_libraries(&thirdparty_dir).context("failed to build thirdparty libraries")?; + std::fs::create_dir_all(&libs_dir).context("failed to create libs/ directory")?; + copy_built_libs(root, &libs_dir).context("failed to copy built libraries")?; + create_symlinks(&libs_dir).context("failed to create symlinks")?; + + let include = build_root(root).join("mupdf/include"); + crate::build::mupdf_wrapper::build_kobo(root, &include) + .context("failed to build mupdf_wrapper")?; + + Ok(()) +} + +/// Returns `true` when the Kobo artefact cache is complete: every +/// [`SONAMES`] entry is present in `libs/` and the `mupdf_wrapper` +/// archive has been built for the Kobo target. +fn kobo_artifacts_present(root: &Path) -> bool { + let libs_dir = root.join("libs"); + if !libs_dir.exists() { + return false; + } + if !SONAMES.iter().all(|lib| libs_dir.join(lib).exists()) { + return false; + } + root.join("target/mupdf_wrapper/Kobo/libmupdf_wrapper.a") + .exists() +} + +/// Create the `.so` → versioned-SONAME symlinks the Cadmus runtime +/// expects inside `libs/`. The actual SONAME of each `.so` is +/// discovered with `arm-linux-gnueabihf-readelf`; the symlink name is +/// the base name from [`SONAMES`]. +fn create_symlinks(libs_dir: &Path) -> Result<()> { + for &lib in SONAMES { + let target = soname(libs_dir, lib)?; + let link_path = libs_dir.join(lib); + if !link_path.exists() { + #[cfg(unix)] + std::os::unix::fs::symlink(&target, &link_path)?; + } + } + Ok(()) +} + +/// Resolve the SONAME of `lib` inside `libs_dir`. +/// +/// If `lib` already exists at its base name, the SONAME is read out +/// of its ELF dynamic section with `arm-linux-gnueabihf-readelf`. +/// Otherwise, the directory is scanned for a single versioned file +/// starting with `.` and that filename is returned. +pub fn soname(libs_dir: &Path, lib: &str) -> Result { + if libs_dir.join(lib).exists() { + read_elf_soname(libs_dir, lib) + } else { + find_versioned_soname(libs_dir, lib) + } +} + +/// Read the SONAME of `lib` from its ELF dynamic section using +/// `arm-linux-gnueabihf-readelf`. The SONAME is the last +/// bracketed token on the `SONAME` line of the dynamic section. +fn read_elf_soname(libs_dir: &Path, lib: &str) -> Result { + let so_path = libs_dir.join(lib); + let so_path_str = so_path + .to_str() + .with_context(|| format!("shared library path is not valid UTF-8: {so_path:?}"))?; + let output = cmd::output( + "arm-linux-gnueabihf-readelf", + &["-d", so_path_str], + libs_dir, + &[], + )?; + output + .lines() + .find(|line| line.contains("SONAME")) + .and_then(|line| line.split_whitespace().last()) + .map(|token| { + token + .trim_start_matches('[') + .trim_end_matches(']') + .to_string() + }) + .with_context(|| format!("failed to find SONAME in readelf output for {lib}")) +} + +/// Locate the SONAME of `lib` by scanning `libs_dir` for a single +/// versioned file whose name starts with `.`. +/// +/// # Errors +/// +/// Returns an error if zero or more than one matching file is found. +fn find_versioned_soname(libs_dir: &Path, lib: &str) -> Result { + let prefix = format!("{lib}."); + let matching: Vec<_> = std::fs::read_dir(libs_dir)? + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().starts_with(&prefix)) + .collect(); + + match matching.len() { + 1 => Ok(matching[0].file_name().to_string_lossy().into_owned()), + 0 => anyhow::bail!( + "no versioned file found for {lib} in {}", + libs_dir.display() + ), + _ => anyhow::bail!( + "multiple versioned files found for {lib} in {}", + libs_dir.display() + ), + } +} + +/// Recursive copy that mirrors a source tree to `dst`, skipping git +/// metadata (`*.git`, `*.gitattributes`), build artefacts (`build/`, +/// `objs/`) and `autom4te.cache/`. +/// +/// Symlinks are preserved as symlinks; regular files and directories +/// are copied recursively. Used by [`source::copy_source`] and by +/// the native build to snapshot the MuPDF source tree before +/// patching. +pub fn cp_r(src: &Path, dst: &Path) -> Result<()> { + std::fs::create_dir_all(dst)?; + for entry in std::fs::read_dir(src)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.starts_with(".git") + || name_str == "build" + || name_str == "objs" + || name_str == "autom4te.cache" + { + continue; + } + let ft = entry.file_type()?; + let dst_child = dst.join(&name); + if ft.is_dir() { + cp_r(&entry.path(), &dst_child)?; + } else if ft.is_symlink() { + if let Ok(target) = std::fs::read_link(entry.path()) { + #[cfg(unix)] + std::os::unix::fs::symlink(&target, &dst_child)?; + } + } else { + std::fs::copy(entry.path(), &dst_child)?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn symlink_list_has_no_duplicates() { + let mut link_names: Vec<&str> = SONAMES.to_vec(); + link_names.sort_unstable(); + let original_len = link_names.len(); + link_names.dedup(); + assert_eq!(link_names.len(), original_len, "duplicate link names found"); + } + + #[test] + fn soname_finds_single_versioned_file() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("libfoo.so.1.2.3"), b"").unwrap(); + + let resolved = soname(tmp.path(), "libfoo.so").unwrap(); + assert_eq!(resolved, "libfoo.so.1.2.3"); + } + + #[test] + fn soname_errors_when_no_versioned_file_present() { + let tmp = tempfile::tempdir().unwrap(); + + let err = soname(tmp.path(), "libfoo.so").unwrap_err(); + let msg = format!("{err}"); + assert!( + msg.contains("no versioned file found for libfoo.so"), + "unexpected error: {msg}" + ); + } + + #[test] + fn soname_errors_when_multiple_versioned_files_present() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("libfoo.so.1"), b"").unwrap(); + std::fs::write(tmp.path().join("libfoo.so.2"), b"").unwrap(); + + let err = soname(tmp.path(), "libfoo.so").unwrap_err(); + let msg = format!("{err}"); + assert!( + msg.contains("multiple versioned files found for libfoo.so"), + "unexpected error: {msg}" + ); + } +} diff --git a/crates/build-deps/src/build/kobo/mupdf.rs b/crates/build-deps/src/build/kobo/mupdf.rs new file mode 100644 index 00000000..0c42cff1 --- /dev/null +++ b/crates/build-deps/src/build/kobo/mupdf.rs @@ -0,0 +1,221 @@ +//! Build MuPDF into the final `libmupdf.so` shared object that the +//! Kobo runtime links against. +//! +//! After the per-library recipes have produced their shared objects in +//! sibling directories (zlib, freetype2, harfbuzz, …) this module +//! collects every `*.o` produced by `make libs` inside MuPDF and +//! relinks them into a single `libmupdf.so` that depends on the +//! sibling libraries. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; + +use crate::cmd; + +/// Build `libmupdf.so` from the patched MuPDF source tree under +/// `build_dir`, linking against the shared libraries produced by the +/// other Kobo recipes. +pub fn build_mupdf(build_dir: &Path) -> Result<()> { + clean_git_metadata(build_dir)?; + clean_old_build(build_dir)?; + + cmd::run("make", &["verbose=yes", "generate"], build_dir, &[]) + .context("failed to run make generate for mupdf")?; + + let xcflags = "-I../libwebp/src -DHAVE_WEBP=1"; + cmd::run( + "make", + &[ + "verbose=yes", + "mujs=no", + "tesseract=no", + "extract=no", + "archive=no", + "brotli=no", + "barcode=no", + "commercial=no", + "USE_SYSTEM_LIBS=yes", + "OS=kobo", + "build=release", + &format!("XCFLAGS={xcflags}"), + "libs", + ], + build_dir, + &[], + ) + .context("failed to build mupdf libs")?; + + let obj_files = collect_object_files(build_dir)?; + let libmupdf_so = build_dir.join("build/release/libmupdf.so"); + let libmupdf_so_str = libmupdf_so + .to_str() + .context("libmupdf.so path is not valid UTF-8")?; + + let mut link_args = vec!["-Wl,--gc-sections", "-o", libmupdf_so_str]; + link_args.extend(obj_files.iter().map(|s| s.as_str())); + link_args.extend_from_slice(&[ + "-lm", + "-L../freetype2/objs/.libs", + "-lfreetype", + "-L../harfbuzz/build/src", + "-lharfbuzz", + "-L../gumbo/.libs", + "-lgumbo", + "-L../jbig2dec/.libs", + "-ljbig2dec", + "-L../libjpeg/.libs", + "-ljpeg", + "-L../openjpeg/build/bin", + "-lopenjp2", + "-L../zlib", + "-lz", + "-L../libwebp/src/.libs", + "-lwebp", + "-L../libwebp/src/demux/.libs", + "-lwebpdemux", + "-shared", + "-Wl,-soname", + "-Wl,libmupdf.so", + "-Wl,--no-undefined", + ]); + + cmd::run("arm-linux-gnueabihf-gcc", &link_args, build_dir, &[]) + .context("failed to link libmupdf.so") +} + +/// Strip `.git*` entries from `build_dir`. MuPDF's makefiles +/// misinterpret git attributes when present, so they are removed +/// before the build starts. +fn clean_git_metadata(build_dir: &Path) -> Result<()> { + let gitattributes = build_dir.join(".gitattributes"); + if !gitattributes.exists() { + return Ok(()); + } + for entry in std::fs::read_dir(build_dir)? { + let entry = entry?; + let name = entry.file_name(); + if name.to_string_lossy().starts_with(".git") { + if entry.path().is_dir() { + std::fs::remove_dir_all(entry.path()).with_context(|| { + format!("failed to remove directory {}", entry.path().display()) + })?; + } else { + std::fs::remove_file(entry.path()) + .with_context(|| format!("failed to remove file {}", entry.path().display()))?; + } + } + } + Ok(()) +} + +/// Remove any pre-existing `build/` directory created by a previous +/// `make` invocation. +fn clean_old_build(build_dir: &Path) -> Result<()> { + let mupdf_build = build_dir.join("build"); + if mupdf_build.exists() { + std::fs::remove_dir_all(&mupdf_build)?; + } + Ok(()) +} + +/// Walk `build_dir/build/release/` and return every `*.o` path except +/// the CJK font and lcms sub-archives that MuPDF bundles by default +/// but that Cadmus does not need at runtime. +fn collect_object_files(build_dir: &Path) -> Result> { + let release_dir = build_dir.join("build/release"); + let excluded_substrings = [ + "SourceHanSerif-Regular", + "DroidSansFallbackFull", + "NotoSerifTangut", + "color-lcms", + ]; + + let mut objects = Vec::new(); + for entry in walk_release(&release_dir)? { + let path = entry?; + if path.extension().is_none_or(|e| e != "o") { + continue; + } + let path_str = path.to_string_lossy(); + if excluded_substrings + .iter() + .any(|needle| path_str.contains(needle)) + { + continue; + } + objects.push(path_str.into_owned()); + } + Ok(objects) +} + +fn walk_release(dir: &Path) -> Result>> { + let mut out = Vec::new(); + walk_recursive(dir, &mut out)?; + Ok(out) +} + +fn walk_recursive(dir: &Path, out: &mut Vec>) -> Result<()> { + for entry in + std::fs::read_dir(dir).with_context(|| format!("failed to read {}", dir.display()))? + { + let entry = entry?; + let path = entry.path(); + let ft = entry.file_type()?; + if ft.is_dir() { + walk_recursive(&path, out)?; + } else { + out.push(Ok(path)); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn collect_object_files_skips_excluded_substrings() { + let tmp = tempfile::tempdir().unwrap(); + let release = tmp.path().join("build/release"); + std::fs::create_dir_all(&release).unwrap(); + + for name in [ + "extract.o", + "SourceHanSerif-Regular.o", + "DroidSansFallbackFull.o", + "NotoSerifTangut.o", + "color-lcms.o", + "fz-image.o", + ] { + std::fs::write(release.join(name), b"").unwrap(); + } + + let objects = collect_object_files(tmp.path()).unwrap(); + let names: Vec<_> = objects + .iter() + .map(|p| { + std::path::Path::new(p) + .file_name() + .unwrap() + .to_string_lossy() + .into_owned() + }) + .collect(); + + assert!(names.contains(&"extract.o".to_string())); + assert!(names.contains(&"fz-image.o".to_string())); + for excluded in [ + "SourceHanSerif-Regular.o", + "DroidSansFallbackFull.o", + "NotoSerifTangut.o", + "color-lcms.o", + ] { + assert!( + !names.iter().any(|n| n == excluded), + "{excluded} should have been excluded" + ); + } + } +} diff --git a/crates/build-deps/src/build/kobo/recipes.rs b/crates/build-deps/src/build/kobo/recipes.rs new file mode 100644 index 00000000..7138dd73 --- /dev/null +++ b/crates/build-deps/src/build/kobo/recipes.rs @@ -0,0 +1,373 @@ +//! Per-library build recipes for the Kobo cross-build. +//! +//! Each recipe matches the upstream build system of one thirdparty +//! library (autotools, CMake, plain `cc` invocation, meson). All +//! recipes share the same CFLAGS via [`cross_env`], targeting the +//! Kobo's Cortex-A9 CPU with NEON. + +use std::path::Path; + +use anyhow::{Context, Result}; + +use crate::cmd; + +/// Common cross-compilation environment used by most recipes. +fn cross_env() -> [(&'static str, &'static str); 12] { + [ + ("CC", "arm-linux-gnueabihf-gcc"), + ("CC_BUILD", "cc"), + ("CXX", "arm-linux-gnueabihf-g++"), + ("AR", "arm-linux-gnueabihf-ar"), + ("AS", "arm-linux-gnueabihf-as"), + ("NM", "arm-linux-gnueabihf-nm"), + ("STRIP", "arm-linux-gnueabihf-strip"), + ("RANLIB", "arm-linux-gnueabihf-ranlib"), + ("LD", "arm-linux-gnueabihf-ld"), + ("OBJDUMP", "arm-linux-gnueabihf-objdump"), + ("CFLAGS", "-O2 -mcpu=cortex-a9 -mfpu=neon"), + ("CXXFLAGS", "-O2 -mcpu=cortex-a9 -mfpu=neon"), + ] +} + +/// Build a single library by name, dispatching to the recipe that +/// matches its upstream build system. +pub fn build_library(name: &str, build_dir: &Path) -> Result<()> { + println!("Building {name}..."); + + let env = cross_env(); + match name { + "zlib" => build_zlib(build_dir, &env), + "bzip2" => build_bzip2(build_dir), + "libpng" => build_libpng(build_dir), + "libjpeg" => build_libjpeg(build_dir, &env), + "openjpeg" => build_openjpeg(build_dir, &env), + "jbig2dec" => build_jbig2dec(build_dir, &env), + "libwebp" => build_libwebp(build_dir, &env), + "freetype2" => build_freetype2(build_dir), + "harfbuzz" => build_harfbuzz(build_dir), + "gumbo" => build_gumbo(build_dir, &env), + "djvulibre" => build_djvulibre(build_dir), + "mupdf" => super::mupdf::build_mupdf(build_dir), + _ => anyhow::bail!("unknown library: {name}"), + } +} + +fn build_zlib(build_dir: &Path, env: &[(&str, &str)]) -> Result<()> { + let zlib_env = { + let mut e: Vec<(&str, &str)> = env.to_vec(); + e.push(("CHOST", "arm-linux-gnueabihf")); + e + }; + cmd::run("./configure", &[], build_dir, &zlib_env).context("failed to configure zlib")?; + cmd::run( + "make", + &[ + "-j4", + "AR=arm-linux-gnueabihf-ar", + "ARFLAGS=rc", + "RANLIB=arm-linux-gnueabihf-ranlib", + ], + build_dir, + env, + ) + .context("failed to build zlib") +} + +fn build_bzip2(build_dir: &Path) -> Result<()> { + let cc = "arm-linux-gnueabihf-gcc"; + let cflags: Vec<&str> = + "-fpic -fPIC -Wall -Winline -O2 -mcpu=cortex-a9 -mfpu=neon -g -D_FILE_OFFSET_BITS=64" + .split_whitespace() + .collect(); + + let sources = [ + "blocksort.c", + "huffman.c", + "crctable.c", + "randtable.c", + "compress.c", + "decompress.c", + "bzlib.c", + ]; + + for src in &sources { + let obj = src.replace(".c", ".o"); + let mut args: Vec<&str> = vec!["-c", src, "-o", &obj]; + args.extend(cflags.iter()); + cmd::run(cc, &args, build_dir, &[]).with_context(|| format!("failed to compile {src}"))?; + } + + let objs: Vec = sources.iter().map(|s| s.replace(".c", ".o")).collect(); + let obj_refs: Vec<&str> = objs.iter().map(|s| s.as_str()).collect(); + + let mut link_args: Vec<&str> = vec![ + "-shared", + "-Wl,-soname", + "-Wl,libbz2.so.1.0", + "-o", + "libbz2.so.1.0.6", + ]; + link_args.extend(obj_refs.iter()); + cmd::run(cc, &link_args, build_dir, &[]).context("failed to link libbz2.so")?; + + let so_1_0 = build_dir.join("libbz2.so.1.0"); + if so_1_0.exists() { + std::fs::remove_file(&so_1_0)?; + } + #[cfg(unix)] + std::os::unix::fs::symlink("libbz2.so.1.0.6", &so_1_0)?; + + let so_unversioned = build_dir.join("libbz2.so"); + if so_unversioned.exists() { + std::fs::remove_file(&so_unversioned)?; + } + #[cfg(unix)] + std::os::unix::fs::symlink("libbz2.so.1.0.6", &so_unversioned)?; + + Ok(()) +} + +fn build_libpng(build_dir: &Path) -> Result<()> { + let env = [ + ("CC", "arm-linux-gnueabihf-gcc"), + ("CC_BUILD", "cc"), + ("CXX", "arm-linux-gnueabihf-g++"), + ("AR", "arm-linux-gnueabihf-ar"), + ("AS", "arm-linux-gnueabihf-as"), + ("LD", "arm-linux-gnueabihf-ld"), + ("NM", "arm-linux-gnueabihf-nm"), + ("OBJDUMP", "arm-linux-gnueabihf-objdump"), + ("RANLIB", "arm-linux-gnueabihf-ranlib"), + ("STRIP", "arm-linux-gnueabihf-strip"), + ("CFLAGS", "-O2 -mcpu=cortex-a9 -mfpu=neon"), + ("CXXFLAGS", "-O2 -mcpu=cortex-a9 -mfpu=neon"), + ("CPPFLAGS", "-I../zlib"), + ("LDFLAGS", "-L../zlib"), + ]; + cmd::run( + "./configure", + &["--host=arm-linux-gnueabihf"], + build_dir, + &env, + ) + .context("failed to configure libpng")?; + cmd::run("make", &["-j4"], build_dir, &env).context("failed to build libpng") +} + +fn build_libjpeg(build_dir: &Path, env: &[(&str, &str)]) -> Result<()> { + cmd::run( + "./configure", + &["--host=arm-linux-gnueabihf"], + build_dir, + env, + ) + .context("failed to configure libjpeg")?; + cmd::run("make", &["-j4"], build_dir, env).context("failed to build libjpeg")?; + + let include_link = build_dir.join("include"); + if !include_link.exists() { + #[cfg(unix)] + std::os::unix::fs::symlink(".", &include_link)?; + } + let lib_link = build_dir.join("lib"); + if !lib_link.exists() { + #[cfg(unix)] + std::os::unix::fs::symlink(".libs", &lib_link)?; + } + Ok(()) +} + +fn build_openjpeg(build_dir: &Path, env: &[(&str, &str)]) -> Result<()> { + let cmake_build = build_dir.join("build"); + if cmake_build.exists() { + std::fs::remove_dir_all(&cmake_build)?; + } + std::fs::create_dir(&cmake_build)?; + + cmd::run( + "cmake", + &[ + "-DCMAKE_BUILD_TYPE=Release", + "-DBUILD_CODEC=off", + "-DBUILD_STATIC_LIBS=off", + "-DCMAKE_SYSTEM_NAME=Linux", + "-DCMAKE_C_COMPILER=arm-linux-gnueabihf-gcc", + "-DCMAKE_CXX_COMPILER=arm-linux-gnueabihf-g++", + "-DCMAKE_AR=arm-linux-gnueabihf-ar", + "..", + ], + &cmake_build, + env, + ) + .context("failed to configure openjpeg")?; + cmd::run("make", &["-j4"], &cmake_build, env).context("failed to build openjpeg")?; + + let config_src = cmake_build.join("src/lib/openjp2/opj_config.h"); + let config_dest = build_dir.join("src/lib/openjp2/opj_config.h"); + if config_src.exists() { + std::fs::copy(&config_src, &config_dest)?; + } + Ok(()) +} + +fn build_jbig2dec(build_dir: &Path, env: &[(&str, &str)]) -> Result<()> { + cmd::run( + "./autogen.sh", + &["--host=arm-linux-gnueabihf"], + build_dir, + env, + ) + .context("failed to run autogen.sh for jbig2dec")?; + cmd::run("make", &["-j4"], build_dir, env).context("failed to build jbig2dec") +} + +fn build_libwebp(build_dir: &Path, env: &[(&str, &str)]) -> Result<()> { + cmd::run("make", &["distclean"], build_dir, &[]).ok(); + if !build_dir.join("configure").exists() { + cmd::run("sh", &["autogen.sh"], build_dir, &[("NOCONFIGURE", "1")]) + .context("failed to run autogen.sh for libwebp")?; + } + cmd::run( + "./configure", + &[ + "--host=arm-linux-gnueabihf", + "--enable-shared", + "--disable-static", + "--disable-libwebpmux", + "--enable-libwebpdecoder", + "--enable-libwebpdemux", + "--disable-webp-tools", + ], + build_dir, + env, + ) + .context("failed to configure libwebp")?; + cmd::run("make", &["-j4"], build_dir, env).context("failed to build libwebp") +} + +fn build_freetype2(build_dir: &Path) -> Result<()> { + let env = [ + ("CC", "arm-linux-gnueabihf-gcc"), + ("CC_BUILD", "cc"), + ("CXX", "arm-linux-gnueabihf-g++"), + ("AR", "arm-linux-gnueabihf-ar"), + ("AS", "arm-linux-gnueabihf-as"), + ("LD", "arm-linux-gnueabihf-ld"), + ("NM", "arm-linux-gnueabihf-nm"), + ("OBJDUMP", "arm-linux-gnueabihf-objdump"), + ("RANLIB", "arm-linux-gnueabihf-ranlib"), + ("STRIP", "arm-linux-gnueabihf-strip"), + ("CFLAGS", "-O2 -mcpu=cortex-a9 -mfpu=neon"), + ("CXXFLAGS", "-O2 -mcpu=cortex-a9 -mfpu=neon"), + ("ZLIB_CFLAGS", "-I../zlib"), + ("ZLIB_LIBS", "-L../zlib -lz"), + ("BZIP2_CFLAGS", "-I../bzip2"), + ("BZIP2_LIBS", "-L../bzip2 -lbz2"), + ("LIBPNG_CFLAGS", "-I../libpng"), + ("LIBPNG_LIBS", "-L../libpng/.libs -lpng16"), + ]; + + std::fs::create_dir_all(build_dir.join("objs"))?; + + let autogen = build_dir.join("autogen.sh"); + if autogen.exists() { + cmd::run("./autogen.sh", &[], build_dir, &env) + .context("failed to run autogen.sh for freetype2")?; + } + + cmd::run( + "./configure", + &[ + "--host=arm-linux-gnueabihf", + "--with-zlib=yes", + "--with-png=yes", + "--with-bzip2=yes", + "--with-harfbuzz=no", + "--with-brotli=no", + "--disable-static", + ], + build_dir, + &env, + ) + .context("failed to configure freetype2")?; + cmd::run("make", &["-j4"], build_dir, &env).context("failed to build freetype2") +} + +fn build_harfbuzz(build_dir: &Path) -> Result<()> { + cmd::run( + "meson", + &[ + "setup", + "-Dglib=disabled", + "-Dicu=disabled", + "-Dcairo=disabled", + "-Dfreetype=enabled", + "--cross-file", + "kobo-options.txt", + "build", + ], + build_dir, + &[], + ) + .context("failed to configure harfbuzz")?; + cmd::run("meson", &["compile", "-C", "build"], build_dir, &[]) + .context("failed to build harfbuzz") +} + +fn build_gumbo(build_dir: &Path, env: &[(&str, &str)]) -> Result<()> { + if !build_dir.join("configure").exists() { + cmd::run("./autogen.sh", &[], build_dir, env) + .context("failed to run autogen.sh for gumbo")?; + } + cmd::run( + "./configure", + &["--host=arm-linux-gnueabihf"], + build_dir, + env, + ) + .context("failed to configure gumbo")?; + cmd::run("make", &["-j4"], build_dir, env).context("failed to build gumbo") +} + +fn build_djvulibre(build_dir: &Path) -> Result<()> { + let parent = build_dir.parent().context("build_dir has no parent")?; + let jpeg_abs = + std::fs::canonicalize(parent.join("libjpeg")).context("failed to resolve libjpeg path")?; + let jpeg_str = jpeg_abs + .to_str() + .context("jpeg path not UTF-8")? + .to_string(); + let jpeg_cflags = format!("-I{}", jpeg_str); + let jpeg_libs = format!("-L{}/.libs -ljpeg", jpeg_str); + let jpeg_with_arg = format!("--with-jpeg={}", jpeg_str); + + let env = [ + ("CC", "arm-linux-gnueabihf-gcc"), + ("CXX", "arm-linux-gnueabihf-g++"), + ("AR", "arm-linux-gnueabihf-ar"), + ("AS", "arm-linux-gnueabihf-as"), + ("LD", "arm-linux-gnueabihf-ld"), + ("NM", "arm-linux-gnueabihf-nm"), + ("OBJDUMP", "arm-linux-gnueabihf-objdump"), + ("RANLIB", "arm-linux-gnueabihf-ranlib"), + ("STRIP", "arm-linux-gnueabihf-strip"), + ("CFLAGS", "-O2 -mcpu=cortex-a9 -mfpu=neon"), + ("CXXFLAGS", "-O2 -mcpu=cortex-a9 -mfpu=neon"), + ("JPEG_DIR", &jpeg_str), + ("JPEG_CFLAGS", &jpeg_cflags), + ("JPEG_LIBS", &jpeg_libs), + ]; + cmd::run( + "./autogen.sh", + &[ + "--host=arm-linux-gnueabihf", + "--disable-xmltools", + "--disable-desktopfiles", + &jpeg_with_arg, + ], + build_dir, + &env, + ) + .context("failed to run autogen.sh for djvulibre")?; + cmd::run("make", &["-j4"], build_dir, &env).context("failed to build djvulibre") +} diff --git a/crates/build-deps/src/build/kobo/source.rs b/crates/build-deps/src/build/kobo/source.rs new file mode 100644 index 00000000..81db732f --- /dev/null +++ b/crates/build-deps/src/build/kobo/source.rs @@ -0,0 +1,90 @@ +//! Prepare a clean, patched copy of each thirdparty library's source +//! tree inside the per-target build directory. +//! +//! The Kobo cross-build copies the submodule into +//! `target/cadmus-build-deps///`, then layers in the +//! build scripts and patches kept under `build-scripts//`. +//! +//! For MuPDF, the per-library `kobo.patch` is applied first, then the +//! shared WebP support patch series is applied via +//! [`crate::build::mupdf::apply_webp_patches_if_needed`]. + +use std::path::Path; + +use anyhow::{Context, Result}; + +use crate::build::mupdf; +use crate::cmd; + +/// Copy a library's source tree into `build_dir` and overlay any +/// build scripts kept under `build-scripts//` (typically +/// `kobo.patch`, `kobo-options.txt`, etc.). +/// +/// Skips git metadata, `build/`, `objs/` and `autom4te.cache/` via +/// [`super::cp_r`]. +pub fn copy_source(src_dir: &Path, build_dir: &Path, name: &str, root: &Path) -> Result<()> { + println!("Copying {name} source..."); + + super::cp_r(src_dir, build_dir)?; + + let scripts_dir = root.join("build-scripts").join(name); + if scripts_dir.exists() { + for entry in std::fs::read_dir(&scripts_dir) + .with_context(|| format!("failed to read build-scripts/{name}"))? + { + let entry = entry?; + let src = entry.path(); + let dest = build_dir.join(entry.file_name()); + std::fs::copy(&src, &dest).with_context(|| { + format!("failed to copy {} to {}", src.display(), dest.display()) + })?; + } + } + + Ok(()) +} + +/// Apply the `kobo.patch` (and, for MuPDF, the WebP support patch +/// series) to a freshly copied build tree. +/// +/// `patch --forward` is used so already-applied patches are skipped. +/// A `--dry-run` fallback detects patches that were applied in +/// reverse and treats them as no-ops. +pub fn apply_patches(build_dir: &Path, name: &str, root: &Path) -> Result<()> { + let patches_dir = root.join("build-scripts").join(name); + + let kobo_patch = patches_dir.join("kobo.patch"); + if kobo_patch.exists() { + let kobo_patch_str = kobo_patch + .to_str() + .context("kobo.patch path is not valid UTF-8")?; + match cmd::run( + "patch", + &["-p", "1", "-i", kobo_patch_str, "--forward"], + build_dir, + &[], + ) { + Ok(()) => {} + Err(_) => { + let dry_run = std::process::Command::new("patch") + .args(["-p", "1", "-i", kobo_patch_str, "--dry-run"]) + .current_dir(build_dir) + .output(); + if let Ok(output) = dry_run { + let out = String::from_utf8_lossy(&output.stdout); + if out.contains("Reversed") || out.contains("Skipping") { + return Ok(()); + } + } + return Err(anyhow::anyhow!("failed to apply kobo.patch for {name}")); + } + } + } + + if name == "mupdf" { + mupdf::apply_webp_patches_if_needed(build_dir, root) + .with_context(|| format!("failed to apply MuPDF WebP patches for {name}"))?; + } + + Ok(()) +} diff --git a/crates/build-deps/src/build/mod.rs b/crates/build-deps/src/build/mod.rs new file mode 100644 index 00000000..eab45f53 --- /dev/null +++ b/crates/build-deps/src/build/mod.rs @@ -0,0 +1,22 @@ +//! Build orchestration for the thirdparty C/C++ libraries. +//! +//! Two submodules cover the two build targets Cadmus supports: +//! +//! * [`kobo`] cross-compiles the libraries for the Kobo e-reader's +//! ARM CPU using the Linaro toolchain. +//! * [`native`] builds MuPDF and libwebp for the Linux/macOS host so +//! the rest of the workspace can run unit tests and the emulator +//! without cross-compilation. +//! +//! The [`mupdf`] module holds the cross-flow MuPDF source +//! preparation (WebP support patches) shared by both [`kobo`] and +//! [`native`]. +//! +//! The [`mupdf_wrapper`] module compiles the small C glue library +//! (`mupdf_wrapper.c`) that exposes a curated subset of MuPDF entry +//! points to Rust for both build flows. + +pub mod kobo; +pub mod mupdf; +pub mod mupdf_wrapper; +pub mod native; diff --git a/crates/build-deps/src/build/mupdf.rs b/crates/build-deps/src/build/mupdf.rs new file mode 100644 index 00000000..a682139d --- /dev/null +++ b/crates/build-deps/src/build/mupdf.rs @@ -0,0 +1,58 @@ +//! Shared MuPDF source preparation. +//! +//! Both the native and Kobo build flows need to apply the Cadmus +//! WebP support patch series to a MuPDF source tree before it is +//! compiled. The series is defined in +//! [`crate::versions::MUPDF_WEBP_PATCHES`] and is identical for both +//! targets, so the application logic lives here in one place. +//! +//! A `.webp-patched` marker file is written under the patched tree on +//! success. Re-application is skipped while the marker is present, +//! which keeps re-runs cheap when the build tree is reused (the +//! native flow) and stays correct when the build tree is recreated +//! from scratch (the Kobo flow). + +use std::path::Path; + +use anyhow::{Context, Result}; + +use crate::cmd; +use crate::markers; + +/// Apply the Cadmus WebP support patch series to a MuPDF source tree +/// if the patches have not been applied yet. +/// +/// A `.webp-patched` marker file is written under `mupdf_dir` after a +/// successful application and re-applications are skipped while it +/// exists. Returns `Ok(true)` when patches were applied during this +/// call, `Ok(false)` when they were already in place. +/// +/// # Errors +/// +/// Returns an error if the patch list cannot be enumerated, any patch +/// fails to apply, or the marker file cannot be written. +pub fn apply_webp_patches_if_needed(mupdf_dir: &Path, root: &Path) -> Result { + if markers::is_webp_patched(mupdf_dir) { + println!("MuPDF WebP patches already applied."); + return Ok(false); + } + + println!("Applying MuPDF WebP patches..."); + let patches_dir = root.join("build-scripts/mupdf"); + for patch in crate::versions::MUPDF_WEBP_PATCHES { + let patch_path = patches_dir.join(patch); + let patch_str = patch_path + .to_str() + .context("patch path is not valid UTF-8")?; + cmd::run("patch", &["-p", "1", "-i", patch_str], mupdf_dir, &[]) + .with_context(|| format!("failed to apply {patch}"))?; + } + + markers::write_marker( + mupdf_dir, + markers::WEBP_PATCHED_MARKER, + "mupdf", + "WebP patch", + )?; + Ok(true) +} diff --git a/crates/build-deps/src/build/mupdf_wrapper.rs b/crates/build-deps/src/build/mupdf_wrapper.rs new file mode 100644 index 00000000..242973b2 --- /dev/null +++ b/crates/build-deps/src/build/mupdf_wrapper.rs @@ -0,0 +1,119 @@ +//! Build the `mupdf_wrapper` C glue library that exposes a small set +//! of MuPDF entry points to Rust. +//! +//! The wrapper has two variants: one for the native host (Linux or +//! macOS) and one cross-compiled for Kobo (ARM). Both are emitted into +//! `target/mupdf_wrapper//` and statically linked into the +//! `cadmus` crate. +//! +//! The include path passed to the compiler must always point at the +//! MuPDF headers that match the source tree that was actually +//! compiled. For native builds that is the patched tree under +//! `target/cadmus-build-deps//mupdf/include/`. For Kobo the +//! submodule headers at `thirdparty/mupdf/include/` are used as-is. +//! +//! The compilation step goes through the [`cc`] crate so the build +//! system handles dependency caching, parallel compilation, compiler +//! detection and archive creation in one place. + +use std::path::Path; + +use anyhow::{Context, Result}; + +/// Build `libmupdf_wrapper.a` for the native host if it does not +/// already exist. +/// +/// `include` must point at the MuPDF `include/` directory whose +/// headers were used to build `libmupdf.a` (see +/// [`ensure_native_artifacts`][crate::build::native::ensure_native_artifacts]). +pub fn build_native_if_needed(root: &Path, include: &Path) -> Result<()> { + let target_os = native_target_os(); + let lib = native_lib_path(root, target_os); + + if lib.exists() { + println!("mupdf_wrapper already built for {target_os}."); + return Ok(()); + } + + build(root, target_os, None, include, &[]) +} + +fn native_target_os() -> &'static str { + if cfg!(target_os = "macos") { + "Darwin" + } else { + "Linux" + } +} + +fn native_lib_path(root: &Path, target_os: &str) -> std::path::PathBuf { + root.join(format!( + "target/mupdf_wrapper/{target_os}/libmupdf_wrapper.a" + )) +} + +/// Build `libmupdf_wrapper.a` for the Kobo (ARM) target, reusing the +/// existing archive when present. +/// +/// `include` must point at the patched MuPDF headers under +/// `target/cadmus-build-deps//mupdf/include` so the wrapper +/// sees the WebP support patches applied by +/// [`crate::build::kobo::source::apply_patches`]. When the archive +/// is already on disk the function returns without recompiling, so +/// the caller only needs to make sure the patched headers are on +/// disk when a build is actually required. +pub fn build_kobo(root: &Path, include: &Path) -> Result<()> { + let lib = kobo_lib_path(root); + if lib.exists() { + println!("mupdf_wrapper already built for Kobo."); + return Ok(()); + } + + build(root, "Kobo", Some("arm-linux-gnueabihf-gcc"), include, &[]) +} + +fn kobo_lib_path(root: &Path) -> std::path::PathBuf { + root.join("target/mupdf_wrapper/Kobo/libmupdf_wrapper.a") +} + +fn build( + root: &Path, + target_os: &str, + compiler: Option<&str>, + include: &Path, + extra_cflags: &[&str], +) -> Result<()> { + let wrapper_dir = root.join("mupdf_wrapper"); + let build_dir = root.join(format!("target/mupdf_wrapper/{target_os}")); + + std::fs::create_dir_all(&build_dir) + .with_context(|| format!("failed to create {}", build_dir.display()))?; + + let mut build = cc::Build::new(); + build + .file(wrapper_dir.join("mupdf_wrapper.c")) + .include(include) + .out_dir(&build_dir); + + for flag in extra_cflags { + build.flag(flag); + } + + if let Some(cc) = compiler { + build.compiler(cc); + } + + build + .try_compile("mupdf_wrapper") + .with_context(|| format!("failed to compile mupdf_wrapper.c for {target_os}"))?; + + // `cc::Build` leaves `mupdf_wrapper.o` next to the archive. + // Remove it so the output directory matches the shape produced by + // the previous shell-out implementation. + let obj = build_dir.join("mupdf_wrapper.o"); + if obj.exists() { + std::fs::remove_file(&obj).ok(); + } + + Ok(()) +} diff --git a/crates/build-deps/src/build/native.rs b/crates/build-deps/src/build/native.rs new file mode 100644 index 00000000..834187c5 --- /dev/null +++ b/crates/build-deps/src/build/native.rs @@ -0,0 +1,579 @@ +//! Build MuPDF and its companion shared libraries for the native host +//! (Linux or macOS) using system libraries via `pkg-config`. +//! +//! The patched MuPDF source tree is the canonical source of truth: the +//! WebP support patches are applied to a copy of the submodule under +//! `target/cadmus-build-deps//mupdf/`, and that copy is the one +//! compiled and the one whose headers must be used by the Rust build +//! script and the `mupdf_wrapper` C glue. + +use std::env; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; + +use crate::build::{kobo, mupdf}; +use crate::cmd; +use crate::markers; +use crate::versions::MUPDF_VERSION; + +/// Default Cargo target triple when `TARGET` is unset (build script context +/// outside of `cargo build`). +fn target_triple() -> String { + std::env::var("TARGET").unwrap_or_else(|_| "x86_64-unknown-linux-gnu".to_string()) +} + +/// Build-root directory used for native dependency builds. +pub fn build_root(root: &Path) -> PathBuf { + root.join("target/cadmus-build-deps").join(target_triple()) +} + +/// Outputs of [`ensure_native_artifacts`]. +pub struct NativeArtifacts { + /// Path to the patched MuPDF `include/` directory. The Rust build + /// script must pass this to the C wrapper compiler so that + /// `` resolves against the patched headers. + pub include: PathBuf, +} + +/// Build all native MuPDF/libwebp artifacts and return paths needed by +/// the rest of the build. +/// +/// Performs the following steps: +/// +/// 1. If the cached native build outputs (libwebp `.built` marker, +/// MuPDF `.built` marker, MuPDF `.a` archives, patched include +/// tree) are all present, skip submodule initialisation entirely +/// and only re-link the artifacts. This keeps warm-cache builds +/// fast and avoids the ~5 minute recursive submodule clone done by +/// CI when submodules are not yet on disk. +/// 2. Otherwise initialise git submodules, verify the MuPDF submodule +/// matches [`MUPDF_VERSION`], build +/// libwebp from source, copy the MuPDF source to a per-target build +/// directory, apply the WebP support patches if not already +/// applied, build MuPDF, and link its static archives into +/// `target/mupdf_wrapper//`. +/// +/// # Errors +/// +/// Returns an error if submodules cannot be initialised, the MuPDF +/// version does not match, or any of the underlying build steps fail. +pub fn ensure_native_artifacts(root: &Path) -> Result { + if !native_cache_complete(root) { + crate::ensure_submodules(root).context("failed to initialise git submodules")?; + } + + let mupdf_src = root.join("thirdparty/mupdf"); + let mupdf_build = build_root(root).join("mupdf"); + let version_header = mupdf_src.join("include/mupdf/fitz/version.h"); + + if !native_cache_complete(root) { + let current_version = read_mupdf_version(&version_header); + if current_version.as_deref() != Some(MUPDF_VERSION) { + bail!( + "MuPDF sources not found or version mismatch: have {:?}, need {}", + current_version, + MUPDF_VERSION + ); + } + + build_libwebp_native(root)?; + + if !mupdf_build.exists() { + kobo::cp_r(&mupdf_src, &mupdf_build).context("failed to copy mupdf source")?; + } + + mupdf::apply_webp_patches_if_needed(&mupdf_build, root) + .context("failed to apply MuPDF WebP patches")?; + + let mupdf_a = build_root(root).join("mupdf/build/release/libmupdf.a"); + if !mupdf_a.exists() { + build_mupdf_native(root).context("failed to build MuPDF")?; + } + } + + link_mupdf_artifacts(root).context("failed to link MuPDF artifacts")?; + + Ok(NativeArtifacts { + include: mupdf_build.join("include"), + }) +} + +/// Returns `true` when every native build artefact is already on disk +/// and a fresh build can be skipped. Used by +/// [`ensure_native_artifacts`] to avoid `git submodule update +/// --init --recursive` on warm CI caches. +fn native_cache_complete(root: &Path) -> bool { + let build_root = build_root(root); + let libwebp_dir = build_root.join("libwebp"); + let mupdf_dir = build_root.join("mupdf"); + let mupdf_a = mupdf_dir.join("build/release/libmupdf.a"); + let mupdf_third_a = mupdf_dir.join("build/release/libmupdf-third.a"); + let mupdf_include = mupdf_dir.join("include"); + + markers::is_built(&libwebp_dir) + && markers::is_built(&mupdf_dir) + && mupdf_a.exists() + && mupdf_third_a.exists() + && mupdf_include.is_dir() +} + +/// Build libwebp from source for native development using the host +/// compiler and `pkg-config`. +/// +/// The output (`src/.libs/libwebp.a`) is repacked from libwebp's +/// per-subsystem archives (see [`combine_libwebp_static_archives`]) so +/// downstream code can link with a single `-lwebp`. +/// +/// Idempotent: a `.built` marker file is written under +/// `target/cadmus-build-deps//libwebp/` and re-runs are skipped +/// while it exists. +/// +/// # Errors +/// +/// Returns an error if the host toolchain is missing or the configure +/// and `make` invocations fail. +pub fn build_libwebp_native(root: &Path) -> Result<()> { + let build_root = build_root(root); + let libwebp_dir = build_root.join("libwebp"); + + if markers::is_built(&libwebp_dir) { + println!("libwebp already built for native."); + return Ok(()); + } + + std::fs::create_dir_all(&libwebp_dir)?; + + let src_dir = root.join("thirdparty/libwebp"); + if !libwebp_dir.join("configure").exists() { + kobo::cp_r(&src_dir, &libwebp_dir)?; + } + + cmd::run("make", &["distclean"], &libwebp_dir, &[]).ok(); + + println!("Building libwebp for native development..."); + + cmd::run("sh", &["autogen.sh"], &libwebp_dir, &[("NOCONFIGURE", "1")]) + .context("failed to run autogen.sh for libwebp")?; + + cmd::run( + "./configure", + &[ + "--disable-shared", + "--enable-static", + "--disable-libwebpmux", + "--enable-libwebpdecoder", + "--enable-libwebpdemux", + "--disable-webp-tools", + "--with-pic", + ], + &libwebp_dir, + &[], + ) + .context("failed to configure libwebp")?; + + cmd::run("make", &["-j4"], &libwebp_dir, &[]).context("failed to build libwebp")?; + + combine_libwebp_static_archives(&libwebp_dir)?; + + let libwebp_a = libwebp_dir.join("src/.libs/libwebp.a"); + cmd::run( + "ranlib", + &[libwebp_a.to_str().context("non-UTF-8 libwebp.a path")?], + &libwebp_dir, + &[], + ) + .context("failed to ranlib libwebp.a")?; + + markers::mark_built(&libwebp_dir, "libwebp")?; + println!("✓ libwebp built successfully"); + Ok(()) +} + +/// libwebp's build system creates separate `.a` files in sub-directories +/// (`dec/`, `dsp/`, `enc/`, `utils/`) but does not assemble them into a +/// single `src/.libs/libwebp.a` when building static-only. Extract the +/// object files from each sub-archive and repack them into one +/// `libwebp.a` so downstream code can link with a single `-lwebp`. +fn combine_libwebp_static_archives(libwebp_dir: &Path) -> Result<()> { + let libs_dir = libwebp_dir.join("src/.libs"); + std::fs::create_dir_all(&libs_dir).context("failed to create src/.libs for libwebp")?; + + let sublibs = [ + "dec/.libs/libwebpdecode.a", + "dsp/.libs/libwebpdsp.a", + "enc/.libs/libwebpencode.a", + "utils/.libs/libwebputils.a", + ]; + + let ar_tool = resolve_ar_tool(); + let ranlib_tool = resolve_ranlib_tool(); + + let mut objects = Vec::new(); + for sublib in sublibs { + let archive_path = libwebp_dir.join("src").join(sublib); + let subdir = tempfile::Builder::new() + .prefix("libwebp-obj-") + .tempdir_in(&libs_dir) + .with_context(|| { + format!( + "failed to create temp dir for extracting {}", + archive_path.display() + ) + })?; + cmd::run( + &ar_tool, + &[ + "x", + archive_path.to_str().context("non-UTF-8 archive path")?, + ], + subdir.path(), + &[], + ) + .with_context(|| format!("failed to extract {} with `ar x`", archive_path.display()))?; + + for entry in std::fs::read_dir(subdir.path()) + .with_context(|| format!("failed to read {}", subdir.path().display()))? + { + let entry = entry + .with_context(|| format!("failed to read entry in {}", subdir.path().display()))?; + let path = entry.path(); + if entry.file_type().map(|t| t.is_file()).unwrap_or(false) { + objects.push(path); + } + } + + let _ = subdir.keep(); + } + + if objects.is_empty() { + bail!("no object files extracted from libwebp sub-archives"); + } + + let libwebp_a = libs_dir.join("libwebp.a"); + if libwebp_a.exists() { + std::fs::remove_file(&libwebp_a) + .with_context(|| format!("failed to remove existing {}", libwebp_a.display()))?; + } + + let mut args: Vec = vec![ + "rcs".to_string(), + libwebp_a + .file_name() + .context("could not get filename for libwebp_a")? + .to_string_lossy() + .into_owned(), + ]; + for obj in &objects { + let rel = obj + .strip_prefix(&libs_dir) + .with_context(|| format!("{} is not under {}", obj.display(), libs_dir.display()))?; + args.push(rel.to_string_lossy().into_owned()); + } + let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect(); + + cmd::run(&ar_tool, &arg_refs, &libs_dir, &[]) + .with_context(|| format!("failed to run `ar rcs` to build {}", libwebp_a.display()))?; + + cmd::run( + &ranlib_tool, + &[libwebp_a.to_str().context("non-UTF-8 libwebp.a path")?], + &libs_dir, + &[], + ) + .with_context(|| format!("failed to run `ranlib` on {}", libwebp_a.display()))?; + + Ok(()) +} + +/// Return the path to the host `ar` binary, honouring the `AR` env var +/// when set by Cargo or the user. The host `ar` produces archives that +/// the platform linker accepts, including the long-name table that +/// libwebp's per-subsystem archives require. +fn resolve_ar_tool() -> String { + env::var("AR") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "ar".to_string()) +} + +/// Return the path to the host `ranlib` binary, honouring the `RANLIB` +/// env var when set. +fn resolve_ranlib_tool() -> String { + env::var("RANLIB") + .ok() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| "ranlib".to_string()) +} + +/// Parse the `FZ_VERSION` macro out of MuPDF's `version.h`. +/// +/// Returns `None` if the file is missing, unreadable, or does not +/// contain a `FZ_VERSION` string literal. Used to verify the submodule +/// matches [`MUPDF_VERSION`] before +/// kicking off a build. +pub fn read_mupdf_version(header: &Path) -> Option { + let content = std::fs::read_to_string(header).ok()?; + + for line in content.lines() { + if line.contains("FZ_VERSION") && line.contains('"') { + let start = line.find('"')? + 1; + let end = line.rfind('"')?; + if start < end { + return Some(line[start..end].to_owned()); + } + } + } + + None +} + +/// Build MuPDF and libmupdf-third using system libraries for the native +/// host platform. +/// +/// Idempotent: a `.built` marker file is written under +/// `target/cadmus-build-deps//mupdf/` and re-runs are skipped +/// while it exists. +pub fn build_mupdf_native(root: &Path) -> Result<()> { + let build_root = build_root(root); + let mupdf_dir = build_root.join("mupdf"); + + if markers::is_built(&mupdf_dir) { + println!("MuPDF already built for native."); + return Ok(()); + } + + let src_dir = root.join("thirdparty/mupdf"); + if !mupdf_dir.exists() { + kobo::cp_r(&src_dir, &mupdf_dir)?; + } + + println!("Building MuPDF for native development..."); + + for entry in ["gitattributes", ".gitattributes"] { + let path = mupdf_dir.join(entry); + if path.exists() { + std::fs::remove_file(&path).ok(); + } + } + + cmd::run("make", &["clean"], &mupdf_dir, &[]).ok(); + cmd::run("make", &["verbose=yes", "generate"], &mupdf_dir, &[])?; + + let target = target_triple(); + let sys_cflags = collect_system_cflags()?; + let xcflags = format!( + "-DFZ_ENABLE_ICC=0 -DFZ_ENABLE_SPOT_RENDERING=0 \ + -DFZ_ENABLE_ODT_OUTPUT=0 -DFZ_ENABLE_OCR_OUTPUT=0 \ + -DHAVE_WEBP=1 -I{root}/target/cadmus-build-deps/{target}/libwebp/src {sys_cflags}", + root = root.display(), + target = target + ); + + let xlibs = format!( + "-L{root}/target/cadmus-build-deps/{target}/libwebp/src/.libs -lwebp \ + -L{root}/target/cadmus-build-deps/{target}/libwebp/src/demux/.libs -lwebpdemux", + root = root.display(), + target = target + ); + + cmd::run( + "make", + &[ + "verbose=yes", + "mujs=no", + "tesseract=no", + "extract=no", + "archive=no", + "brotli=no", + "barcode=no", + "commercial=no", + "USE_SYSTEM_LIBS=yes", + &format!("XCFLAGS={xcflags}"), + &format!("XLIBS={xlibs}"), + "libs", + ], + &mupdf_dir, + &[], + ) + .context("failed to build MuPDF libs")?; + + markers::mark_built(&mupdf_dir, "mupdf")?; + Ok(()) +} + +/// Collect MuPDF CFLAGS via `pkg-config` on macOS only. +/// +/// On Linux, MuPDF's build system detects system libraries +/// automatically. On macOS, explicit CFLAGS gathered from pkg-config +/// must be injected through `XCFLAGS` so that the headers for +/// freetype, harfbuzz, etc. resolve correctly. +fn collect_system_cflags() -> Result { + if !cfg!(target_os = "macos") { + return Ok(String::new()); + } + + let libs = [ + "freetype2", + "harfbuzz", + "libopenjp2", + "libjpeg", + "libwebp", + "zlib", + "jbig2dec", + "gumbo", + ]; + + let mut flags = String::new(); + for lib in libs { + if let Ok(f) = cmd::output("pkg-config", &["--cflags", lib], Path::new("."), &[]) + && !f.is_empty() + { + flags.push(' '); + flags.push_str(&f); + } + } + + Ok(flags.trim().to_owned()) +} + +/// Symlink the freshly built `libmupdf.a` and `libmupdf-third.a` into +/// `target/mupdf_wrapper//` so the Rust build script can +/// pick them up via `cargo:rustc-link-search`. +pub fn link_mupdf_artifacts(root: &Path) -> Result<()> { + let platform_dir = if cfg!(target_os = "macos") { + "Darwin" + } else { + "Linux" + }; + + let target_dir = root.join(format!("target/mupdf_wrapper/{platform_dir}")); + std::fs::create_dir_all(&target_dir) + .context("failed to create target/mupdf_wrapper directory")?; + + let release_dir = build_root(root).join("mupdf/build/release"); + + let libmupdf = release_dir.join("libmupdf.a"); + if !libmupdf.exists() { + bail!("libmupdf.a not found after build -- check MuPDF build output"); + } + + symlink_force(&libmupdf, &target_dir.join("libmupdf.a"))?; + println!("✓ Created libmupdf.a in target/mupdf_wrapper/{platform_dir}"); + + let libmupdf_third = release_dir.join("libmupdf-third.a"); + if !libmupdf_third.exists() { + println!("Creating empty libmupdf-third.a (system libs used instead)..."); + write_empty_archive(&libmupdf_third).context("failed to create empty libmupdf-third.a")?; + } + + symlink_force(&libmupdf_third, &target_dir.join("libmupdf-third.a"))?; + println!("✓ Created libmupdf-third.a"); + + Ok(()) +} + +/// Create a symlink at `link` pointing to `target`, removing any +/// existing file or symlink at `link` first. +fn symlink_force(target: &Path, link: &Path) -> Result<()> { + if link.exists() || link.symlink_metadata().is_ok() { + std::fs::remove_file(link) + .with_context(|| format!("failed to remove existing {}", link.display()))?; + } + + #[cfg(unix)] + std::os::unix::fs::symlink(target, link) + .with_context(|| format!("failed to create symlink {}", link.display()))?; + + #[cfg(not(unix))] + std::fs::copy(target, link) + .with_context(|| format!("failed to copy {} to {}", target.display(), link.display()))?; + + Ok(()) +} + +/// Write an empty `ar` archive at `path` using the host `ar` tool. +fn write_empty_archive(path: &Path) -> Result<()> { + let ar_tool = resolve_ar_tool(); + let parent = path.parent().context("empty archive path has no parent")?; + std::fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + let path_str = path.to_str().context("non-UTF-8 empty archive path")?; + cmd::run(&ar_tool, &["rcs", path_str], parent, &[]) + .with_context(|| format!("failed to run `ar rcs` to create {}", path.display()))?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn read_mupdf_version_parses_define() { + let tmp = tempfile::tempdir().unwrap(); + let header = tmp.path().join("version.h"); + std::fs::write( + &header, + r#"/* MuPDF version */ +#define FZ_VERSION "1.27.0" +#define FZ_VERSION_MAJOR 1 +"#, + ) + .unwrap(); + + let version = read_mupdf_version(&header); + assert_eq!(version.as_deref(), Some("1.27.0")); + } + + #[test] + fn read_mupdf_version_returns_none_for_malformed_header() { + let tmp = tempfile::tempdir().unwrap(); + let header = tmp.path().join("version.h"); + std::fs::write(&header, "/* no version define here */\n").unwrap(); + + let version = read_mupdf_version(&header); + assert!(version.is_none()); + } + + #[test] + fn apply_mupdf_webp_patches_writes_marker_and_is_idempotent() { + let root = tempfile::tempdir().unwrap(); + let mupdf = root.path().join("mupdf"); + let patches = root.path().join("build-scripts/mupdf"); + std::fs::create_dir_all(&patches).unwrap(); + + let source = mupdf.join("hello.txt"); + std::fs::create_dir_all(source.parent().unwrap()).unwrap(); + std::fs::write(&source, "first\n").unwrap(); + + let patch = patches.join("hello-kobo.patch"); + std::fs::write( + &patch, + "--- a/hello.txt\n+++ b/hello.txt\n@@ -1 +1 @@\n-first\n+second\n", + ) + .unwrap(); + + // We can't easily override `MUPDF_WEBP_PATCHES` from the test + // because it is a `pub const`. Instead, exercise the marker + // helper directly: writing the marker twice must leave a + // single file behind and `is_webp_patched` must report true + // on the second call. + let marker_dir = mupdf.clone(); + crate::markers::write_marker( + &marker_dir, + crate::markers::WEBP_PATCHED_MARKER, + "mupdf", + "WebP patch", + ) + .unwrap(); + assert!(crate::markers::is_webp_patched(&marker_dir)); + crate::markers::write_marker( + &marker_dir, + crate::markers::WEBP_PATCHED_MARKER, + "mupdf", + "WebP patch", + ) + .unwrap(); + assert!(crate::markers::is_webp_patched(&marker_dir)); + } +} diff --git a/crates/build-deps/src/cmd.rs b/crates/build-deps/src/cmd.rs new file mode 100644 index 00000000..baed7da9 --- /dev/null +++ b/crates/build-deps/src/cmd.rs @@ -0,0 +1,67 @@ +//! Shell-out helpers used by the rest of `build_deps`. +//! +//! All build steps fork out to the host or cross toolchain (`make`, +//! `cmake`, `meson`, `ar`, `patch`, `readelf`, ...). The helpers in +//! this module wrap those invocations so that callers can write +//! straight-line code without manually wiring up status checks, error +//! contexts or working-directory handling. + +use std::{ + ffi::OsStr, + path::Path, + process::{Command, ExitStatus}, +}; + +use anyhow::{Context, Result, bail}; + +/// Run `program` with `args` in `dir`, inheriting the parent +/// environment plus `env`. The invocation is echoed to stdout and any +/// non-zero exit is reported as an error. +pub fn run(program: &str, args: &[&str], dir: &Path, env: &[(&str, &str)]) -> Result<()> { + let status = build_command(program, args, dir, env) + .status() + .with_context(|| format!("failed to spawn `{program}`"))?; + + check_status(program, status) +} + +/// Run `program` with `args` in `dir` and return its trimmed stdout +/// as UTF-8. Non-zero exits and non-UTF-8 output are surfaced as +/// errors. +pub fn output(program: &str, args: &[&str], dir: &Path, env: &[(&str, &str)]) -> Result { + let out = build_command(program, args, dir, env) + .output() + .with_context(|| format!("failed to spawn `{program}`"))?; + + check_status(program, out.status)?; + + let stdout = String::from_utf8(out.stdout) + .with_context(|| format!("`{program}` produced non-UTF-8 output"))?; + + Ok(stdout.trim().to_owned()) +} + +fn build_command(program: &str, args: &[&str], dir: &Path, env: &[(&str, &str)]) -> Command { + let display_args = args.join(" "); + println!("$ {program} {display_args}"); + + let mut cmd = Command::new(program); + cmd.args(args).current_dir(dir); + + for (key, value) in env { + cmd.env(OsStr::new(key), OsStr::new(value)); + } + + cmd +} + +fn check_status(program: &str, status: ExitStatus) -> Result<()> { + if status.success() { + return Ok(()); + } + + match status.code() { + Some(code) => bail!("`{program}` exited with status {code}"), + None => bail!("`{program}` was terminated by a signal"), + } +} diff --git a/crates/build-deps/src/lib.rs b/crates/build-deps/src/lib.rs new file mode 100644 index 00000000..f69a399b --- /dev/null +++ b/crates/build-deps/src/lib.rs @@ -0,0 +1,67 @@ +//! Build helpers shared between the `cadmus` crate's `build.rs` and +//! the `xtask` binary. +//! +//! The crate is intentionally free of UI and runtime logic: it owns +//! the recipes and orchestration required to produce the third-party +//! C/C++ artifacts the rest of the workspace links against, namely +//! MuPDF, libwebp and the small set of compression and font libraries +//! that come with the Kobo build. +//! +//! Consumers should treat the public API in this crate as a stable +//! contract: a single public entry point per concern. +//! +//! # Native vs Kobo +//! +//! Two build flows are supported and they are intentionally kept +//! separate: +//! +//! * The **native** flow (Linux or macOS development hosts) uses +//! system libraries through `pkg-config` and is exposed by +//! [`build::native`]. The MuPDF source tree is copied into a +//! per-target directory, the WebP support patches are applied, and +//! that patched tree is the canonical source of truth used both for +//! the compiled library and for the `mupdf_wrapper` C glue. +//! * The **Kobo** cross-build targets `arm-unknown-linux-gnueabihf` +//! and is exposed by [`build::kobo`]. All libraries are built from +//! source from the git submodules under `thirdparty/`, with +//! per-library recipes in [`build::kobo::recipes`] and the final +//! `libmupdf.so` produced by [`build::kobo::mupdf`]. +//! +//! Both flows rely on the git submodules declared in `.gitmodules`. +//! [`ensure_submodules`] is the single entry point used to make sure +//! the working copy is ready before any other function is called. + +pub mod build; +pub mod cmd; +pub mod markers; +pub mod versions; + +use std::path::Path; + +use anyhow::{Context, Result}; + +/// Initialise git submodules. Required by both the native and Kobo +/// build flows because every thirdparty dependency lives in a +/// submodule under `thirdparty/`. +/// +/// Returns an error when `.gitmodules` is missing (i.e. the working +/// copy was not produced by a recursive git clone) or when `git +/// submodule update --init --recursive` fails. +pub fn ensure_submodules(root: &Path) -> Result<()> { + let gitmodules = root.join(".gitmodules"); + if !gitmodules.exists() { + return Err(anyhow::anyhow!( + ".gitmodules not found -- is this a git clone? Run: git clone --recursive" + )); + } + + println!("Initializing git submodules..."); + crate::cmd::run( + "git", + &["submodule", "update", "--init", "--recursive"], + root, + &[], + ) + .context("failed to initialize git submodules")?; + Ok(()) +} diff --git a/crates/build-deps/src/markers.rs b/crates/build-deps/src/markers.rs new file mode 100644 index 00000000..670d52e9 --- /dev/null +++ b/crates/build-deps/src/markers.rs @@ -0,0 +1,56 @@ +//! Marker files written into build directories so subsequent builds can +//! skip work that is already done. +//! +//! Marker files live next to the artifacts they describe. Removing +//! them is the supported way to force a rebuild of just the affected +//! library without clearing the whole target directory. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; + +/// File name written into a MuPDF source tree after the WebP support +/// patches have been applied. Presence of this file indicates the +/// patches are already in place and re-application can be skipped. +pub const WEBP_PATCHED_MARKER: &str = ".webp-patched"; + +/// File name written into a per-library build directory after the +/// library's build recipe has completed successfully. Presence of +/// this file indicates the build is cached and can be skipped. +pub const BUILT_MARKER: &str = ".built"; + +/// Returns the absolute path of the [`BUILT_MARKER`] for `dir`. +pub fn built_marker_path(dir: &Path) -> PathBuf { + dir.join(BUILT_MARKER) +} + +/// Returns `true` if [`BUILT_MARKER`] is present in `dir`. +pub fn is_built(dir: &Path) -> bool { + built_marker_path(dir).exists() +} + +/// Write [`BUILT_MARKER`] in `dir`, recording that `name` has been +/// built successfully. +/// +/// # Errors +/// +/// Returns an error if the marker file cannot be written. +pub fn mark_built(dir: &Path, name: &str) -> Result<()> { + write_marker(dir, BUILT_MARKER, name, "build") +} + +/// Returns `true` if [`WEBP_PATCHED_MARKER`] is present in `mupdf_dir`. +pub fn is_webp_patched(mupdf_dir: &Path) -> bool { + mupdf_dir.join(WEBP_PATCHED_MARKER).exists() +} + +/// Write an empty marker file at `/`, recording that the +/// build step named `name` (described as `state`) has completed. +/// +/// # Errors +/// +/// Returns an error if the marker file cannot be written. +pub fn write_marker(dir: &Path, marker: &str, name: &str, state: &str) -> Result<()> { + std::fs::write(dir.join(marker), "") + .with_context(|| format!("failed to write {state} marker for {name}")) +} diff --git a/crates/build-deps/src/versions.rs b/crates/build-deps/src/versions.rs new file mode 100644 index 00000000..03788054 --- /dev/null +++ b/crates/build-deps/src/versions.rs @@ -0,0 +1,117 @@ +//! Build-time constants that pin the thirdparty toolchain to specific +//! versions and document the layout each build flow expects on disk. +//! +//! With git submodules, individual library versions are tracked by +//! the submodule commit SHAs in `.gitmodules`. Only the constants +//! that the build code needs to know about at run time are kept +//! here. + +/// MuPDF version expected in `thirdparty/mupdf/include/mupdf/fitz/version.h`. +/// +/// Changing this constant will cause the native build to panic on +/// the first run after the change unless the submodule is also +/// updated. The CI cache key in +/// The shared cargo cache key in `.github/workflows/cargo.yml` includes +/// so caches stay consistent. +pub const MUPDF_VERSION: &str = "1.27.2"; + +/// All thirdparty libraries in dependency order for cross-compiling +/// to the Kobo target. Order matters: each library is built with the +/// previous one available on its include/library search path. +pub const LIBRARY_NAMES: &[&str] = &[ + "zlib", + "bzip2", + "libpng", + "libjpeg", + "openjpeg", + "jbig2dec", + "libwebp", + "freetype2", + "harfbuzz", + "gumbo", + "djvulibre", + "mupdf", +]; + +/// Base names of the thirdparty shared libraries that must end up in +/// the Kobo `libs/` directory. +/// +/// The base name (e.g. `libz.so`) is what the Cadmus runtime loads; +/// the actual SONAME suffix (`libz.so.1.2.13`) is discovered at +/// runtime via `arm-linux-gnueabihf-readelf -d` because upstream +/// libraries do not follow a consistent ABI versioning scheme. +pub const SONAMES: &[&str] = &[ + "libz.so", + "libbz2.so", + "libpng16.so", + "libjpeg.so", + "libopenjp2.so", + "libjbig2dec.so", + "libfreetype.so", + "libharfbuzz.so", + "libgumbo.so", + "libwebp.so", + "libwebpdemux.so", + "libdjvulibre.so", + "libmupdf.so", +]; + +/// Patch series applied to the MuPDF source tree to add WebP support. +/// +/// Applied by [`crate::build::mupdf::apply_webp_patches_if_needed`] +/// and by [`crate::build::kobo::source::apply_patches`] when building MuPDF. +pub const MUPDF_WEBP_PATCHES: &[&str] = &[ + "webp-upstream-697749-kobo.patch", + "webp-image-h-kobo.patch", + "webp-load-webp-deviations-kobo.patch", +]; + +/// Cross-compilation environment variables injected when `cargo xtask +/// build-kobo` runs `cargo build` for the Kobo ARM target. +pub const CROSS_ENV: &[(&str, &str)] = &[ + ("PKG_CONFIG_ALLOW_CROSS", "1"), + ( + "CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER", + "arm-linux-gnueabihf-gcc", + ), + ("CC_arm_unknown_linux_gnueabihf", "arm-linux-gnueabihf-gcc"), + ("AR_arm_unknown_linux_gnueabihf", "arm-linux-gnueabihf-ar"), +]; + +/// Mapping from built `.so` paths to their destination names in the +/// Kobo `libs/` directory. +/// +/// The source paths are relative to the workspace root and use the +/// `thirdparty//...` form so that the same table can be +/// re-targeted at a per-target build root by stripping the +/// `thirdparty/` prefix (see [`crate::build::kobo::copy_built_libs`]). +pub const BUILT_LIBRARY_COPIES: &[(&str, &str)] = &[ + ("thirdparty/zlib/libz.so", "libz.so"), + ("thirdparty/bzip2/libbz2.so", "libbz2.so"), + ("thirdparty/libpng/.libs/libpng16.so", "libpng16.so"), + ("thirdparty/libjpeg/.libs/libjpeg.so", "libjpeg.so"), + ( + "thirdparty/openjpeg/build/bin/libopenjp2.so", + "libopenjp2.so", + ), + ("thirdparty/jbig2dec/.libs/libjbig2dec.so", "libjbig2dec.so"), + ("thirdparty/libwebp/src/.libs/libwebp.so", "libwebp.so"), + ( + "thirdparty/libwebp/src/demux/.libs/libwebpdemux.so", + "libwebpdemux.so", + ), + ( + "thirdparty/freetype2/objs/.libs/libfreetype.so", + "libfreetype.so", + ), + ( + "thirdparty/harfbuzz/build/src/libharfbuzz.so", + "libharfbuzz.so", + ), + ("thirdparty/gumbo/.libs/libgumbo.so", "libgumbo.so"), + ( + "thirdparty/djvulibre/libdjvu/.libs/libdjvulibre.so", + "libdjvulibre.so", + ), + ("thirdparty/mupdf/build/release/libmupdf.so", "libmupdf.so"), +]; diff --git a/crates/cadmus/Cargo.toml b/crates/cadmus/Cargo.toml index 18d7343d..ceaaa8e6 100644 --- a/crates/cadmus/Cargo.toml +++ b/crates/cadmus/Cargo.toml @@ -20,3 +20,6 @@ kobo = ["cadmus-core/kobo"] cadmus-core = { path = "../core" } tikv-jemallocator = { version = "0.6", features = ["profiling"], optional = true } tracing = "0.1" + +[lints] +workspace = true diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 4ec5baed..60569b56 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -110,6 +110,9 @@ name = "dictionary_bench" harness = false [build-dependencies] +anyhow = "1" +build-deps = { path = "../build-deps" } +cc = { workspace = true } uuid = { version = "1.20.0", features = ["v7"] } [features] @@ -131,5 +134,5 @@ telemetry = ["tracing", "profiling"] bench = [] -[lints.rustdoc] -private_intra_doc_links = "allow" +[lints] +workspace = true diff --git a/crates/core/build.rs b/crates/core/build.rs index 4110a687..75081d65 100644 --- a/crates/core/build.rs +++ b/crates/core/build.rs @@ -1,9 +1,30 @@ -use std::env::{self, VarError}; +//! Build script for the `core` crate. +//! +//! All third-party dependency orchestration lives in the +//! `build_deps` crate. This script's responsibilities are limited to: +//! +//! 1. Emitting compile-time metadata (git version, PR info, build +//! provenance). +//! 2. Asking `build_deps` to produce the MuPDF/libwebp artifacts +//! needed for the active target. +//! 3. Translating those artifacts into `cargo:rustc-link-*` +//! directives for the Rust compiler. +//! 4. Generating the locales and bundled asset manifests. +//! +//! The whole script returns `Result`; the single `panic!` lives in +//! [`main`] so failures surface as a coherent error chain with cargo's +//! expected non-zero exit. + +use std::env; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; use uuid::Uuid; +use anyhow::{Context, Result, bail}; +use build_deps::build::kobo; +use build_deps::build::{mupdf_wrapper, native}; + const BUNDLED_ASSET_DIRS: &[&str] = &[ "bin", "css", @@ -18,21 +39,41 @@ const BUNDLED_ASSET_DIRS: &[&str] = &[ /// Set this to any changing value to force build metadata to refresh. const FORCE_REBUILD_ENV: &str = "FORCE_REBUILD"; +/// When set, skip building and linking third-party native dependencies. +/// Used by `cargo doc` and other no-link workflows where only Rust source +/// analysis and metadata generation are needed. +const SKIP_THIRDPARTY_DEPS_ENV: &str = "CADMUS_SKIP_THIRDPARTY_DEPS"; + fn main() { - let target = env::var("TARGET").unwrap(); + if let Err(err) = try_main() { + panic!("cadmus build script failed: {err:#}"); + } +} + +/// Returns `true` when `CADMUS_SKIP_THIRDPARTY_DEPS` is set, +/// indicating that native dependency building and linking should be skipped. +fn skip_thirdparty_deps() -> bool { + env::var_os(SKIP_THIRDPARTY_DEPS_ENV).is_some() +} + +fn try_main() -> Result<()> { + let target = env::var("TARGET").context("TARGET not set")?; println!("cargo:rerun-if-changed=.git/HEAD"); println!("cargo:rerun-if-env-changed={FORCE_REBUILD_ENV}"); - let (git_version, pr_info) = get_version_info().expect("Failed to get version info"); - println!("cargo:rustc-env=GIT_VERSION={}", git_version); + println!("cargo:rerun-if-env-changed={SKIP_THIRDPARTY_DEPS_ENV}"); + println!("cargo:rerun-if-env-changed=TARGET"); + + let (git_version, pr_info) = get_version_info()?; + println!("cargo:rustc-env=GIT_VERSION={git_version}"); if let Some(pr) = pr_info { - println!("cargo:rustc-env=PR_INFO={}", pr); + println!("cargo:rustc-env=PR_INFO={pr}"); } let build_uuid = Uuid::now_v7().to_string(); - println!("cargo:rustc-env=BUILD_UUID={}", build_uuid); + println!("cargo:rustc-env=BUILD_UUID={build_uuid}"); - let build_attributes = get_build_attributes(); + let build_attributes = get_build_attributes()?; println!("cargo:rustc-env=BUILD_USER={}", build_attributes.user); println!("cargo:rustc-env=BUILD_HOST={}", build_attributes.host); println!( @@ -40,40 +81,79 @@ fn main() { build_attributes.timestamp ); - // GitHub OAuth App client ID for device flow authentication. println!("cargo:rerun-if-env-changed=GH_OAUTH_CLIENT_ID"); let client_id = env::var("GH_OAUTH_CLIENT_ID").unwrap_or_else(|_| "GH_OAUTH_CLIENT_ID_NOT_SET".to_string()); - println!("cargo:rustc-env=GH_OAUTH_CLIENT_ID={}", client_id); - - // Cross-compiling for Kobo. - if target == "arm-unknown-linux-gnueabihf" { - println!("cargo:rustc-env=PKG_CONFIG_ALLOW_CROSS=1"); - println!("cargo:rustc-link-search=target/mupdf_wrapper/Kobo"); - println!("cargo:rustc-link-search=libs"); - println!("cargo:rustc-link-lib=dylib=stdc++"); - // Handle the Linux and macOS platforms. - } else { - let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); - match target_os.as_ref() { - "linux" => { - println!("cargo:rustc-link-search=target/mupdf_wrapper/Linux"); - println!("cargo:rustc-link-search=native=thirdparty/libwebp/src/.libs"); - println!("cargo:rustc-link-search=native=thirdparty/libwebp/src/demux/.libs"); - println!("cargo:rustc-link-lib=dylib=stdc++"); - } - "macos" => { - println!("cargo:rustc-link-search=target/mupdf_wrapper/Darwin"); - println!("cargo:rustc-link-search=native=thirdparty/libwebp/src/.libs"); - println!("cargo:rustc-link-search=native=thirdparty/libwebp/src/demux/.libs"); - println!("cargo:rustc-link-lib=dylib=c++"); - } - _ => panic!("Unsupported platform: {}.", target_os), + println!("cargo:rustc-env=GH_OAUTH_CLIENT_ID={client_id}"); + + let root = workspace_root()?; + + if !skip_thirdparty_deps() { + if target == "arm-unknown-linux-gnueabihf" { + emit_kobo_link_directives(); + kobo::ensure_kobo_artifacts(&root)?; + } else { + let artifacts = + native::ensure_native_artifacts(&root).context("failed to build native deps")?; + mupdf_wrapper::build_native_if_needed(&root, &artifacts.include) + .context("failed to build mupdf_wrapper")?; + emit_native_link_directives(&root, &target)?; } - println!("cargo:rustc-link-lib=mupdf-third"); + emit_common_link_directives(); + } + + generate_locales()?; + generate_bundled_assets()?; + + Ok(()) +} + +fn emit_native_link_directives(root: &Path, target: &str) -> Result<()> { + let target_os = env::var("CARGO_CFG_TARGET_OS").context("CARGO_CFG_TARGET_OS not set")?; + let build_deps = root.join("target/cadmus-build-deps").join(target); + let libwebp = build_deps.join("libwebp"); + + match target_os.as_ref() { + "linux" => { + println!("cargo:rustc-link-search=target/mupdf_wrapper/Linux"); + println!( + "cargo:rustc-link-search=native={}/src/.libs", + libwebp.display() + ); + println!( + "cargo:rustc-link-search=native={}/src/demux/.libs", + libwebp.display() + ); + println!("cargo:rustc-link-lib=dylib=stdc++"); + } + "macos" => { + println!("cargo:rustc-link-search=target/mupdf_wrapper/Darwin"); + println!( + "cargo:rustc-link-search=native={}/src/.libs", + libwebp.display() + ); + println!( + "cargo:rustc-link-search=native={}/src/demux/.libs", + libwebp.display() + ); + println!("cargo:rustc-link-lib=dylib=c++"); + } + other => bail!("unsupported platform: {other}"), } + println!("cargo:rustc-link-lib=mupdf-third"); + Ok(()) +} + +fn emit_kobo_link_directives() { + println!("cargo:rustc-env=PKG_CONFIG_ALLOW_CROSS=1"); + println!("cargo:rustc-link-search=target/mupdf_wrapper/Kobo"); + println!("cargo:rustc-link-search=libs"); + println!("cargo:rustc-link-lib=dylib=stdc++"); +} + +fn emit_common_link_directives() { println!("cargo:rustc-link-lib=z"); println!("cargo:rustc-link-lib=bz2"); println!("cargo:rustc-link-lib=jpeg"); @@ -83,16 +163,14 @@ fn main() { println!("cargo:rustc-link-lib=jbig2dec"); println!("cargo:rustc-link-lib=webp"); println!("cargo:rustc-link-lib=webpdemux"); - - generate_locales(); - generate_bundled_assets(); } -fn generate_locales() { +fn generate_locales() -> Result<()> { println!("cargo:rerun-if-changed=i18n/"); - let out_dir = env::var("OUT_DIR").expect("OUT_DIR not set"); - let mut locales: Vec = std::fs::read_dir("i18n/") - .expect("i18n/ directory not found") + let out_dir = env::var("OUT_DIR").context("OUT_DIR not set")?; + let locales_dir = Path::new("i18n"); + let mut locales: Vec = std::fs::read_dir(locales_dir) + .with_context(|| format!("i18n/ directory not found: {}", locales_dir.display()))? .filter_map(|entry| { let entry = entry.ok()?; if entry.file_type().ok()?.is_dir() { @@ -105,8 +183,9 @@ fn generate_locales() { locales.sort(); let entries: String = locales.iter().map(|l| format!(" \"{l}\",\n")).collect(); let generated = format!("pub const AVAILABLE_LOCALES: &[&str] = &[\n{entries}];\n"); - std::fs::write(std::path::Path::new(&out_dir).join("locales.rs"), generated) - .expect("failed to write locales.rs"); + std::fs::write(Path::new(&out_dir).join("locales.rs"), generated) + .context("failed to write locales.rs")?; + Ok(()) } /// Generates the bundled asset manifest. @@ -116,9 +195,9 @@ fn generate_locales() { /// /// This ensures that e.g. there are no lingering scripts, fonts or libs etc /// when they are removed. -fn generate_bundled_assets() { - let out_dir = env::var("OUT_DIR").expect("OUT_DIR not set"); - let workspace_root = workspace_root(); +fn generate_bundled_assets() -> Result<()> { + let out_dir = env::var("OUT_DIR").context("OUT_DIR not set")?; + let workspace_root = workspace_root()?; let is_kobo = env::var("TARGET").unwrap_or_default() == "arm-unknown-linux-gnueabihf"; let is_release = env::var("PROFILE").unwrap_or_default() == "release"; let mut asset_files = Vec::new(); @@ -131,14 +210,15 @@ fn generate_bundled_assets() { && matches!(*dir, "bin" | "resources" | "hyphenation-patterns") && !asset_dir.is_dir() { - panic!( + bail!( "required asset directory missing: {}. Run `cargo xtask download-assets` before build.", asset_dir.display() ); } println!("cargo:rerun-if-changed={}", asset_dir.display()); - collect_asset_files(&workspace_root, &asset_dir, &mut asset_files); + collect_asset_files(&workspace_root, &asset_dir, &mut asset_files) + .with_context(|| format!("failed to collect asset files under {}", dir))?; } asset_files.sort(); @@ -150,36 +230,40 @@ fn generate_bundled_assets() { let generated = format!("const BUNDLED_ASSET_FILES: &[&str] = &[\n{entries}];\n"); std::fs::write(Path::new(&out_dir).join("bundled_assets.rs"), generated) - .expect("failed to write bundled_assets.rs"); + .context("failed to write bundled_assets.rs")?; + Ok(()) } -fn workspace_root() -> PathBuf { +fn workspace_root() -> Result { let manifest_dir = - PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set")); - + PathBuf::from(env::var("CARGO_MANIFEST_DIR").context("CARGO_MANIFEST_DIR not set")?); manifest_dir .parent() .and_then(Path::parent) - .expect("workspace root not found") - .to_path_buf() + .context("workspace root not found") + .map(Path::to_path_buf) } -fn collect_asset_files(workspace_root: &Path, dir: &Path, asset_files: &mut Vec) { +fn collect_asset_files( + workspace_root: &Path, + dir: &Path, + asset_files: &mut Vec, +) -> Result<()> { if !dir.exists() { - return; + return Ok(()); } for entry in - std::fs::read_dir(dir).unwrap_or_else(|_| panic!("failed to read {}", dir.display())) + std::fs::read_dir(dir).with_context(|| format!("failed to read {}", dir.display()))? { - let entry = entry.unwrap_or_else(|_| panic!("failed to read entry in {}", dir.display())); + let entry = entry.with_context(|| format!("failed to read entry in {}", dir.display()))?; let path = entry.path(); let file_type = entry .file_type() - .unwrap_or_else(|_| panic!("failed to read type for {}", path.display())); + .with_context(|| format!("failed to read type for {}", path.display()))?; if file_type.is_dir() { - collect_asset_files(workspace_root, &path, asset_files); + collect_asset_files(workspace_root, &path, asset_files)?; continue; } @@ -187,16 +271,17 @@ fn collect_asset_files(workspace_root: &Path, dir: &Path, asset_files: &mut Vec< continue; } - let relative_path = path.strip_prefix(workspace_root).unwrap_or_else(|_| { - panic!( + let relative_path = path.strip_prefix(workspace_root).with_context(|| { + format!( "{} is not under {}", path.display(), workspace_root.display() ) - }); + })?; asset_files.push(relative_path.to_string_lossy().replace('\\', "/")); } + Ok(()) } struct BuildAttributes { @@ -206,41 +291,42 @@ struct BuildAttributes { } /// Captures build provenance values passed to rustc as compile-time env vars. -fn get_build_attributes() -> BuildAttributes { - BuildAttributes { - user: command_output("whoami"), - host: command_output("hostname"), - timestamp: build_timestamp(), - } +fn get_build_attributes() -> Result { + Ok(BuildAttributes { + user: command_output("whoami")?, + host: command_output("hostname")?, + timestamp: build_timestamp()?, + }) } /// Runs a command and returns its trimmed stdout, or `unknown` if it fails. -fn command_output(command: &str) -> String { - Command::new(command) +fn command_output(command: &str) -> Result { + let output = Command::new(command) .output() - .ok() - .and_then(|output| { - output.status.success().then(|| { - let value = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if value.is_empty() { - "unknown".to_string() - } else { - value - } - }) - }) - .unwrap_or_else(|| "unknown".to_string()) + .with_context(|| format!("failed to spawn `{command}`"))?; + + if !output.status.success() { + return Ok("unknown".to_string()); + } + + let value = String::from_utf8_lossy(&output.stdout).trim().to_string(); + Ok(if value.is_empty() { + "unknown".to_string() + } else { + value + }) } /// Returns the current Unix epoch timestamp for this build script run. -fn build_timestamp() -> String { +fn build_timestamp() -> Result { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|duration| duration.as_secs().to_string()) - .unwrap_or_else(|_| "unknown".to_string()) + .or_else(|_| Ok("unknown".to_string())) } -fn get_version_info() -> Result<(String, Option), VarError> { +/// Compute the Git version and PR info to embed in the build. +fn get_version_info() -> Result<(String, Option)> { let git_version = Command::new("git") .args(["describe", "--tags", "--always", "--dirty"]) .output() @@ -253,26 +339,23 @@ fn get_version_info() -> Result<(String, Option), VarError> { }) .unwrap_or_else(|| "unknown".to_string()); - let ci_var = env::var("CI").ok(); - match ci_var { - Some(_) => { - if !env::var("GITHUB_EVENT_NAME") - .unwrap_or_default() - .starts_with("pull_request") - { - return Ok((git_version, None)); - } - - let pr_number = env::var("PR_NUMBER").expect("PR_NUMBER not set in CI environment"); - let mut pr_head_sha = - env::var("PR_HEAD_SHA").expect("PR_HEAD_SHA not set in CI environment"); - pr_head_sha = pr_head_sha.get(..7).unwrap_or(&pr_head_sha).to_string(); + if env::var("CI").is_err() { + return Ok((git_version, None)); + } - Ok(( - git_version, - Some(format!("PR #{} ({})", pr_number, pr_head_sha)), - )) - } - _ => Ok((git_version, None)), + if !env::var("GITHUB_EVENT_NAME") + .unwrap_or_default() + .starts_with("pull_request") + { + return Ok((git_version, None)); } + + let pr_number = env::var("PR_NUMBER").context("PR_NUMBER not set in CI environment")?; + let pr_head_sha = env::var("PR_HEAD_SHA").context("PR_HEAD_SHA not set in CI environment")?; + let pr_head_short = pr_head_sha.get(..7).unwrap_or(&pr_head_sha).to_string(); + + Ok(( + git_version, + Some(format!("PR #{} ({})", pr_number, pr_head_short)), + )) } diff --git a/crates/emulator/Cargo.toml b/crates/emulator/Cargo.toml index 95691c72..3c1dd29f 100644 --- a/crates/emulator/Cargo.toml +++ b/crates/emulator/Cargo.toml @@ -21,3 +21,6 @@ tracing = ["cadmus-core/tracing"] profiling = ["cadmus-core/profiling", "tikv-jemallocator"] telemetry = ["cadmus-core/telemetry", "tracing", "profiling", "tikv-jemallocator"] test = ["cadmus-core/test"] + +[lints] +workspace = true diff --git a/crates/fetcher/Cargo.toml b/crates/fetcher/Cargo.toml index 1e6b4b51..1dd49d02 100644 --- a/crates/fetcher/Cargo.toml +++ b/crates/fetcher/Cargo.toml @@ -16,3 +16,6 @@ signal-hook = "0.4.1" version = "0.13.1" features = ["blocking", "json", "query", "rustls"] default-features = false + +[lints] +workspace = true diff --git a/crates/importer/Cargo.toml b/crates/importer/Cargo.toml index 8c2eaac3..aba2d9e8 100644 --- a/crates/importer/Cargo.toml +++ b/crates/importer/Cargo.toml @@ -11,3 +11,6 @@ path = "src/main.rs" [dependencies] cadmus-core = { path = "../core" } getopts = "0.2.24" + +[lints] +workspace = true diff --git a/renovate.json b/renovate.json index fbbde2ae..c358c0af 100644 --- a/renovate.json +++ b/renovate.json @@ -3,6 +3,9 @@ "minimumReleaseAge": "7 days", "labels": ["dependencies"], "reviewers": ["OGKevin"], + "git-submodules": { + "enabled": true + }, "packageRules": [ { "matchManagers": ["cargo"], @@ -29,7 +32,7 @@ "groupSlug": "js-tooling-non-major" }, { - "matchFileNames": ["xtask/src/tasks/util/thirdparty.rs"], + "matchManagers": ["git-submodules"], "addLabels": ["thirdparty"] } ], @@ -112,7 +115,7 @@ { "customType": "regex", "managerFilePatterns": ["/^\\.github\\/workflows\\/cargo\\.yml$/"], - "matchStrings": ["NEXTEST_VERSION=(?[^\\n]+)"], + "matchStrings": ["NEXTEST_VERSION:\\s+\\\"(?[^\\\"]+)\\\""], "datasourceTemplate": "github-releases", "depNameTemplate": "nextest-rs/nextest", "extractVersionTemplate": "^cargo-nextest-(?.+)$", @@ -140,111 +143,7 @@ }, { "customType": "regex", - "managerFilePatterns": [ - "/^xtask\\/src\\/tasks\\/util\\/thirdparty\\.rs$/" - ], - "matchStrings": [ - "ZLIB_VERSION: &str = \\\"(?[^\\\"]+)\\\"" - ], - "datasourceTemplate": "github-releases", - "depNameTemplate": "madler/zlib", - "versioningTemplate": "semver" - }, - { - "customType": "regex", - "managerFilePatterns": [ - "/^xtask\\/src\\/tasks\\/util\\/thirdparty\\.rs$/" - ], - "matchStrings": [ - "LIBPNG_VERSION: &str = \\\"(?[^\\\"]+)\\\"" - ], - "datasourceTemplate": "github-releases", - "depNameTemplate": "pnggroup/libpng", - "versioningTemplate": "semver" - }, - { - "customType": "regex", - "managerFilePatterns": [ - "/^xtask\\/src\\/tasks\\/util\\/thirdparty\\.rs$/" - ], - "matchStrings": [ - "DJVULIBRE_VERSION: &str = \\\"(?[^\\\"]+)\\\"" - ], - "datasourceTemplate": "github-tags", - "depNameTemplate": "barak/djvulibre", - "extractVersionTemplate": "^release\\.(?.+)$", - "versioningTemplate": "semver" - }, - { - "customType": "regex", - "managerFilePatterns": [ - "/^xtask\\/src\\/tasks\\/util\\/thirdparty\\.rs$/" - ], - "matchStrings": [ - "LIBJPEG_VERSION: &str = \\\"(?[^\\\"]+)\\\"" - ], - "datasourceTemplate": "github-tags", - "depNameTemplate": "libjpeg-turbo/libjpeg-turbo", - "extractVersionTemplate": "^jpeg-(?.+)$", - "versioningTemplate": "regex:^(?\\d+)(?[a-z]?)$" - }, - { - "customType": "regex", - "managerFilePatterns": [ - "/^xtask\\/src\\/tasks\\/util\\/thirdparty\\.rs$/" - ], - "matchStrings": [ - "FREETYPE2_VERSION: &str = \\\"(?[^\\\"]+)\\\"" - ], - "datasourceTemplate": "github-tags", - "depNameTemplate": "freetype/freetype", - "extractVersionTemplate": "^VER-(?\\d+)-(?\\d+)-(?\\d+)$", - "versioningTemplate": "semver" - }, - { - "customType": "regex", - "managerFilePatterns": [ - "/^xtask\\/src\\/tasks\\/util\\/thirdparty\\.rs$/" - ], - "matchStrings": [ - "OPENJPEG_VERSION: &str = \\\"(?[^\\\"]+)\\\"" - ], - "datasourceTemplate": "github-tags", - "depNameTemplate": "uclouvain/openjpeg", - "extractVersionTemplate": "^v(?.+)$", - "versioningTemplate": "semver" - }, - { - "customType": "regex", - "managerFilePatterns": [ - "/^xtask\\/src\\/tasks\\/util\\/thirdparty\\.rs$/" - ], - "matchStrings": [ - "HARFBUZZ_VERSION: &str = \\\"(?[^\\\"]+)\\\"" - ], - "datasourceTemplate": "github-tags", - "depNameTemplate": "harfbuzz/harfbuzz", - "extractVersionTemplate": "^(?.+)$", - "versioningTemplate": "semver" - }, - { - "customType": "regex", - "managerFilePatterns": [ - "/^xtask\\/src\\/tasks\\/util\\/thirdparty\\.rs$/" - ], - "matchStrings": [ - "GUMBO_VERSION: &str = \\\"(?[^\\\"]+)\\\"" - ], - "datasourceTemplate": "github-tags", - "depNameTemplate": "google/gumbo-parser", - "extractVersionTemplate": "^v(?.+)$", - "versioningTemplate": "semver" - }, - { - "customType": "regex", - "managerFilePatterns": [ - "/^xtask\\/src\\/tasks\\/util\\/thirdparty\\.rs$/" - ], + "managerFilePatterns": ["/^crates\\/build-deps\\/src\\/versions\\.rs$/"], "matchStrings": [ "MUPDF_VERSION: &str = \\\"(?[^\\\"]+)\\\"" ], @@ -263,45 +162,6 @@ "datasourceTemplate": "github-releases", "depNameTemplate": "ArtifexSoftware/mupdf-downloads", "versioningTemplate": "semver" - }, - { - "customType": "regex", - "managerFilePatterns": [ - "/^xtask\\/src\\/tasks\\/util\\/thirdparty\\.rs$/" - ], - "matchStrings": [ - "JBIG2DEC_VERSION: &str = \\\"(?[^\\\"]+)\\\"" - ], - "datasourceTemplate": "github-releases", - "depNameTemplate": "ArtifexSoftware/jbig2dec", - "versioningTemplate": "semver" - }, - { - "customType": "regex", - "managerFilePatterns": [ - "/^xtask\\/src\\/tasks\\/util\\/thirdparty\\.rs$/" - ], - "matchStrings": [ - "BZIP2_VERSION: &str = \\\"(?[^\\\"]+)\\\"" - ], - "datasourceTemplate": "gitlab-tags", - "depNameTemplate": "bzip2/bzip2", - "registryUrlTemplate": "https://gitlab.com", - "extractVersionTemplate": "^bzip2-(?.+)$", - "versioningTemplate": "semver" - }, - { - "customType": "regex", - "managerFilePatterns": [ - "/^xtask\\/src\\/tasks\\/util\\/thirdparty\\.rs$/" - ], - "matchStrings": [ - "LIBWEBP_VERSION: &str = \\\"(?[^\\\"]+)\\\"" - ], - "datasourceTemplate": "github-tags", - "depNameTemplate": "webmproject/libwebp", - "extractVersionTemplate": "^v(?.+)$", - "versioningTemplate": "semver" } ] } diff --git a/thirdparty/bzip2 b/thirdparty/bzip2 new file mode 160000 index 00000000..6a8690fc --- /dev/null +++ b/thirdparty/bzip2 @@ -0,0 +1 @@ +Subproject commit 6a8690fc8d26c815e798c588f796eabe9d684cf0 diff --git a/thirdparty/bzip2/Makefile-kobo b/thirdparty/bzip2/Makefile-kobo deleted file mode 100644 index d2b4c90b..00000000 --- a/thirdparty/bzip2/Makefile-kobo +++ /dev/null @@ -1,65 +0,0 @@ - -# This Makefile builds a shared version of the library, -# libbz2.so.1.0.6, with soname libbz2.so.1.0, -# at least on x86-Linux (RedHat 7.2), -# with gcc-2.96 20000731 (Red Hat Linux 7.1 2.96-98). -# Please see the README file for some important info -# about building the library like this. - -# ------------------------------------------------------------------ -# This file is part of bzip2/libbzip2, a program and library for -# lossless, block-sorting data compression. -# -# bzip2/libbzip2 version 1.0.6 of 6 September 2010 -# Copyright (C) 1996-2010 Julian Seward -# -# Please read the WARNING, DISCLAIMER and PATENTS sections in the -# README file. -# -# This program is released under the terms of the license contained -# in the file LICENSE. -# ------------------------------------------------------------------ - - - -SHELL=/bin/sh -TRIPLE=arm-linux-gnueabihf -CC=$(TRIPLE)-gcc -CXX=$(TRIPLE)-g++ -AR=$(TRIPLE)-ar -AS=$(TRIPLE)-as -BIGFILES=-D_FILE_OFFSET_BITS=64 -CFLAGS=-fpic -fPIC -Wall -Winline -O2 -mcpu=cortex-a9 -mfpu=neon -g $(BIGFILES) - -OBJS= blocksort.o \ - huffman.o \ - crctable.o \ - randtable.o \ - compress.o \ - decompress.o \ - bzlib.o - -all: $(OBJS) - $(CC) -shared -Wl,-soname -Wl,libbz2.so.1.0 -o libbz2.so.1.0.6 $(OBJS) - $(CC) $(CFLAGS) -o bzip2-shared bzip2.c libbz2.so.1.0.6 - rm -f libbz2.so.1.0 libbz2.so - ln -s libbz2.so.1.0.6 libbz2.so.1.0 - ln -s libbz2.so.1.0.6 libbz2.so - -clean: - rm -f $(OBJS) bzip2.o libbz2.so.1.0.6 libbz2.so.1.0 bzip2-shared - -blocksort.o: blocksort.c - $(CC) $(CFLAGS) -c blocksort.c -huffman.o: huffman.c - $(CC) $(CFLAGS) -c huffman.c -crctable.o: crctable.c - $(CC) $(CFLAGS) -c crctable.c -randtable.o: randtable.c - $(CC) $(CFLAGS) -c randtable.c -compress.o: compress.c - $(CC) $(CFLAGS) -c compress.c -decompress.o: decompress.c - $(CC) $(CFLAGS) -c decompress.c -bzlib.o: bzlib.c - $(CC) $(CFLAGS) -c bzlib.c diff --git a/thirdparty/bzip2/build-kobo.sh b/thirdparty/bzip2/build-kobo.sh deleted file mode 100755 index 3ff7b310..00000000 --- a/thirdparty/bzip2/build-kobo.sh +++ /dev/null @@ -1,3 +0,0 @@ -#! /bin/sh - -make -f Makefile-kobo diff --git a/thirdparty/djvulibre b/thirdparty/djvulibre new file mode 160000 index 00000000..03e6abd7 --- /dev/null +++ b/thirdparty/djvulibre @@ -0,0 +1 @@ +Subproject commit 03e6abd7295c74f795ed70a7118294b25351e0f4 diff --git a/thirdparty/djvulibre/build-kobo.sh b/thirdparty/djvulibre/build-kobo.sh deleted file mode 100755 index 2d891f00..00000000 --- a/thirdparty/djvulibre/build-kobo.sh +++ /dev/null @@ -1,11 +0,0 @@ -#! /bin/sh - -TRIPLE=arm-linux-gnueabihf -JPEG_DIR=$(realpath ../libjpeg) -export CC=${TRIPLE}-gcc -export CFLAGS="-O2 -mcpu=cortex-a9 -mfpu=neon" -export CXXFLAGS="$CFLAGS" -export CXX=${TRIPLE}-g++ -export AS=${TRIPLE}-as - -./autogen.sh --host=${TRIPLE} --disable-xmltools --disable-desktopfiles --with-jpeg="${JPEG_DIR}" && make diff --git a/thirdparty/freetype2 b/thirdparty/freetype2 new file mode 160000 index 00000000..526ec5c4 --- /dev/null +++ b/thirdparty/freetype2 @@ -0,0 +1 @@ +Subproject commit 526ec5c47b9ebccc4754c85ac0c0cdf7c85a5e9b diff --git a/thirdparty/freetype2/build-kobo.sh b/thirdparty/freetype2/build-kobo.sh deleted file mode 100755 index 1566c940..00000000 --- a/thirdparty/freetype2/build-kobo.sh +++ /dev/null @@ -1,17 +0,0 @@ -#! /bin/sh - -export TRIPLE=arm-linux-gnueabihf -export CC=${TRIPLE}-gcc -export CXX=${TRIPLE}-g++ -export CFLAGS="-O2 -mcpu=cortex-a9 -mfpu=neon" -export CXXFLAGS="$CFLAGS" -export ZLIB_CFLAGS="-I../zlib" -export ZLIB_LIBS="-L../zlib -lz" -export BZIP2_CFLAGS="-I../bzip2" -export BZIP2_LIBS="-L../bzip2 -lbz2" -export LIBPNG_CFLAGS="-I../libpng" -export LIBPNG_LIBS="-L../libpng/.libs -lpng16" - -./configure --host=${TRIPLE} --with-zlib=yes --with-png=yes \ - --with-bzip2=yes --with-harfbuzz=no --with-brotli=no \ - --disable-static && make diff --git a/thirdparty/gumbo b/thirdparty/gumbo new file mode 160000 index 00000000..3973c58d --- /dev/null +++ b/thirdparty/gumbo @@ -0,0 +1 @@ +Subproject commit 3973c58d759574f2899528d2b3379e17d66dbcad diff --git a/thirdparty/gumbo/build-kobo.sh b/thirdparty/gumbo/build-kobo.sh deleted file mode 100755 index 9ea13418..00000000 --- a/thirdparty/gumbo/build-kobo.sh +++ /dev/null @@ -1,8 +0,0 @@ -#! /bin/sh - -export TRIPLE=arm-linux-gnueabihf -export CC=${TRIPLE}-gcc -export CXX=${TRIPLE}-g++ - -[ -x configure ] || ./autogen.sh -./configure --host="$TRIPLE" && make diff --git a/thirdparty/harfbuzz b/thirdparty/harfbuzz new file mode 160000 index 00000000..b0ffab42 --- /dev/null +++ b/thirdparty/harfbuzz @@ -0,0 +1 @@ +Subproject commit b0ffab42d473eb380ad0fcf42730e0f1868cbc97 diff --git a/thirdparty/harfbuzz/build-kobo.sh b/thirdparty/harfbuzz/build-kobo.sh deleted file mode 100755 index b5952c63..00000000 --- a/thirdparty/harfbuzz/build-kobo.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh - -meson setup -Dglib=disabled -Dicu=disabled -Dcairo=disabled -Dfreetype=enabled --cross-file kobo-options.txt build -meson compile -C build diff --git a/thirdparty/jbig2dec b/thirdparty/jbig2dec new file mode 160000 index 00000000..1d1347e3 --- /dev/null +++ b/thirdparty/jbig2dec @@ -0,0 +1 @@ +Subproject commit 1d1347e38a55e657dcc4c8f1c77bb3a26bfc9ff3 diff --git a/thirdparty/jbig2dec/build-kobo.sh b/thirdparty/jbig2dec/build-kobo.sh deleted file mode 100755 index 5440e4b8..00000000 --- a/thirdparty/jbig2dec/build-kobo.sh +++ /dev/null @@ -1,10 +0,0 @@ -#! /bin/sh - -TRIPLE=arm-linux-gnueabihf -export CC=${TRIPLE}-gcc -export CXX=${TRIPLE}-g++ -export CFLAGS='-O2 -mcpu=cortex-a9 -mfpu=neon' -export CXXFLAGS="$CFLAGS" -export AS=${TRIPLE}-as - -./autogen.sh --host=${TRIPLE} && make diff --git a/thirdparty/libjpeg b/thirdparty/libjpeg new file mode 160000 index 00000000..bf8b31c1 --- /dev/null +++ b/thirdparty/libjpeg @@ -0,0 +1 @@ +Subproject commit bf8b31c1dcec2eac7583b86b3a4695d8efc5a7de diff --git a/thirdparty/libjpeg/build-kobo.sh b/thirdparty/libjpeg/build-kobo.sh deleted file mode 100755 index 4774a718..00000000 --- a/thirdparty/libjpeg/build-kobo.sh +++ /dev/null @@ -1,11 +0,0 @@ -#! /bin/sh - -TRIPLE=arm-linux-gnueabihf -export CC=${TRIPLE}-gcc -export CXX=${TRIPLE}-g++ -export CFLAGS="-O2 -mcpu=cortex-a9 -mfpu=neon" - -./configure --host=${TRIPLE} && make - -[ -L include ] || ln -s . include -[ -L lib ] || ln -s .libs lib diff --git a/thirdparty/libpng b/thirdparty/libpng new file mode 160000 index 00000000..4e3f57d5 --- /dev/null +++ b/thirdparty/libpng @@ -0,0 +1 @@ +Subproject commit 4e3f57d50f552841550a36eabbb3fbcecacb7750 diff --git a/thirdparty/libpng/build-kobo.sh b/thirdparty/libpng/build-kobo.sh deleted file mode 100755 index e7249508..00000000 --- a/thirdparty/libpng/build-kobo.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /bin/sh - -TRIPLE=arm-linux-gnueabihf -ZLIB_DIR=../zlib -export CC=${TRIPLE}-gcc -export CXX=${TRIPLE}-g++ -export CFLAGS="-O2 -mcpu=cortex-a9 -mfpu=neon" -export CXXFLAGS="$CFLAGS" -export CPPFLAGS="-I${ZLIB_DIR}" -export LDFLAGS="-L${ZLIB_DIR}" - -./configure --host=${TRIPLE} && make diff --git a/thirdparty/libwebp b/thirdparty/libwebp new file mode 160000 index 00000000..3c4a0fbf --- /dev/null +++ b/thirdparty/libwebp @@ -0,0 +1 @@ +Subproject commit 3c4a0fbfbcc606193f7e943b7e50af4077ce1a6c diff --git a/thirdparty/libwebp/build-kobo.sh b/thirdparty/libwebp/build-kobo.sh deleted file mode 100755 index ef1ee6a9..00000000 --- a/thirdparty/libwebp/build-kobo.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -TRIPLE=arm-linux-gnueabihf -export CC=${TRIPLE}-gcc -export CXX=${TRIPLE}-g++ -export CFLAGS="-O2 -mcpu=cortex-a9 -mfpu=neon" -export CXXFLAGS="$CFLAGS" - -if [ ! -f configure ]; then - NOCONFIGURE=1 sh autogen.sh -fi - -./configure \ - --host=${TRIPLE} \ - --enable-shared \ - --disable-static \ - --disable-libwebpmux \ - --enable-libwebpdecoder \ - --enable-libwebpdemux \ - --disable-webp-tools && - make diff --git a/thirdparty/mupdf b/thirdparty/mupdf new file mode 160000 index 00000000..73d3100d --- /dev/null +++ b/thirdparty/mupdf @@ -0,0 +1 @@ +Subproject commit 73d3100d46d8a9ad634f6ef035bbe78f0f947886 diff --git a/thirdparty/mupdf/build-kobo.sh b/thirdparty/mupdf/build-kobo.sh deleted file mode 100755 index 10d68d7f..00000000 --- a/thirdparty/mupdf/build-kobo.sh +++ /dev/null @@ -1,11 +0,0 @@ -#! /bin/bash - -[ -e .gitattributes ] && rm -rf .git* - -BUILD_KIND=${1:-release} -rm -rf build -make verbose=yes generate -make verbose=yes mujs=no tesseract=no extract=no archive=no brotli=no barcode=no commercial=no USE_SYSTEM_LIBS=yes OS=kobo build="$BUILD_KIND" XCFLAGS="-I../libwebp/src -DHAVE_WEBP=1" libs - -mapfile -t obj_files < <(find build/"$BUILD_KIND" -name '*.o' | grep -Ev '(SourceHanSerif-Regular|DroidSansFallbackFull|NotoSerifTangut|color-lcms)') -arm-linux-gnueabihf-gcc -Wl,--gc-sections -o build/"$BUILD_KIND"/libmupdf.so "${obj_files[@]}" -lm -L../freetype2/objs/.libs -lfreetype -L../harfbuzz/build/src -lharfbuzz -L../gumbo/.libs -lgumbo -L../jbig2dec/.libs -ljbig2dec -L../libjpeg/.libs -ljpeg -L../openjpeg/build/bin -lopenjp2 -L../zlib -lz -L../libwebp/src/.libs -lwebp -L../libwebp/src/demux/.libs -lwebpdemux -shared -Wl,-soname -Wl,libmupdf.so -Wl,--no-undefined diff --git a/thirdparty/openjpeg b/thirdparty/openjpeg new file mode 160000 index 00000000..6c4a29b0 --- /dev/null +++ b/thirdparty/openjpeg @@ -0,0 +1 @@ +Subproject commit 6c4a29b00211eb0430fa0e5e890f1ce5c80f409f diff --git a/thirdparty/openjpeg/build-kobo.sh b/thirdparty/openjpeg/build-kobo.sh deleted file mode 100755 index 4ace61d1..00000000 --- a/thirdparty/openjpeg/build-kobo.sh +++ /dev/null @@ -1,15 +0,0 @@ -#! /bin/sh - -[ -d build ] && rm -Rf build - -mkdir build -cd build || exit 1 - -TRIPLE=arm-linux-gnueabihf -export CFLAGS="-O2 -mcpu=cortex-a9 -mfpu=neon" -export CXXFLAGS="$CFLAGS" - -cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_CODEC=off -DBUILD_STATIC_LIBS=off -DCMAKE_SYSTEM_NAME=Linux -DCMAKE_C_COMPILER=${TRIPLE}-gcc -DCMAKE_CXX_COMPILER=${TRIPLE}-g++ -DCMAKE_AR=${TRIPLE}-ar .. && make - -cd .. || exit 1 -cp build/src/lib/openjp2/opj_config.h src/lib/openjp2 diff --git a/thirdparty/zlib b/thirdparty/zlib new file mode 160000 index 00000000..da607da7 --- /dev/null +++ b/thirdparty/zlib @@ -0,0 +1 @@ +Subproject commit da607da739fa6047df13e66a2af6b8bec7c2a498 diff --git a/thirdparty/zlib/build-kobo.sh b/thirdparty/zlib/build-kobo.sh deleted file mode 100755 index c4f93979..00000000 --- a/thirdparty/zlib/build-kobo.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/sh - -export CHOST=arm-linux-gnueabihf -export CC=arm-linux-gnueabihf-gcc -export CXX=arm-linux-gnueabihf-g++ -export CFLAGS="-O2 -mcpu=cortex-a9 -mfpu=neon" -export CXXFLAGS="$CFLAGS" - -./configure && make diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 2947196f..51d04e5a 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -14,6 +14,7 @@ path = "src/main.rs" [dependencies] anyhow = "1" +build-deps = { path = "../crates/build-deps" } clap = { version = "4", features = ["derive"] } flate2 = "1" reqwest = { version = "0.13", default-features = false, features = [ @@ -33,5 +34,5 @@ wildmatch = "2" [dev-dependencies] tempfile = "3" -[lints.rustdoc] -private_intra_doc_links = "allow" +[lints] +workspace = true diff --git a/xtask/src/lib.rs b/xtask/src/lib.rs index 2a0631d9..d31a3cd8 100644 --- a/xtask/src/lib.rs +++ b/xtask/src/lib.rs @@ -22,8 +22,7 @@ //! | [`test`](tasks::test) | Run `cargo test` across the feature matrix | //! | [`bench`](tasks::bench) | Run benchmarks with the `bench` feature enabled | //! | [`build-kobo`](tasks::build_kobo) | Cross-compile for Kobo (ARM, Linux & macOS) | -//! | [`setup-native`](tasks::setup_native) | Build MuPDF and the C wrapper for native dev | -//! | [`run-emulator`](tasks::run_emulator) | Run the Cadmus emulator (ensures prereqs are built) | +//! | [`run-emulator`](tasks::run_emulator) | Run the Cadmus emulator | //! | [`install-importer`](tasks::install_importer) | Install the Cadmus importer crate | //! | [`docs`](tasks::docs) | Build the full documentation portal | //! | [`download-assets`](tasks::download_assets) | Download static asset dirs from the latest release | @@ -47,8 +46,7 @@ pub use clap::Parser; pub use tasks::{ bench::BenchArgs, build_kobo::BuildKoboArgs, bundle::BundleArgs, ci::CiArgs, clippy::ClippyArgs, dist::DistArgs, docs::DocsArgs, fmt::FmtArgs, - install_importer::InstallImporterArgs, run_emulator::RunEmulatorArgs, - setup_native::SetupNativeArgs, test::TestArgs, + install_importer::InstallImporterArgs, run_emulator::RunEmulatorArgs, test::TestArgs, }; /// Cadmus build automation. @@ -73,11 +71,9 @@ pub enum Command { Bench(BenchArgs), /// Cross-compile Cadmus for Kobo devices (Linux & macOS). BuildKobo(BuildKoboArgs), - /// Build MuPDF and the C wrapper for native development. - SetupNative(SetupNativeArgs), - /// Run the Cadmus emulator (ensures MuPDF and wrapper are built first). + /// Run the Cadmus emulator. RunEmulator(RunEmulatorArgs), - /// Install the Cadmus importer crate (ensures MuPDF and wrapper are built first). + /// Install the Cadmus importer crate. InstallImporter(InstallImporterArgs), /// Build the full documentation portal (mdBook + cargo doc + Zola). Docs(DocsArgs), @@ -104,7 +100,6 @@ pub fn run() -> Result<()> { Command::Test(args) => tasks::test::run(args), Command::Bench(args) => tasks::bench::run(args), Command::BuildKobo(args) => tasks::build_kobo::run(args), - Command::SetupNative(args) => tasks::setup_native::run(args), Command::RunEmulator(args) => tasks::run_emulator::run(args), Command::InstallImporter(args) => tasks::install_importer::run(args), Command::Docs(args) => tasks::docs::run(args), diff --git a/xtask/src/tasks/build_kobo.rs b/xtask/src/tasks/build_kobo.rs index 95b367c1..9b668a25 100644 --- a/xtask/src/tasks/build_kobo.rs +++ b/xtask/src/tasks/build_kobo.rs @@ -1,97 +1,33 @@ //! `cargo xtask build-kobo` — cross-compile Cadmus for Kobo devices. //! -//! 1. Optionally downloads and builds all thirdparty libraries from source -//! (`--slow` mode, required for CI). -//! 2. Builds the `mupdf_wrapper` C library for the Kobo ARM target. -//! 3. Runs `cargo build --release --target arm-unknown-linux-gnueabihf`. +//! This task is a thin wrapper around `cargo build --release +//! --target arm-unknown-linux-gnueabihf -p cadmus`. All dependency +//! building (thirdparty libs, MuPDF, libwebp, mupdf_wrapper) is +//! handled automatically by `build.rs` when cargo build runs. //! -//! ## Platform requirement +//! Pre-flight steps performed before invoking cargo: //! -//! Cross-compilation requires the Linaro ARM toolchain -//! (`arm-linux-gnueabihf-gcc` and friends). -//! The task checks for the toolchain at runtime and exits with a clear error if -//! it is not available. +//! 1. Verify the Linaro ARM toolchain (`arm-linux-gnueabihf-gcc`) +//! is on `PATH`. //! -//! ## Build modes +//! Git submodules are not initialised up-front here: the Rust build +//! script clones them lazily, only when the cached Kobo build +//! artefacts in `libs/` and `target/cadmus-build-deps/...` are +//! missing. This keeps warm-cache CI runs fast by avoiding the +//! recursive submodule clone done by `actions/checkout`. //! -//! | Mode | Description | -//! |------|-------------| -//! | fast (default) | Downloads pre-built `.so` files and MuPDF sources | -//! | slow | Builds all thirdparty libraries from source | -//! | slow + download-only | Downloads all thirdparty sources without building | -//! | skip | Assumes `libs/` already exists; skips download entirely | +//! The Kobo build is only available on Linux and macOS hosts. -use anyhow::{Context, Result, bail}; +use anyhow::{Result, bail}; use clap::Args; -use super::util::{cmd, fs, github, http, mupdf_wrapper, thirdparty, workspace}; +use build_deps::versions::CROSS_ENV; -/// BUILT_LIBRARY_COPIES maps built `.so` paths to their destination names in -/// `libs/`. The destination names here are the *base* names (e.g. `libz.so`); -/// the actual SONAME is discovered at runtime via [`thirdparty::soname`]. -const BUILT_LIBRARY_COPIES: &[(&str, &str)] = &[ - ("thirdparty/zlib/libz.so", "libz.so"), - ("thirdparty/bzip2/libbz2.so", "libbz2.so"), - ("thirdparty/libpng/.libs/libpng16.so", "libpng16.so"), - ("thirdparty/libjpeg/.libs/libjpeg.so", "libjpeg.so"), - ( - "thirdparty/openjpeg/build/bin/libopenjp2.so", - "libopenjp2.so", - ), - ("thirdparty/jbig2dec/.libs/libjbig2dec.so", "libjbig2dec.so"), - ("thirdparty/libwebp/src/.libs/libwebp.so", "libwebp.so"), - ( - "thirdparty/libwebp/src/demux/.libs/libwebpdemux.so", - "libwebpdemux.so", - ), - ( - "thirdparty/freetype2/objs/.libs/libfreetype.so", - "libfreetype.so", - ), - ( - "thirdparty/harfbuzz/build/src/libharfbuzz.so", - "libharfbuzz.so", - ), - ("thirdparty/gumbo/.libs/libgumbo.so", "libgumbo.so"), - ( - "thirdparty/djvulibre/libdjvu/.libs/libdjvulibre.so", - "libdjvulibre.so", - ), - ("thirdparty/mupdf/build/release/libmupdf.so", "libmupdf.so"), -]; - -const CROSS_ENV: &[(&str, &str)] = &[ - ("PKG_CONFIG_ALLOW_CROSS", "1"), - ( - "CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER", - "arm-linux-gnueabihf-gcc", - ), - ("CC_arm_unknown_linux_gnueabihf", "arm-linux-gnueabihf-gcc"), - ("AR_arm_unknown_linux_gnueabihf", "arm-linux-gnueabihf-ar"), -]; +use super::util::{cmd, workspace}; /// Arguments for `cargo xtask build-kobo`. #[derive(Debug, Args)] pub struct BuildKoboArgs { - /// Build all thirdparty libraries from source instead of downloading - /// pre-built binaries. - /// - /// Required for CI where pre-built binaries are not available. - #[arg(long)] - pub slow: bool, - - /// Skip the library download/build step entirely. - /// - /// Use this when `libs/` already contains the required `.so` files. - #[arg(long)] - pub skip: bool, - - /// Download thirdparty sources without building or cross-compiling. - /// - /// Useful for pre-populating the source cache in CI setup steps. - #[arg(long)] - pub download_only: bool, - /// Cargo feature flags to pass to the Cadmus build (e.g. `test`). #[arg(long)] pub features: Option, @@ -102,9 +38,13 @@ pub struct BuildKoboArgs { /// # Errors /// /// Returns an error if: -/// - The platform is not Linux or macOS. -/// - The Linaro toolchain is not on `PATH`. -/// - Any build step fails. +/// - The host platform is not Linux or macOS. +/// - The Linaro ARM toolchain is not on `PATH`. +/// - The underlying `cargo build` invocation fails. +/// +/// Git submodules are initialised lazily by the Rust build script +/// when the cached Kobo artefacts are missing; this task no longer +/// triggers a recursive submodule clone unconditionally. pub fn run(args: BuildKoboArgs) -> Result<()> { if !cfg!(any(target_os = "linux", target_os = "macos")) { bail!( @@ -117,37 +57,11 @@ pub fn run(args: BuildKoboArgs) -> Result<()> { ensure_linaro_toolchain()?; - match (args.slow, args.skip, args.download_only) { - (_, true, _) => { - println!("Skipping library download (--skip)."); - } - (true, false, true) => { - println!("Downloading thirdparty sources (--slow --download-only)…"); - thirdparty::download_libraries(&root.join("thirdparty"), &[])?; - return Ok(()); - } - (true, false, false) => { - println!("Building thirdparty libraries from source (--slow)…"); - build_thirdparty_slow(&root)?; - } - (false, false, true) => { - println!("Downloading MuPDF sources (--download-only)…"); - thirdparty::download_libraries(&root.join("thirdparty"), &["mupdf"])?; - return Ok(()); - } - (false, false, false) => { - println!("Downloading pre-built libraries (fast mode)…"); - build_thirdparty_fast(&root)?; - } - } - - build_mupdf_wrapper_kobo(&root)?; cargo_build_kobo(&root, args.features.as_deref())?; Ok(()) } -/// Verifies that the Linaro ARM cross-compiler is available on `PATH`. fn ensure_linaro_toolchain() -> Result<()> { cmd::run( "arm-linux-gnueabihf-gcc", @@ -163,98 +77,6 @@ fn ensure_linaro_toolchain() -> Result<()> { }) } -/// Downloads pre-built `.so` files and MuPDF sources (fast mode). -fn build_thirdparty_fast(root: &std::path::Path) -> Result<()> { - download_release_libs(root)?; - - let libs_dir = root.join("libs"); - create_symlinks(&libs_dir)?; - - thirdparty::download_libraries(&root.join("thirdparty"), &["mupdf"]) -} - -/// Builds all thirdparty libraries from source (slow mode). -fn build_thirdparty_slow(root: &std::path::Path) -> Result<()> { - let thirdparty_dir = root.join("thirdparty"); - - thirdparty::download_libraries(&thirdparty_dir, &[])?; - thirdparty::build_libraries(&thirdparty_dir, &[])?; - - let libs_dir = root.join("libs"); - std::fs::create_dir_all(&libs_dir)?; - - copy_built_libs(root, &libs_dir)?; - create_symlinks(&libs_dir) -} - -/// Creates the `.so` version symlinks expected by the Cadmus runtime. -fn create_symlinks(libs_dir: &std::path::Path) -> Result<()> { - for &lib in thirdparty::SONAMES { - let target = thirdparty::soname(libs_dir, lib)?; - let link_path = libs_dir.join(lib); - if !link_path.exists() { - #[cfg(unix)] - std::os::unix::fs::symlink(&target, &link_path)?; - } - } - - Ok(()) -} - -/// Copies the `.so` files produced by the slow build into `libs/`. -fn copy_built_libs(root: &std::path::Path, libs_dir: &std::path::Path) -> Result<()> { - for (src_rel, dest_name) in BUILT_LIBRARY_COPIES { - let src = root.join(src_rel); - let dest = libs_dir.join(dest_name); - std::fs::copy(&src, &dest).map_err(|e| { - anyhow::anyhow!("failed to copy {} → {}: {e}", src.display(), dest.display()) - })?; - } - - Ok(()) -} - -/// Downloads pre-built `.so` release assets from the cadmus GitHub release -/// with checksum verification. -fn download_release_libs(root: &std::path::Path) -> Result<()> { - let version = workspace::current_version()?; - let tag = format!("v{version}"); - let archive_name = "cadmus-kobo.tar.gz"; - - let libs_dir = root.join("libs"); - if libs_dir.exists() { - println!("libs/ directory already exists; skipping download of pre-built libraries."); - return Ok(()); - } - - std::fs::create_dir_all(&libs_dir)?; - - let asset = github::fetch_release_asset("ogkevin/cadmus", &tag, archive_name)?; - let archive = root.join(archive_name); - - match asset.sha256() { - Some(expected) => { - http::download_verified(&asset.browser_download_url, &archive, expected)?; - } - None => { - http::download(&asset.browser_download_url, &archive) - .with_context(|| format!("failed to download {archive_name}"))?; - } - } - - fs::extract_tarball_paths(&archive, root, &["libs"])?; - std::fs::remove_file(&archive).ok(); - - Ok(()) -} - -/// Builds the `mupdf_wrapper` C library for the Kobo ARM target. -fn build_mupdf_wrapper_kobo(root: &std::path::Path) -> Result<()> { - println!("Building mupdf_wrapper for Kobo…"); - mupdf_wrapper::build_kobo(root) -} - -/// Runs `cargo build --release` for the ARM Kobo target. fn cargo_build_kobo(root: &std::path::Path, features: Option<&str>) -> Result<()> { let mut cargo_args = vec![ "build", @@ -275,11 +97,9 @@ fn cargo_build_kobo(root: &std::path::Path, features: Option<&str>) -> Result<() #[cfg(test)] mod tests { - use super::*; - #[test] fn symlink_list_has_no_duplicates() { - let mut link_names: Vec<&str> = thirdparty::SONAMES.to_vec(); + let mut link_names: Vec<&str> = build_deps::versions::SONAMES.to_vec(); link_names.sort_unstable(); let original_len = link_names.len(); link_names.dedup(); diff --git a/xtask/src/tasks/dist.rs b/xtask/src/tasks/dist.rs index 5cfa3971..3ec07be5 100644 --- a/xtask/src/tasks/dist.rs +++ b/xtask/src/tasks/dist.rs @@ -35,7 +35,9 @@ use anyhow::{Context, Result, bail}; use clap::Args; use wildmatch::WildMatch; -use super::util::{cmd, fs, thirdparty, workspace}; +use super::util::{cmd, fs, workspace}; +use build_deps::build::kobo; +use build_deps::versions; /// Arguments for `cargo xtask dist`. #[derive(Debug, Args)] @@ -91,8 +93,8 @@ fn copy_libraries(root: &Path, dist_dir: &Path) -> Result<()> { let libs_dir = root.join("libs"); let dist_libs = dist_dir.join("libs"); - for &lib in thirdparty::SONAMES { - let soname = thirdparty::soname(&libs_dir, lib)?; + for &lib in versions::SONAMES { + let soname = kobo::soname(&libs_dir, lib)?; let src = libs_dir.join(lib); let dest = dist_libs.join(&soname); std::fs::copy(&src, &dest).with_context(|| { diff --git a/xtask/src/tasks/docs.rs b/xtask/src/tasks/docs.rs index 70723dc8..de3e0e50 100644 --- a/xtask/src/tasks/docs.rs +++ b/xtask/src/tasks/docs.rs @@ -107,7 +107,7 @@ fn build_cargo_doc(root: &Path) -> Result<()> { "cargo", &["doc", "--no-deps", "--document-private-items"], root, - &[], + &[("CADMUS_SKIP_THIRDPARTY_DEPS", "1")], ) } diff --git a/xtask/src/tasks/install_importer.rs b/xtask/src/tasks/install_importer.rs index c90d2292..2e52ec1d 100644 --- a/xtask/src/tasks/install_importer.rs +++ b/xtask/src/tasks/install_importer.rs @@ -1,13 +1,12 @@ //! `cargo xtask install-importer` — install the Cadmus importer crate. //! -//! Ensures MuPDF sources and the `mupdf_wrapper` C library are built for the -//! native platform, then runs `cargo install --path crates/importer`. Any -//! extra arguments are forwarded to `cargo install`. +//! Runs `cargo install --path crates/importer`. Any extra arguments are +//! forwarded to `cargo install`. Native dependencies are built automatically +//! by `build.rs`. use anyhow::Result; use clap::Args; -use super::setup_native; use super::util::{cmd, workspace}; /// Arguments for `cargo xtask install-importer`. @@ -18,17 +17,10 @@ pub struct InstallImporterArgs { pub extra: Vec, } -/// Ensures prerequisites are built then installs the importer. -/// -/// # Errors -/// -/// Returns an error if the MuPDF download, wrapper build, or installation -/// fails. +/// Installs the importer crate. pub fn run(args: InstallImporterArgs) -> Result<()> { let root = workspace::root()?; - setup_native::ensure_native_artifacts(&root, false)?; - let importer_path = root.join("crates/importer"); let importer_str = importer_path.to_string_lossy().into_owned(); diff --git a/xtask/src/tasks/mod.rs b/xtask/src/tasks/mod.rs index c0bde381..0ae66c64 100644 --- a/xtask/src/tasks/mod.rs +++ b/xtask/src/tasks/mod.rs @@ -14,6 +14,5 @@ pub mod download_assets; pub mod fmt; pub mod install_importer; pub mod run_emulator; -pub mod setup_native; pub mod test; pub mod util; diff --git a/xtask/src/tasks/run_emulator.rs b/xtask/src/tasks/run_emulator.rs index 49f7d99b..15c18b51 100644 --- a/xtask/src/tasks/run_emulator.rs +++ b/xtask/src/tasks/run_emulator.rs @@ -1,8 +1,8 @@ //! `cargo xtask run-emulator` — run the Cadmus emulator. //! -//! Ensures the native MuPDF build and the embedded documentation EPUB are -//! ready, then launches `cargo run -p emulator`. Any extra arguments are -//! forwarded to the emulator. +//! Ensures the embedded documentation EPUB is ready, then launches +//! `cargo run -p emulator`. Any extra arguments are forwarded to the cargo invocation. +//! Native dependencies (MuPDF, libwebp) are built automatically by `build.rs`. use std::path::Path; @@ -10,7 +10,6 @@ use anyhow::Result; use clap::Args; use super::docs::{self, DocsArgs}; -use super::setup_native::{self, SetupNativeArgs}; use super::util::{cmd, workspace}; /// Arguments for `cargo xtask run-emulator`. @@ -31,20 +30,10 @@ fn mdbook_epub_built(root: &Path) -> bool { .exists() } -/// Ensures prerequisites are built then launches the emulator. -/// -/// # Errors -/// -/// Returns an error if the native setup, documentation build, or emulator -/// launch fails. +/// Ensures documentation is built then launches the emulator. pub fn run(args: RunEmulatorArgs) -> Result<()> { let root = workspace::root()?; - if !setup_native::native_setup_done(&root) { - println!("Native setup not found — running setup-native…"); - setup_native::run(SetupNativeArgs { force: false })?; - } - if !mdbook_epub_built(&root) { println!("Documentation EPUB not found — building mdBook…"); docs::run(DocsArgs { diff --git a/xtask/src/tasks/setup_native.rs b/xtask/src/tasks/setup_native.rs deleted file mode 100644 index 2dedeb31..00000000 --- a/xtask/src/tasks/setup_native.rs +++ /dev/null @@ -1,469 +0,0 @@ -//! `cargo xtask setup-native` — build MuPDF and the C wrapper for native dev. -//! -//! 1. Downloads MuPDF sources if the required version is not already present. -//! 2. Builds the `mupdf_wrapper` C library. -//! 3. Compiles MuPDF using system libraries. -//! 4. Creates symlinks in `target/mupdf_wrapper//` so the Rust -//! build script can find the static libraries. -//! -//! ## Required MuPDF version -//! -//! The version is pinned to [`thirdparty::MUPDF_VERSION`]. If the sources -//! already present on disk match this version the download is skipped. - -use std::path::Path; - -use anyhow::{Context, Result}; -use clap::Args; - -use super::util::thirdparty::MUPDF_VERSION; -use super::util::{cmd, mupdf_wrapper, thirdparty, workspace}; - -/// Marker file written after a successful native MuPDF build. -const NATIVE_BUILT_MARKER: &str = ".built-native"; - -/// Arguments for `cargo xtask setup-native`. -#[derive(Debug, Args)] -pub struct SetupNativeArgs { - /// Force a re-download of MuPDF sources even if the correct version is - /// already present. - #[arg(long)] - pub force: bool, -} - -/// Builds MuPDF and the C wrapper for native (non-cross-compiled) development. -/// -/// # Errors -/// -/// Returns an error if any build step fails or if required tools (`make`, -/// `pkg-config`, `ar`) are not available. -pub fn run(args: SetupNativeArgs) -> Result<()> { - let root = workspace::root()?; - - ensure_native_artifacts(&root, args.force)?; - - println!("\nNative setup complete!"); - println!("You can now run:"); - println!(" cargo test - Run tests"); - println!(" cargo xtask build-kobo - Build for Kobo (Linux & macOS)"); - - Ok(()) -} - -pub fn ensure_native_artifacts(root: &Path, force: bool) -> Result<()> { - thirdparty::download_libraries(&root.join("thirdparty"), &["libwebp"])?; - build_libwebp_native(root)?; - - let mupdf_patched = ensure_mupdf_sources_with_webp_patches(root, force)?; - if mupdf_patched { - remove_native_wrapper_artifact(root)?; - } - - let rebuild_mupdf = mupdf_patched || !native_mupdf_ready(root); - if rebuild_mupdf { - build_mupdf_native(root)?; - write_native_build_marker(root)?; - } else { - println!("Native MuPDF build already present."); - } - - build_mupdf_wrapper_native_if_needed(root)?; - - link_mupdf_artifacts(root)?; - - Ok(()) -} - -/// Builds libwebp from source for native development. -/// -/// # Why we combine static archives manually -/// -/// libwebp's build system creates separate `.a` files in sub-directories -/// (`dec/`, `dsp/`, `enc/`, `utils/`) but does **not** assemble them into a -/// single `src/.libs/libwebp.a` when building static-only. We extract the -/// individual object files from each sub-archive and repack them into one -/// unified `libwebp.a` so that downstream `build.rs` scripts can simply link -/// with `-lwebp`. -pub fn build_libwebp_native(root: &Path) -> Result<()> { - let libwebp_dir = root.join("thirdparty/libwebp"); - if libwebp_dir.join("src/.libs/libwebp.a").exists() { - println!("libwebp already built for native."); - return Ok(()); - } - - println!("Building libwebp for native development…"); - - if !libwebp_dir.join("configure").exists() { - cmd::run("sh", &["autogen.sh"], &libwebp_dir, &[("NOCONFIGURE", "1")]) - .context("failed to run autogen.sh for libwebp")?; - } - - cmd::run( - "./configure", - &[ - "--disable-shared", - "--enable-static", - "--disable-libwebpmux", - "--enable-libwebpdecoder", - "--enable-libwebpdemux", - "--disable-webp-tools", - "--with-pic", - ], - &libwebp_dir, - &[], - ) - .context("failed to configure libwebp")?; - - cmd::run("make", &["-j4"], &libwebp_dir, &[]).context("failed to build libwebp")?; - - combine_libwebp_static_archives(&libwebp_dir)?; - - println!("✓ libwebp built successfully"); - Ok(()) -} - -/// Extracts sub-archives from a static libwebp build and repacks them into -/// a single `src/.libs/libwebp.a`. -/// -/// See [`build_libwebp_native`] for why this is necessary. -fn combine_libwebp_static_archives(libwebp_dir: &Path) -> Result<()> { - let libs_dir = libwebp_dir.join("src/.libs"); - std::fs::create_dir_all(&libs_dir).context("failed to create src/.libs for libwebp")?; - - let sublibs = [ - "dec/.libs/libwebpdecode.a", - "dsp/.libs/libwebpdsp.a", - "enc/.libs/libwebpencode.a", - "utils/.libs/libwebputils.a", - ]; - - let mut objects = vec![]; - for sublib in &sublibs { - let path = libwebp_dir.join("src").join(sublib); - let extract_dir = libs_dir.join(format!("extract_{}", sublib.replace('/', "_"))); - std::fs::create_dir_all(&extract_dir)?; - cmd::run("ar", &["x", path.to_str().unwrap()], &extract_dir, &[]) - .with_context(|| format!("failed to extract objects from {}", path.display()))?; - for entry in std::fs::read_dir(&extract_dir)? { - objects.push(entry?.path()); - } - } - - let libwebp_a = libs_dir.join("libwebp.a"); - let mut ar_args: Vec<&str> = vec!["rcs", libwebp_a.to_str().unwrap()]; - for obj in &objects { - ar_args.push(obj.to_str().unwrap()); - } - cmd::run("ar", &ar_args, &libwebp_dir, &[]).context("failed to create combined libwebp.a")?; - - Ok(()) -} - -/// Ensures MuPDF sources at the required version are present in -/// `thirdparty/mupdf/` and that Cadmus' WebP patch series is applied. -/// -/// If the version header is missing or reports a different version the -/// existing directory is removed and the sources are re-downloaded. -pub fn ensure_mupdf_sources(root: &Path, force: bool) -> Result<()> { - ensure_mupdf_sources_with_webp_patches(root, force).map(|_| ()) -} - -fn ensure_mupdf_sources_with_webp_patches(root: &Path, force: bool) -> Result { - let mupdf_dir = root.join("thirdparty/mupdf"); - let version_header = mupdf_dir.join("include/mupdf/fitz/version.h"); - let current_version = read_mupdf_version(&version_header); - - if force || current_version.as_deref() != Some(MUPDF_VERSION) { - if let Some(v) = ¤t_version { - println!("MuPDF version mismatch: have '{v}', need '{MUPDF_VERSION}'"); - } - - println!("Downloading MuPDF {MUPDF_VERSION} sources…"); - - if mupdf_dir.exists() { - thirdparty::clean_untracked(&mupdf_dir) - .context("failed to clean untracked files from thirdparty/mupdf")?; - } - - thirdparty::download_libraries(&root.join("thirdparty"), &["mupdf"])?; - } else { - println!("MuPDF {MUPDF_VERSION} already present."); - } - - apply_mupdf_webp_patches_if_needed(&mupdf_dir) -} - -fn apply_mupdf_webp_patches_if_needed(mupdf_dir: &Path) -> Result { - let patched = thirdparty::apply_mupdf_webp_patches_if_needed(mupdf_dir)?; - if patched { - remove_native_build_marker(mupdf_dir); - } - - Ok(patched) -} - -fn remove_native_build_marker(mupdf_dir: &Path) { - let marker = mupdf_dir.join(NATIVE_BUILT_MARKER); - if marker.exists() { - std::fs::remove_file(&marker).ok(); - } -} - -/// Reads the MuPDF version string from the version header file. -/// -/// Returns `None` if the file does not exist or the version cannot be parsed. -fn read_mupdf_version(header: &Path) -> Option { - let content = std::fs::read_to_string(header).ok()?; - - // The header contains a line like: #define FZ_VERSION "1.27.0" - for line in content.lines() { - if line.contains("FZ_VERSION") && line.contains('"') { - let start = line.find('"')? + 1; - let end = line.rfind('"')?; - if start < end { - return Some(line[start..end].to_owned()); - } - } - } - - None -} - -/// Returns `true` when the full native setup is complete. -/// -/// Checks that the build marker, the compiled `libmupdf.a`, the C wrapper -/// library `libmupdf_wrapper.a`, and both symlinks in -/// `target/mupdf_wrapper//` are all present. -pub fn native_setup_done(root: &Path) -> bool { - let platform_dir = if cfg!(target_os = "macos") { - "Darwin" - } else { - "Linux" - }; - - let wrapper_dir = root.join(format!("target/mupdf_wrapper/{platform_dir}")); - - native_mupdf_ready(root) - && wrapper_dir.join("libmupdf.a").exists() - && wrapper_dir.join("libmupdf_wrapper.a").exists() -} - -/// Returns `true` when native MuPDF libraries are present and marked as built. -fn native_mupdf_ready(root: &Path) -> bool { - let marker = root.join("thirdparty/mupdf").join(NATIVE_BUILT_MARKER); - if !marker.exists() { - return false; - } - - let libmupdf = root.join("thirdparty/mupdf/build/release/libmupdf.a"); - - libmupdf.exists() -} - -/// Writes the native build marker in the MuPDF source directory. -fn write_native_build_marker(root: &Path) -> Result<()> { - let marker = root.join("thirdparty/mupdf").join(NATIVE_BUILT_MARKER); - std::fs::write(&marker, "").with_context(|| { - format!( - "failed to write native build marker at {}", - marker.display() - ) - }) -} - -/// Builds the `mupdf_wrapper` C static library for the native platform. -fn build_mupdf_wrapper_native_if_needed(root: &Path) -> Result<()> { - println!("Ensuring mupdf_wrapper is available…"); - mupdf_wrapper::build_native_if_needed(root) -} - -fn remove_native_wrapper_artifact(root: &Path) -> Result<()> { - let platform_dir = if cfg!(target_os = "macos") { - "Darwin" - } else { - "Linux" - }; - let lib = root.join(format!( - "target/mupdf_wrapper/{platform_dir}/libmupdf_wrapper.a" - )); - - if lib.exists() { - std::fs::remove_file(&lib) - .with_context(|| format!("failed to remove stale {}", lib.display()))?; - } - - Ok(()) -} - -/// Compiles MuPDF using system libraries for the native platform. -fn build_mupdf_native(root: &Path) -> Result<()> { - println!("Building MuPDF for native development…"); - - let mupdf_dir = root.join("thirdparty/mupdf"); - - // Remove git metadata that interferes with the MuPDF build system. - for entry in ["gitattributes", ".gitattributes"] { - let path = mupdf_dir.join(entry); - if path.exists() { - std::fs::remove_file(&path).ok(); - } - } - - cmd::run("make", &["clean"], &mupdf_dir, &[]).ok(); - cmd::run("make", &["verbose=yes", "generate"], &mupdf_dir, &[])?; - - let sys_cflags = collect_system_cflags()?; - let xcflags = format!( - "-DFZ_ENABLE_ICC=0 -DFZ_ENABLE_SPOT_RENDERING=0 \ - -DFZ_ENABLE_ODT_OUTPUT=0 -DFZ_ENABLE_OCR_OUTPUT=0 \ - -DHAVE_WEBP=1 -I{root}/thirdparty/libwebp/src {sys_cflags}", - root = root.display() - ); - - // Linker flags for libwebp static libraries - let xlibs = format!( - "-L{root}/thirdparty/libwebp/src/.libs -lwebp \ - -L{root}/thirdparty/libwebp/src/demux/.libs -lwebpdemux", - root = root.display() - ); - - cmd::run( - "make", - &[ - "verbose=yes", - "mujs=no", - "tesseract=no", - "extract=no", - "archive=no", - "brotli=no", - "barcode=no", - "commercial=no", - "USE_SYSTEM_LIBS=yes", - &format!("XCFLAGS={xcflags}"), - &format!("XLIBS={xlibs}"), - "libs", - ], - &mupdf_dir, - &[], - ) -} - -/// Collects system library CFLAGS via `pkg-config` on macOS. -/// -/// On Linux, MuPDF's build system detects system libraries automatically. -/// On macOS it needs explicit CFLAGS gathered from pkg-config. -fn collect_system_cflags() -> Result { - if !cfg!(target_os = "macos") { - return Ok(String::new()); - } - - let libs = [ - "freetype2", - "harfbuzz", - "libopenjp2", - "libjpeg", - "libwebp", - "zlib", - "jbig2dec", - "gumbo", - ]; - - let mut flags = String::new(); - for lib in libs { - if let Ok(f) = cmd::output("pkg-config", &["--cflags", lib], Path::new("."), &[]) { - if !f.is_empty() { - flags.push(' '); - flags.push_str(&f); - } - } - } - - Ok(flags.trim().to_owned()) -} - -/// Creates symlinks in `target/mupdf_wrapper//` pointing to the -/// compiled MuPDF static libraries. -fn link_mupdf_artifacts(root: &Path) -> Result<()> { - let platform_dir = if cfg!(target_os = "macos") { - "Darwin" - } else { - "Linux" - }; - - let target_dir = root.join(format!("target/mupdf_wrapper/{platform_dir}")); - std::fs::create_dir_all(&target_dir) - .context("failed to create target/mupdf_wrapper directory")?; - - let release_dir = root.join("thirdparty/mupdf/build/release"); - - let libmupdf = release_dir.join("libmupdf.a"); - if !libmupdf.exists() { - anyhow::bail!("libmupdf.a not found after build -- check MuPDF build output"); - } - - symlink_force(&libmupdf, &target_dir.join("libmupdf.a"))?; - println!("✓ Created libmupdf.a in target/mupdf_wrapper/{platform_dir}"); - - let libmupdf_third = release_dir.join("libmupdf-third.a"); - if !libmupdf_third.exists() { - println!("Creating empty libmupdf-third.a (system libs used instead)…"); - cmd::run("ar", &["cr", "libmupdf-third.a"], &release_dir, &[])?; - } - - symlink_force(&libmupdf_third, &target_dir.join("libmupdf-third.a"))?; - println!("✓ Created libmupdf-third.a"); - - Ok(()) -} - -/// Creates a symlink at `link` pointing to `target`, removing any existing -/// file or symlink at `link` first. -fn symlink_force(target: &Path, link: &Path) -> Result<()> { - if link.exists() || link.symlink_metadata().is_ok() { - std::fs::remove_file(link) - .with_context(|| format!("failed to remove existing {}", link.display()))?; - } - - #[cfg(unix)] - std::os::unix::fs::symlink(target, link) - .with_context(|| format!("failed to create symlink {}", link.display()))?; - - #[cfg(not(unix))] - std::fs::copy(target, link) - .with_context(|| format!("failed to copy {} to {}", target.display(), link.display()))?; - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::fs; - - #[test] - fn read_mupdf_version_parses_define() { - let tmp = tempfile::tempdir().unwrap(); - let header = tmp.path().join("version.h"); - fs::write( - &header, - r#"/* MuPDF version */ -#define FZ_VERSION "1.27.0" -#define FZ_VERSION_MAJOR 1 -"#, - ) - .unwrap(); - - let version = read_mupdf_version(&header); - assert_eq!(version.as_deref(), Some("1.27.0")); - } - - #[test] - fn read_mupdf_version_returns_none_for_malformed_header() { - let tmp = tempfile::tempdir().unwrap(); - let header = tmp.path().join("version.h"); - fs::write(&header, "/* no version define here */\n").unwrap(); - - let version = read_mupdf_version(&header); - assert!(version.is_none()); - } -} diff --git a/xtask/src/tasks/util/mod.rs b/xtask/src/tasks/util/mod.rs index d0ae5c82..3f7030d7 100644 --- a/xtask/src/tasks/util/mod.rs +++ b/xtask/src/tasks/util/mod.rs @@ -5,6 +5,4 @@ pub mod fs; pub mod github; pub mod http; pub mod matrix; -pub mod mupdf_wrapper; -pub mod thirdparty; pub mod workspace; diff --git a/xtask/src/tasks/util/mupdf_wrapper.rs b/xtask/src/tasks/util/mupdf_wrapper.rs deleted file mode 100644 index 4a7bfb72..00000000 --- a/xtask/src/tasks/util/mupdf_wrapper.rs +++ /dev/null @@ -1,115 +0,0 @@ -//! MuPDF wrapper C library compilation helpers. -//! -//! Compiles `mupdf_wrapper.c` into a static library using the C compiler -//! directly. -//! -//! ## Output -//! -//! The compiled object and static library are placed in -//! `target/mupdf_wrapper//`: -//! -//! - `mupdf_wrapper.o` -//! - `libmupdf_wrapper.a` - -use std::path::Path; - -use anyhow::{Context, Result}; - -use super::cmd; - -/// Compiles the mupdf_wrapper C library for the native platform only if the -/// output artifact does not already exist. -/// -/// # Errors -/// -/// Returns an error if compilation or archiving fails. -pub fn build_native_if_needed(root: &Path) -> Result<()> { - let target_os = native_target_os(); - let lib = native_lib_path(root, target_os); - - if lib.exists() { - println!("mupdf_wrapper already built for {target_os}."); - return Ok(()); - } - - build(root, target_os, "cc", "ar", &[]) -} - -fn native_target_os() -> &'static str { - if cfg!(target_os = "macos") { - "Darwin" - } else { - "Linux" - } -} - -fn native_lib_path(root: &Path, target_os: &str) -> std::path::PathBuf { - root.join(format!( - "target/mupdf_wrapper/{target_os}/libmupdf_wrapper.a" - )) -} - -/// Compiles the mupdf_wrapper C library for the Kobo ARM target. -/// -/// Uses the Linaro cross-compiler (`arm-linux-gnueabihf-gcc`) and -/// `arm-linux-gnueabihf-ar`. -/// -/// # Errors -/// -/// Returns an error if compilation or archiving fails. -pub fn build_kobo(root: &Path) -> Result<()> { - build( - root, - "Kobo", - "arm-linux-gnueabihf-gcc", - "arm-linux-gnueabihf-ar", - &[], - ) -} - -/// Compiles `mupdf_wrapper.c` into `libmupdf_wrapper.a`. -/// -/// Equivalent to the shell commands: -/// ```text -/// $CC -I../thirdparty/mupdf/include -c mupdf_wrapper.c -o $BUILD_DIR/mupdf_wrapper.o -/// $AR -rcs $BUILD_DIR/libmupdf_wrapper.a $BUILD_DIR/mupdf_wrapper.o -/// ``` -/// -/// # Errors -/// -/// Returns an error if the compiler or archiver invocation fails. -fn build(root: &Path, target_os: &str, cc: &str, ar: &str, extra_cflags: &[&str]) -> Result<()> { - let wrapper_dir = root.join("mupdf_wrapper"); - let build_dir = root.join(format!("target/mupdf_wrapper/{target_os}")); - - std::fs::create_dir_all(&build_dir) - .with_context(|| format!("failed to create {}", build_dir.display()))?; - - let include = root.join("thirdparty/mupdf/include"); - let obj = build_dir.join("mupdf_wrapper.o"); - let lib = build_dir.join("libmupdf_wrapper.a"); - - let include_flag = format!("-I{}", include.display()); - let obj_str = obj.to_string_lossy().into_owned(); - - let mut compile_args = vec![ - include_flag.as_str(), - "-c", - "mupdf_wrapper.c", - "-o", - &obj_str, - ]; - - for flag in extra_cflags { - compile_args.push(flag); - } - - cmd::run(cc, &compile_args, &wrapper_dir, &[]) - .with_context(|| format!("failed to compile mupdf_wrapper.c with {cc}"))?; - - let lib_str = lib.to_string_lossy().into_owned(); - let obj_str2 = obj.to_string_lossy().into_owned(); - - cmd::run(ar, &["-rcs", &lib_str, &obj_str2], &wrapper_dir, &[]) - .with_context(|| format!("failed to archive libmupdf_wrapper.a with {ar}")) -} diff --git a/xtask/src/tasks/util/thirdparty.rs b/xtask/src/tasks/util/thirdparty.rs deleted file mode 100644 index 27e68fd5..00000000 --- a/xtask/src/tasks/util/thirdparty.rs +++ /dev/null @@ -1,573 +0,0 @@ -//! Thirdparty library download and build helpers. -//! -//! Library source URLs are defined as constants so Renovate can track them. -//! -//! ## Download -//! -//! [`download_libraries`] fetches each library's source. Most libraries are -//! downloaded as tarballs and extracted with the top-level directory stripped. -//! Libraries that use git submodules (currently freetype2) are cloned with -//! `--recurse-submodules` so submodule contents are always present. If the -//! cloned source ships an `autogen.sh` script, it is run immediately after -//! cloning to generate the `configure` script that `build-kobo.sh` expects. -//! -//! ## Build -//! -//! [`build_libraries`] iterates over the packages in dependency order, applies -//! `kobo.patch` if present, then invokes each library's own `build-kobo.sh` -//! script. - -use std::path::Path; - -use anyhow::{Context, Result, bail}; - -use super::{cmd, fs, http}; - -/// Base names of all thirdparty shared libraries. -/// -/// SONAMEs are discovered at runtime via `arm-linux-gnueabihf-readelf -d` -/// because upstream libraries do not follow a consistent ABI versioning scheme. -pub const SONAMES: &[&str] = &[ - "libz.so", - "libbz2.so", - "libpng16.so", - "libjpeg.so", - "libopenjp2.so", - "libjbig2dec.so", - "libfreetype.so", - "libharfbuzz.so", - "libgumbo.so", - "libwebp.so", - "libwebpdemux.so", - "libdjvulibre.so", - "libmupdf.so", -]; - -/// Returns the SONAME of `lib` in `libs_dir`. -/// -/// When the library file exists, `arm-linux-gnueabihf-readelf -d` is used to -/// extract the SONAME from the binary. When only a versioned file exists -/// (e.g. `libz.so.1.2.13` without `libz.so`), the versioned filename is -/// returned directly. -/// -/// # Errors -/// -/// Returns an error if `arm-linux-gnueabihf-readelf` fails or the SONAME -/// cannot be determined. -pub fn soname(libs_dir: &Path, lib: &str) -> Result { - let so_path = libs_dir.join(lib); - if so_path.exists() { - let so_path_str = so_path - .to_str() - .with_context(|| format!("shared library path is not valid UTF-8: {so_path:?}"))?; - let output = cmd::output( - "arm-linux-gnueabihf-readelf", - &["-d", so_path_str], - libs_dir, - &[], - )?; - let soname = output - .lines() - .find(|line| line.contains("SONAME")) - .and_then(|line| line.split_whitespace().last()) - .map(|token| { - token - .trim_start_matches('[') - .trim_end_matches(']') - .to_string() - }) - .with_context(|| format!("failed to find SONAME in readelf output for {lib}"))?; - Ok(soname) - } else { - let prefix = format!("{}.", lib); - let matching: Vec<_> = std::fs::read_dir(libs_dir)? - .filter_map(|e| e.ok()) - .filter(|e| e.file_name().to_string_lossy().starts_with(&prefix)) - .collect(); - - match matching.len() { - 1 => Ok(matching[0].file_name().to_string_lossy().into_owned()), - 0 => bail!( - "no versioned file found for {} in {}", - lib, - libs_dir.display() - ), - _ => bail!( - "multiple versioned files found for {} in {}", - lib, - libs_dir.display() - ), - } - } -} - -/// Version strings for thirdparty libraries tracked by Renovate. -/// -/// Every thirdparty library must have a `VERSION` constant here. The -/// constant is the single source of truth — the download URL is derived -/// from it at call time in [`library_source`]. A corresponding Renovate -/// regex custom manager in `renovate.json` matches each constant and -/// opens PRs when new upstream releases are available. -/// -/// When adding a new thirdparty library, add a `VERSION` constant here -/// and a matching Renovate regex manager entry in `renovate.json`. -pub const ZLIB_VERSION: &str = "1.3.2"; -pub const LIBPNG_VERSION: &str = "1.6.53"; -pub const DJVULIBRE_VERSION: &str = "3.5.30"; -/// IJG libjpeg version tracked via the libjpeg-turbo `jpeg-` tag mirror. -pub const LIBJPEG_VERSION: &str = "10"; - -/// bzip2 version, tracked and downloaded via GitLab `bzip2/bzip2`. -pub const BZIP2_VERSION: &str = "1.0.8"; -/// OpenJPEG version, derived from the archive URL. -pub const OPENJPEG_VERSION: &str = "2.5.4"; -/// jbig2dec version, tracked via GitHub Releases on `ArtifexSoftware/jbig2dec`. -pub const JBIG2DEC_VERSION: &str = "0.20"; -/// FreeType version, cloned from `freetype/freetype` at tag `VER-X-Y-Z`. -/// -/// Tracked by Renovate via the `github-tags` datasource with -/// `extractVersionTemplate: "^VER-(?.+)$"`. freetype2 is cloned -/// rather than downloaded as a tarball because its build system requires the -/// `nyorain/dlg` git submodule, which is absent from archive tarballs. -pub const FREETYPE2_VERSION: &str = "2.14.1"; -/// HarfBuzz version, derived from the archive URL. -pub const HARFBUZZ_VERSION: &str = "14.2.0"; -/// Gumbo version, derived from the archive URL. -pub const GUMBO_VERSION: &str = "0.10.1"; -/// libwebp version, derived from the archive URL. -pub const LIBWEBP_VERSION: &str = "1.2.3"; - -/// MuPDF version, tracked via GitHub Releases on `ArtifexSoftware/mupdf-downloads`. -pub const MUPDF_VERSION: &str = "1.27.2"; - -const MUPDF_WEBP_PATCHES: &[&str] = &[ - "webp-upstream-697749-kobo.patch", // verbatim KOReader upstream - "webp-image-h-kobo.patch", // image.h declarations (our wrapper needs these) - "webp-load-webp-deviations-kobo.patch", // Cadmus deviations: demux cleanup, animation, epsilon, yres, ICC warning -]; - -/// Marker file written after all MuPDF WebP patches succeed. -const WEBP_PATCHED_MARKER: &str = ".webp-patched"; - -/// All libraries in dependency order for building. -const LIBRARY_NAMES: &[&str] = &[ - "zlib", - "bzip2", - "libpng", - "libjpeg", - "openjpeg", - "jbig2dec", - "libwebp", - "freetype2", - "harfbuzz", - "gumbo", - "djvulibre", - "mupdf", -]; - -/// Describes how a thirdparty library's source is obtained. -pub enum LibrarySource { - /// Download a tarball and extract it with the top-level directory stripped. - Tarball(String), - /// Clone a git repository at a specific tag, recursing into submodules. - Git { repo: String, tag: String }, -} - -/// Returns the source descriptor for a named library. -/// -/// # Errors -/// -/// Returns an error if `name` is not a known library. -pub fn library_source(name: &str) -> Result { - match name { - "zlib" => Ok(LibrarySource::Tarball(format!( - "https://github.com/madler/zlib/releases/download/v{v}/zlib-{v}.tar.gz", - v = ZLIB_VERSION - ))), - "bzip2" => Ok(LibrarySource::Tarball(format!( - "https://gitlab.com/bzip2/bzip2/-/archive/bzip2-{v}/bzip2-bzip2-{v}.tar.gz", - v = BZIP2_VERSION - ))), - "libpng" => Ok(LibrarySource::Tarball(format!( - "https://github.com/pnggroup/libpng/archive/refs/tags/v{v}.tar.gz", - v = LIBPNG_VERSION - ))), - "libjpeg" => Ok(LibrarySource::Tarball(format!( - "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/jpeg-{v}.tar.gz", - v = LIBJPEG_VERSION - ))), - "openjpeg" => Ok(LibrarySource::Tarball(format!( - "https://github.com/uclouvain/openjpeg/archive/v{v}.tar.gz", - v = OPENJPEG_VERSION - ))), - "jbig2dec" => Ok(LibrarySource::Tarball(format!( - "https://github.com/ArtifexSoftware/jbig2dec/releases/download/{v}/jbig2dec-{v}.tar.gz", - v = JBIG2DEC_VERSION - ))), - "freetype2" => Ok(LibrarySource::Git { - repo: "https://github.com/freetype/freetype".to_owned(), - tag: format!("VER-{}", FREETYPE2_VERSION.replace('.', "-")), - }), - "harfbuzz" => Ok(LibrarySource::Tarball(format!( - "https://github.com/harfbuzz/harfbuzz/archive/{v}.tar.gz", - v = HARFBUZZ_VERSION - ))), - "gumbo" => Ok(LibrarySource::Tarball(format!( - "https://github.com/google/gumbo-parser/archive/v{v}.tar.gz", - v = GUMBO_VERSION - ))), - "libwebp" => Ok(LibrarySource::Tarball(format!( - "https://github.com/webmproject/libwebp/archive/refs/tags/v{v}.tar.gz", - v = LIBWEBP_VERSION - ))), - "djvulibre" => Ok(LibrarySource::Tarball(format!( - "https://github.com/barak/djvulibre/archive/refs/tags/release.{v}.tar.gz", - v = DJVULIBRE_VERSION - ))), - "mupdf" => Ok(LibrarySource::Tarball(format!( - "https://github.com/ArtifexSoftware/mupdf-downloads/releases/download/{v}/mupdf-{v}-source.tar.gz", - v = MUPDF_VERSION - ))), - _ => bail!("unknown thirdparty library: {name}"), - } -} - -/// Downloads source for the given libraries into `thirdparty/`. -/// -/// When `names` is empty all libraries are downloaded. Tarballs are extracted -/// with the top-level directory stripped. Libraries with a [`LibrarySource::Git`] -/// source are cloned with `--recurse-submodules` so submodule contents are -/// always present. -/// -/// Skips libraries with persisted marker files: -/// - source-ready marker ([`SOURCE_READY_MARKER`]) -/// - built marker ([`BUILT_MARKER`]) -/// -/// This avoids fragile file-heuristic detection across heterogeneous upstream -/// source trees. -/// -/// # Errors -/// -/// Returns an error if any download, extraction, or clone fails. -pub fn download_libraries(thirdparty_dir: &Path, names: &[&str]) -> Result<()> { - let targets: Vec<&str> = if names.is_empty() { - LIBRARY_NAMES.to_vec() - } else { - names.to_vec() - }; - - for name in targets { - let dest_dir = thirdparty_dir.join(name); - - if is_source_ready(&dest_dir) || is_built(&dest_dir) { - println!("Skipping {name} (source ready)…"); - continue; - } - - println!("Downloading {name}…"); - - match library_source(name)? { - LibrarySource::Tarball(url) => { - let tarball = thirdparty_dir.join(format!("{name}.tgz")); - - if dest_dir.exists() { - clean_untracked(&dest_dir)?; - } else { - std::fs::create_dir_all(&dest_dir) - .with_context(|| format!("failed to create {}", dest_dir.display()))?; - } - - http::download(&url, &tarball) - .with_context(|| format!("failed to download {name}"))?; - - fs::extract_tarball_strip_one(&tarball, &dest_dir) - .with_context(|| format!("failed to extract {name}"))?; - - std::fs::remove_file(&tarball).ok(); - - write_marker(&dest_dir, SOURCE_READY_MARKER, name, "source")?; - } - LibrarySource::Git { repo, tag } => { - if !dest_dir.exists() { - std::fs::create_dir_all(&dest_dir) - .with_context(|| format!("failed to create {}", dest_dir.display()))?; - } - - git_clone_tag(&repo, &tag, &dest_dir) - .with_context(|| format!("failed to clone {name}"))?; - - let autogen = dest_dir.join("autogen.sh"); - if autogen.exists() { - cmd::run("./autogen.sh", &[], &dest_dir, &[]) - .with_context(|| format!("failed to run autogen.sh for {name}"))?; - } - - write_marker(&dest_dir, SOURCE_READY_MARKER, name, "source")?; - } - } - } - - Ok(()) -} - -/// Clones `repo` at `tag` into `dest`, recursing into submodules. -/// -/// Clones into a temporary sibling directory first, then moves the contents -/// into `dest`. This preserves any files already in `dest` that are tracked -/// in the cadmus repository (e.g. `build-kobo.sh`), matching the behaviour of -/// tarball extraction with `strip_one`. -fn git_clone_tag(repo: &str, tag: &str, dest: &Path) -> Result<()> { - let tmp = dest.with_extension("_clone_tmp"); - - if tmp.exists() { - std::fs::remove_dir_all(&tmp) - .with_context(|| format!("failed to remove {}", tmp.display()))?; - } - - cmd::run( - "git", - &[ - "clone", - "--depth=1", - "--recurse-submodules", - "--branch", - tag, - repo, - tmp.to_str().context("tmp path is not valid UTF-8")?, - ], - std::path::Path::new("."), - &[], - )?; - - for entry in - std::fs::read_dir(&tmp).with_context(|| format!("failed to read {}", tmp.display()))? - { - let entry = entry.with_context(|| format!("failed to read entry in {}", tmp.display()))?; - - if entry.file_name() == ".git" { - continue; - } - - let target = dest.join(entry.file_name()); - if target.exists() { - if target.is_dir() { - std::fs::remove_dir_all(&target).ok(); - } else { - std::fs::remove_file(&target).ok(); - } - } - std::fs::rename(entry.path(), &target).with_context(|| { - format!( - "failed to move {} to {}", - entry.path().display(), - target.display() - ) - })?; - } - - std::fs::remove_dir_all(&tmp).with_context(|| format!("failed to remove {}", tmp.display()))?; - - Ok(()) -} - -/// Sentinel file written inside a library directory after source extraction. -/// -/// Its presence means the source tree was fetched and unpacked successfully. -pub const SOURCE_READY_MARKER: &str = ".source-ready"; - -/// Sentinel file written inside a library directory after a successful build. -/// -/// Its presence means the library was already compiled and cached — both the -/// patch and the build step can be skipped on the next run. -pub const BUILT_MARKER: &str = ".built-kobo"; - -/// Returns `true` if `dir` already has a completed source download marker. -fn is_source_ready(dir: &Path) -> bool { - dir.join(SOURCE_READY_MARKER).exists() -} - -/// Returns `true` if the library in `dir` was already built and the sentinel -/// file is present. -fn is_built(dir: &Path) -> bool { - dir.join(BUILT_MARKER).exists() -} - -/// Writes a marker file inside `dir` to persist task completion state. -fn write_marker(dir: &Path, marker: &str, name: &str, state: &str) -> Result<()> { - std::fs::write(dir.join(marker), "") - .with_context(|| format!("failed to write {state} marker for {name}")) -} - -/// Builds the given libraries for the Kobo ARM target. -/// -/// When `names` is empty all libraries are built in dependency order. For -/// each library, `kobo.patch` is applied if present, then `./build-kobo.sh` -/// is invoked. A sentinel file ([`BUILT_MARKER`]) is written on success so -/// that a warm CI cache can skip already-built libraries without re-applying -/// the patch or re-running the build script. -/// -/// # Errors -/// -/// Returns an error if patching or building any library fails. -pub fn build_libraries(thirdparty_dir: &Path, names: &[&str]) -> Result<()> { - let targets: Vec<&str> = if names.is_empty() { - LIBRARY_NAMES.to_vec() - } else { - names.to_vec() - }; - - for name in targets { - let lib_dir = thirdparty_dir.join(name); - - if !lib_dir.exists() { - bail!( - "thirdparty/{name} not found — run `cargo xtask build-kobo --download-only` first" - ); - } - - if is_built(&lib_dir) { - println!("Skipping {name} (already built)…"); - continue; - } - - println!("Building {name}…"); - - let patch = lib_dir.join("kobo.patch"); - if patch.exists() { - cmd::run("patch", &["-p", "1", "-i", "kobo.patch"], &lib_dir, &[]) - .with_context(|| format!("failed to apply kobo.patch for {name}"))?; - } - - if name == "mupdf" { - apply_mupdf_webp_patches_if_needed(&lib_dir)?; - } - - let envs = [ - ("AR", "arm-linux-gnueabihf-ar"), - ("AS", "arm-linux-gnueabihf-as"), - ("STRIP", "arm-linux-gnueabihf-strip"), - ("RANLIB", "arm-linux-gnueabihf-ranlib"), - ("LD", "arm-linux-gnueabihf-ld"), - ("CC_FOR_BUILD", "cc"), - ("CXX_FOR_BUILD", "c++"), - ("CC_BUILD", "cc"), - ]; - cmd::run("./build-kobo.sh", &[], &lib_dir, &envs) - .with_context(|| format!("failed to build {name}"))?; - - write_marker(&lib_dir, BUILT_MARKER, name, "build")?; - write_marker(&lib_dir, SOURCE_READY_MARKER, name, "source")?; - } - - Ok(()) -} - -/// Applies Cadmus' MuPDF WebP patch series unless it was already applied. -/// -/// Returns `true` when patches were applied during this call. -pub fn apply_mupdf_webp_patches_if_needed(mupdf_dir: &Path) -> Result { - if mupdf_webp_patches_applied(mupdf_dir) { - println!("MuPDF WebP patches already applied."); - Ok(false) - } else { - println!("Applying MuPDF WebP patches…"); - for patch in MUPDF_WEBP_PATCHES { - cmd::run("patch", &["-p", "1", "-i", patch], mupdf_dir, &[]) - .with_context(|| format!("failed to apply {patch}"))?; - } - - write_marker(mupdf_dir, WEBP_PATCHED_MARKER, "mupdf", "WebP patch")?; - Ok(true) - } -} - -fn mupdf_webp_patches_applied(mupdf_dir: &Path) -> bool { - mupdf_dir.join(WEBP_PATCHED_MARKER).exists() -} - -/// Removes untracked files from a directory using `git ls-files`, falling back -/// to removing and recreating the directory when git is unavailable. -pub fn clean_untracked(dir: &Path) -> Result<()> { - let result = std::process::Command::new("git") - .args(["ls-files", "-o", "--directory", "-z"]) - .arg(dir.file_name().unwrap_or(dir.as_os_str())) - .current_dir(dir.parent().unwrap_or(dir)) - .output(); - - match result { - Ok(output) if output.status.success() => { - for entry in output.stdout.split(|&b| b == 0) { - if entry.is_empty() { - continue; - } - - let path = dir - .parent() - .unwrap_or(dir) - .join(std::str::from_utf8(entry).unwrap_or("")); - - if path.is_dir() { - std::fs::remove_dir_all(&path).ok(); - } else { - std::fs::remove_file(&path).ok(); - } - } - } - _ => { - std::fs::remove_dir_all(dir) - .with_context(|| format!("failed to remove {}", dir.display()))?; - std::fs::create_dir_all(dir) - .with_context(|| format!("failed to recreate {}", dir.display()))?; - } - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn library_source_is_defined_for_all_known_libraries() { - for name in LIBRARY_NAMES { - let source = library_source(name).unwrap(); - match source { - LibrarySource::Tarball(url) => { - assert!( - url.starts_with("http"), - "tarball URL for {name} should start with http" - ); - assert!( - url.contains(".tar.gz"), - "tarball URL for {name} should contain .tar.gz" - ); - } - LibrarySource::Git { repo, tag } => { - assert!( - repo.starts_with("https://"), - "git repo for {name} should use https" - ); - assert!(!tag.is_empty(), "git tag for {name} should not be empty"); - } - } - } - } - - #[test] - fn library_source_errors_on_unknown_library() { - assert!(library_source("nonexistent").is_err()); - } - - #[test] - fn library_names_has_no_duplicates() { - let mut names = LIBRARY_NAMES.to_vec(); - names.sort_unstable(); - names.dedup(); - assert_eq!( - names.len(), - LIBRARY_NAMES.len(), - "duplicate library names found" - ); - } -} From 98a83774cbd56c7bf0548ea0ccf8852374129879 Mon Sep 17 00:00:00 2001 From: Kevin Hellemun <17928966+OGKevin@users.noreply.github.com> Date: Sun, 7 Jun 2026 18:29:04 +0200 Subject: [PATCH 2/2] chore: more review Change-Id: bc5eecbd94d24946167efb1a7985f961 Change-Id-Short: onullnomqvmx --- .github/actions/cargo-cache/action.yml | 67 ++++++++++++++ .github/workflows/build-release-candidate.yml | 15 +--- .github/workflows/cadmus-docs.yml | 15 +--- .github/workflows/cargo.yml | 69 +++++++------- .github/workflows/copilot-setup-steps.yml | 15 +--- .github/workflows/release.yml | 15 +--- crates/build-deps/src/build/kobo.rs | 28 ++++-- crates/build-deps/src/build/native.rs | 34 ++++--- crates/build-deps/src/markers.rs | 90 +++++++++++++++++-- crates/core/build.rs | 39 +++++--- 10 files changed, 266 insertions(+), 121 deletions(-) create mode 100644 .github/actions/cargo-cache/action.yml diff --git a/.github/actions/cargo-cache/action.yml b/.github/actions/cargo-cache/action.yml new file mode 100644 index 00000000..a5c041a5 --- /dev/null +++ b/.github/actions/cargo-cache/action.yml @@ -0,0 +1,67 @@ +name: Cargo shared cache +description: > + Restore or save the shared Cargo cache. The cache key incorporates the + Rust toolchain fingerprint, the Cargo lockfile, build scripts, the + mupdf_wrapper C glue, and the gitlink SHAs of every thirdparty + submodule so that the key changes when any submodule pointer moves. + + Defaults to restore-only. Pass `mode: save` in the cache-warmup job to + write a fresh cache. + +inputs: + cachekey: + description: "The dtolnay/rust-toolchain cachekey output" + required: true + mode: + description: "`restore` (default) or `save` (read-write)" + required: false + default: restore + +outputs: + cache-hit: + description: "Whether a matching cache entry was found" + value: ${{ steps.save.outputs.cache-hit || steps.restore.outputs.cache-hit }} + +runs: + using: composite + steps: + - name: Compute submodule hash + id: submodules + shell: bash + run: | + # Hash all thirdparty submodule gitlink SHAs so the cache key + # changes when any submodule pointer moves. + sha=$(git ls-tree HEAD thirdparty/ | sha256sum | cut -c1-12) + echo "hash=$sha" >> "$GITHUB_OUTPUT" + + - name: Restore shared cargo cache + if: inputs.mode == 'restore' + id: restore + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + libs/ + key: ${{ runner.os }}-cargo-shared-${{ inputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }}-${{ steps.submodules.outputs.hash }} + restore-keys: | + ${{ runner.os }}-cargo-shared-${{ inputs.cachekey }}- + + - name: Save shared cargo cache + if: inputs.mode == 'save' + id: save + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + libs/ + key: ${{ runner.os }}-cargo-shared-${{ inputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }}-${{ steps.submodules.outputs.hash }} + restore-keys: | + ${{ runner.os }}-cargo-shared-${{ inputs.cachekey }}- diff --git a/.github/workflows/build-release-candidate.yml b/.github/workflows/build-release-candidate.yml index cbe5b28e..a47a800e 100644 --- a/.github/workflows/build-release-candidate.yml +++ b/.github/workflows/build-release-candidate.yml @@ -47,19 +47,10 @@ jobs: with: targets: arm-unknown-linux-gnueabihf - - name: Restore shared cargo cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + - uses: ./.github/actions/cargo-cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - libs/ - key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }} - restore-keys: | - ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}- + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} + mode: restore - name: Build Kobo artifacts uses: ./.github/actions/build-kobo diff --git a/.github/workflows/cadmus-docs.yml b/.github/workflows/cadmus-docs.yml index fe127509..591de5ef 100644 --- a/.github/workflows/cadmus-docs.yml +++ b/.github/workflows/cadmus-docs.yml @@ -72,19 +72,10 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} - - name: Restore shared cargo artifacts - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + - uses: ./.github/actions/cargo-cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - libs/ - key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }} - restore-keys: | - ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}- + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} + mode: restore - name: Build documentation portal env: diff --git a/.github/workflows/cargo.yml b/.github/workflows/cargo.yml index 23f00f7c..6ec14d04 100644 --- a/.github/workflows/cargo.yml +++ b/.github/workflows/cargo.yml @@ -16,8 +16,12 @@ on: - ".github/workflows/cargo.yml" - ".github/actions/build-kobo/**" - ".github/actions/build-docs-epub/**" + - ".github/actions/cargo-cache/**" - ".github/actions/install-doc-tools/**" - "build-scripts/**" + - ".gitmodules" + - "mupdf_wrapper/**" + - "thirdparty/**" concurrency: group: "cargo-${{ github.event_name }}-${{ github.event.number || github.ref }}" @@ -34,20 +38,10 @@ jobs: id: rust-toolchain with: components: rustfmt - - &restore-shared-cache - name: Restore shared cargo cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + - name: Restore shared cargo cache + uses: ./.github/actions/cargo-cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - libs/ - key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }} - restore-keys: | - ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}- + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} - name: Check formatting run: cargo xtask fmt @@ -62,7 +56,10 @@ jobs: - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@stable id: rust-toolchain - - *restore-shared-cache + - name: Restore shared cargo cache + uses: ./.github/actions/cargo-cache + with: + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} - name: Generate feature matrix id: matrix run: cargo xtask ci matrix @@ -75,7 +72,10 @@ jobs: - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@stable id: rust-toolchain - - *restore-shared-cache + - name: Restore shared cargo cache + uses: ./.github/actions/cargo-cache + with: + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} - name: Build documentation EPUB uses: ./.github/actions/build-docs-epub with: @@ -125,18 +125,10 @@ jobs: run: echo "$HOME/linaro-toolchain/bin" >> "$GITHUB_PATH" - name: Cache shared cargo artifacts id: cache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae + uses: ./.github/actions/cargo-cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - libs/ - key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }} - restore-keys: | - ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}- + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} + mode: save - name: Cache cargo-nextest uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae @@ -191,7 +183,10 @@ jobs: with: components: clippy - *set-build-metadata - - *restore-shared-cache + - name: Restore shared cargo cache + uses: ./.github/actions/cargo-cache + with: + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} - name: Download documentation EPUB uses: actions/download-artifact@v8 with: @@ -231,7 +226,10 @@ jobs: with: components: clippy - *set-build-metadata - - *restore-shared-cache + - name: Restore shared cargo cache + uses: ./.github/actions/cargo-cache + with: + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} - name: Download all clippy JSON artifacts uses: actions/download-artifact@v8 with: @@ -295,7 +293,10 @@ jobs: if: runner.os == 'macOS' run: brew install sdl2 freetype harfbuzz openjpeg jpeg-turbo libpng jbig2dec gumbo-parser djvulibre pkg-config - - *restore-shared-cache + - name: Restore shared cargo cache + uses: ./.github/actions/cargo-cache + with: + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} - name: Cache cargo-nextest uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae @@ -333,7 +334,10 @@ jobs: with: targets: arm-unknown-linux-gnueabihf - - *restore-shared-cache + - name: Restore shared cargo cache + uses: ./.github/actions/cargo-cache + with: + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} - name: Download documentation EPUB uses: actions/download-artifact@v8 with: @@ -400,7 +404,10 @@ jobs: with: targets: arm-unknown-linux-gnueabihf - - *restore-shared-cache + - name: Restore shared cargo cache + uses: ./.github/actions/cargo-cache + with: + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} - name: Download documentation EPUB uses: actions/download-artifact@v8 with: diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index bbbb2380..d509391d 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -26,19 +26,10 @@ jobs: components: rustfmt, clippy targets: arm-unknown-linux-gnueabihf - - name: Restore shared cargo cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + - uses: ./.github/actions/cargo-cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - libs/ - key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }} - restore-keys: | - ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}- + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} + mode: restore - name: Restore Linaro toolchain id: cache-linaro diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d8d302aa..ebc0e6cc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,19 +74,10 @@ jobs: with: targets: arm-unknown-linux-gnueabihf - - name: Restore shared cargo cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + - uses: ./.github/actions/cargo-cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - libs/ - key: ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('**/Cargo.lock', 'crates/build-deps/**', 'build-scripts/**', '.gitmodules', 'mupdf_wrapper/mupdf_wrapper.c', 'crates/core/build.rs') }} - restore-keys: | - ${{ runner.os }}-cargo-shared-${{ steps.rust-toolchain.outputs.cachekey }}- + cachekey: ${{ steps.rust-toolchain.outputs.cachekey }} + mode: restore - name: Build documentation EPUB uses: ./.github/actions/build-docs-epub diff --git a/crates/build-deps/src/build/kobo.rs b/crates/build-deps/src/build/kobo.rs index b49bad70..4a8bacf5 100644 --- a/crates/build-deps/src/build/kobo.rs +++ b/crates/build-deps/src/build/kobo.rs @@ -70,7 +70,8 @@ fn build_libraries(thirdparty_dir: &Path) -> Result<()> { } let build_dir = build_root.join(name); - if markers::is_built(&build_dir) { + let submodule_path = format!("thirdparty/{name}"); + if markers::is_built(root, &build_dir, &submodule_path) { println!("Skipping {name} (already built)..."); continue; } @@ -84,7 +85,7 @@ fn build_libraries(thirdparty_dir: &Path) -> Result<()> { source::apply_patches(&build_dir, name, root)?; recipes::build_library(name, &build_dir)?; - markers::mark_built(&build_dir, name)?; + markers::mark_built(root, &build_dir, name, &submodule_path)?; } Ok(()) @@ -155,9 +156,13 @@ pub fn ensure_kobo_artifacts(root: &Path) -> Result<()> { Ok(()) } -/// Returns `true` when the Kobo artefact cache is complete: every -/// [`SONAMES`] entry is present in `libs/` and the `mupdf_wrapper` -/// archive has been built for the Kobo target. +/// Returns `true` when the Kobo artefact cache is complete: +/// +/// * every [`SONAMES`] entry is present in `libs/`, +/// * the `mupdf_wrapper` archive has been built for the Kobo target, and +/// * every per-library `.built` marker under +/// `target/cadmus-build-deps//` matches the current submodule +/// gitlink SHA. fn kobo_artifacts_present(root: &Path) -> bool { let libs_dir = root.join("libs"); if !libs_dir.exists() { @@ -166,8 +171,19 @@ fn kobo_artifacts_present(root: &Path) -> bool { if !SONAMES.iter().all(|lib| libs_dir.join(lib).exists()) { return false; } - root.join("target/mupdf_wrapper/Kobo/libmupdf_wrapper.a") + if !root + .join("target/mupdf_wrapper/Kobo/libmupdf_wrapper.a") .exists() + { + return false; + } + + let kobo_build_root = build_root(root); + versions::LIBRARY_NAMES.iter().all(|name| { + let build_dir = kobo_build_root.join(name); + let submodule_path = format!("thirdparty/{name}"); + markers::is_built(root, &build_dir, &submodule_path) + }) } /// Create the `.so` → versioned-SONAME symlinks the Cadmus runtime diff --git a/crates/build-deps/src/build/native.rs b/crates/build-deps/src/build/native.rs index 834187c5..5bcf8233 100644 --- a/crates/build-deps/src/build/native.rs +++ b/crates/build-deps/src/build/native.rs @@ -59,15 +59,15 @@ pub struct NativeArtifacts { /// Returns an error if submodules cannot be initialised, the MuPDF /// version does not match, or any of the underlying build steps fail. pub fn ensure_native_artifacts(root: &Path) -> Result { - if !native_cache_complete(root) { - crate::ensure_submodules(root).context("failed to initialise git submodules")?; - } - let mupdf_src = root.join("thirdparty/mupdf"); let mupdf_build = build_root(root).join("mupdf"); - let version_header = mupdf_src.join("include/mupdf/fitz/version.h"); - if !native_cache_complete(root) { + let cache_hit = native_cache_complete(root); + + if !cache_hit { + crate::ensure_submodules(root).context("failed to initialise git submodules")?; + + let version_header = mupdf_src.join("include/mupdf/fitz/version.h"); let current_version = read_mupdf_version(&version_header); if current_version.as_deref() != Some(MUPDF_VERSION) { bail!( @@ -100,9 +100,14 @@ pub fn ensure_native_artifacts(root: &Path) -> Result { } /// Returns `true` when every native build artefact is already on disk -/// and a fresh build can be skipped. Used by +/// and **matches the current submodule revision**. Used by /// [`ensure_native_artifacts`] to avoid `git submodule update /// --init --recursive` on warm CI caches. +/// +/// Each `.built` marker now stores the submodule gitlink SHA that was +/// used for the last successful build. If the submodule pointer has +/// moved (e.g. after a `git submodule update`), the cache is stale and +/// a full rebuild is triggered. fn native_cache_complete(root: &Path) -> bool { let build_root = build_root(root); let libwebp_dir = build_root.join("libwebp"); @@ -111,8 +116,8 @@ fn native_cache_complete(root: &Path) -> bool { let mupdf_third_a = mupdf_dir.join("build/release/libmupdf-third.a"); let mupdf_include = mupdf_dir.join("include"); - markers::is_built(&libwebp_dir) - && markers::is_built(&mupdf_dir) + markers::is_built(root, &libwebp_dir, "thirdparty/libwebp") + && markers::is_built(root, &mupdf_dir, "thirdparty/mupdf") && mupdf_a.exists() && mupdf_third_a.exists() && mupdf_include.is_dir() @@ -137,7 +142,7 @@ pub fn build_libwebp_native(root: &Path) -> Result<()> { let build_root = build_root(root); let libwebp_dir = build_root.join("libwebp"); - if markers::is_built(&libwebp_dir) { + if markers::is_built(root, &libwebp_dir, "thirdparty/libwebp") { println!("libwebp already built for native."); return Ok(()); } @@ -177,15 +182,16 @@ pub fn build_libwebp_native(root: &Path) -> Result<()> { combine_libwebp_static_archives(&libwebp_dir)?; let libwebp_a = libwebp_dir.join("src/.libs/libwebp.a"); + let ranlib_tool = resolve_ranlib_tool(); cmd::run( - "ranlib", + &ranlib_tool, &[libwebp_a.to_str().context("non-UTF-8 libwebp.a path")?], &libwebp_dir, &[], ) .context("failed to ranlib libwebp.a")?; - markers::mark_built(&libwebp_dir, "libwebp")?; + markers::mark_built(root, &libwebp_dir, "libwebp", "thirdparty/libwebp")?; println!("✓ libwebp built successfully"); Ok(()) } @@ -338,7 +344,7 @@ pub fn build_mupdf_native(root: &Path) -> Result<()> { let build_root = build_root(root); let mupdf_dir = build_root.join("mupdf"); - if markers::is_built(&mupdf_dir) { + if markers::is_built(root, &mupdf_dir, "thirdparty/mupdf") { println!("MuPDF already built for native."); return Ok(()); } @@ -398,7 +404,7 @@ pub fn build_mupdf_native(root: &Path) -> Result<()> { ) .context("failed to build MuPDF libs")?; - markers::mark_built(&mupdf_dir, "mupdf")?; + markers::mark_built(root, &mupdf_dir, "mupdf", "thirdparty/mupdf")?; Ok(()) } diff --git a/crates/build-deps/src/markers.rs b/crates/build-deps/src/markers.rs index 670d52e9..ac40acaf 100644 --- a/crates/build-deps/src/markers.rs +++ b/crates/build-deps/src/markers.rs @@ -4,6 +4,14 @@ //! Marker files live next to the artifacts they describe. Removing //! them is the supported way to force a rebuild of just the affected //! library without clearing the whole target directory. +//! +//! # Version-aware markers +//! +//! [`mark_built`] writes the current submodule gitlink SHA into +//! the `.built` marker. [`is_built`] compares the stored SHA +//! against the live submodule revision. When the submodule pointer +//! changes (e.g. after `git submodule update`), the marker becomes +//! stale and the library is rebuilt automatically. use std::path::{Path, PathBuf}; @@ -24,19 +32,66 @@ pub fn built_marker_path(dir: &Path) -> PathBuf { dir.join(BUILT_MARKER) } -/// Returns `true` if [`BUILT_MARKER`] is present in `dir`. -pub fn is_built(dir: &Path) -> bool { - built_marker_path(dir).exists() +/// Returns the current gitlink (tree-entry) SHA for the submodule at +/// `submodule_path` relative to `root`. Returns `None` when git is +/// unavailable or the path does not track a submodule. +/// +/// The output for ls-tree is: +/// `160000 commit \t` +pub fn submodule_commit(root: &Path, submodule_path: &str) -> Option { + let output = std::process::Command::new("git") + .args(["ls-tree", "HEAD", submodule_path]) + .current_dir(root) + .output() + .ok()?; + + if !output.status.success() { + return None; + } + + let stdout = String::from_utf8_lossy(&output.stdout); + stdout.split_whitespace().nth(2).map(|s| s.to_owned()) +} + +/// Returns `true` when `dir` has a `.built` marker whose content +/// matches the current gitlink SHA for `submodule_path`. +/// +/// An empty or missing marker is treated as stale, so old-style +/// markers (written by a previous version of this crate) will +/// trigger a rebuild. +pub fn is_built(root: &Path, dir: &Path, submodule_path: &str) -> bool { + let marker_path = built_marker_path(dir); + let stored_hash = match std::fs::read_to_string(&marker_path) { + Ok(s) => s.trim().to_owned(), + Err(_) => return false, + }; + if stored_hash.is_empty() { + return false; + } + + let current_hash = match submodule_commit(root, submodule_path) { + Some(h) => h, + None => return false, + }; + + stored_hash == current_hash } -/// Write [`BUILT_MARKER`] in `dir`, recording that `name` has been -/// built successfully. +/// Write `.built` marker in `dir` with the current gitlink SHA +/// for `submodule_path`, recording that `name` has been built +/// successfully against that revision. /// /// # Errors /// -/// Returns an error if the marker file cannot be written. -pub fn mark_built(dir: &Path, name: &str) -> Result<()> { - write_marker(dir, BUILT_MARKER, name, "build") +/// Returns an error if the submodule commit cannot be resolved or the +/// marker file cannot be written. +pub fn mark_built(root: &Path, dir: &Path, name: &str, submodule_path: &str) -> Result<()> { + let hash = submodule_commit(root, submodule_path) + .with_context(|| format!("failed to resolve submodule commit for {submodule_path}"))?; + let marker_path = dir.join(BUILT_MARKER); + std::fs::write(&marker_path, hash.as_bytes()) + .with_context(|| format!("failed to write build marker for {name}"))?; + Ok(()) } /// Returns `true` if [`WEBP_PATCHED_MARKER`] is present in `mupdf_dir`. @@ -54,3 +109,22 @@ pub fn write_marker(dir: &Path, marker: &str, name: &str, state: &str) -> Result std::fs::write(dir.join(marker), "") .with_context(|| format!("failed to write {state} marker for {name}")) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn submodule_commit_returns_sha_for_known_path() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(); + let sha = submodule_commit(root, "thirdparty/mupdf"); + assert!(sha.is_some(), "mupdf submodule should resolve"); + let sha = sha.unwrap(); + assert_eq!(sha.len(), 40); + assert!(sha.chars().all(|c| c.is_ascii_hexdigit())); + } +} diff --git a/crates/core/build.rs b/crates/core/build.rs index 75081d65..12fbf463 100644 --- a/crates/core/build.rs +++ b/crates/core/build.rs @@ -21,7 +21,7 @@ use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; use uuid::Uuid; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Ok, Result, bail}; use build_deps::build::kobo; use build_deps::build::{mupdf_wrapper, native}; @@ -88,23 +88,34 @@ fn try_main() -> Result<()> { let root = workspace_root()?; - if !skip_thirdparty_deps() { - if target == "arm-unknown-linux-gnueabihf" { - emit_kobo_link_directives(); - kobo::ensure_kobo_artifacts(&root)?; - } else { - let artifacts = - native::ensure_native_artifacts(&root).context("failed to build native deps")?; - mupdf_wrapper::build_native_if_needed(&root, &artifacts.include) - .context("failed to build mupdf_wrapper")?; - emit_native_link_directives(&root, &target)?; + generate_locales()?; + generate_bundled_assets()?; + + if skip_thirdparty_deps() { + println!("Skipping thirdparty deps"); + + return Ok(()); + } + + let host = env::var("HOST").context("HOST not set")?; + if target == "arm-unknown-linux-gnueabihf" { + emit_kobo_link_directives(); + kobo::ensure_kobo_artifacts(&root)?; + } else { + if host != target { + bail!( + "cross-compilation detected: HOST={host} != TARGET={target}. Run with CADMUS_SKIP_THIRDPARTY_DEPS=1 or set up cross-compilation support." + ); } - emit_common_link_directives(); + let artifacts = + native::ensure_native_artifacts(&root).context("failed to build native deps")?; + mupdf_wrapper::build_native_if_needed(&root, &artifacts.include) + .context("failed to build mupdf_wrapper")?; + emit_native_link_directives(&root, &target)?; } - generate_locales()?; - generate_bundled_assets()?; + emit_common_link_directives(); Ok(()) }