diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..7e578dd0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +/backend/target diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3b9c1b0..d92e3be0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,4 +63,17 @@ jobs: run: flutter analyze --no-fatal-infos - name: formatting - run: dart format --set-exit-if-changed . \ No newline at end of file + run: dart format --set-exit-if-changed . + e2eTest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Set up Docker + uses: docker/setup-docker-action@v5 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Run E2E test + run: ./scripts/test-e2e.sh hello diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 4a95fe83..349a6244 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -6,7 +6,7 @@ on: - master tags: - '*' - pull_request: + pull_request_target: branches: - master @@ -32,6 +32,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -51,7 +53,7 @@ jobs: - name: Determine image tag id: vars run: | - if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + if [[ "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then echo "tag=pr-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT elif [[ "${GITHUB_REF}" == "refs/heads/master" ]]; then echo "tag=git" >> $GITHUB_OUTPUT diff --git a/backend/aurcache-builder/src/build.rs b/backend/aurcache-builder/src/build.rs index 619e19d1..6b2c792b 100644 --- a/backend/aurcache-builder/src/build.rs +++ b/backend/aurcache-builder/src/build.rs @@ -1,4 +1,4 @@ -use crate::env::job_timeout_from_env; +use crate::env::{builder_image_from_env, job_timeout_from_env}; use crate::logger::BuildLogger; use crate::path_utils::create_active_build_path; use crate::types::BuildStates; @@ -12,15 +12,13 @@ use bollard::query_parameters::{ use futures::StreamExt; use sea_orm::{ActiveModelTrait, DatabaseConnection, IntoActiveModel, Set, TransactionTrait}; use std::collections::HashMap; +use std::fs; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; -use std::{env, fs}; use tokio::sync::Mutex; use tokio::time::timeout; use tracing::{debug, info}; -static BUILDER_IMAGE_DEFAULT: &str = "ghcr.io/lukas-heiligenbrunner/aurcache-builder:latest"; - pub struct Builder { pub(crate) db: DatabaseConnection, pub(crate) job_containers: Arc>>, @@ -63,13 +61,7 @@ impl Builder { info!("Preparing build #{}", self.build_model.id.get()?); let target_platform = self.prepare_build().await?; - let builder_image = match env::var("BUILDER_IMAGE") { - Ok(v) => { - info!("Using non-default Builder image: {v}"); - v - } - Err(_) => BUILDER_IMAGE_DEFAULT.to_string(), - }; + let builder_image = builder_image_from_env(); info!( "Build #{}: Repull builder image", @@ -84,7 +76,7 @@ impl Builder { ); let pkgname = self.package_model.name.get()?; - let host_active_build_path = create_active_build_path(pkgname.clone())?; + let host_active_build_path = create_active_build_path(pkgname)?; let create_info = self .create_build_container(target_platform, builder_image.as_str()) @@ -139,7 +131,7 @@ impl Builder { "Build {}: Remove shared build folder", self.build_model.id.get()? ); - fs::remove_dir(host_active_build_path)?; + fs::remove_dir_all(host_active_build_path)?; Ok(()) } diff --git a/backend/aurcache-builder/src/docker.rs b/backend/aurcache-builder/src/docker.rs index 8c38599b..dc0b323d 100644 --- a/backend/aurcache-builder/src/docker.rs +++ b/backend/aurcache-builder/src/docker.rs @@ -138,21 +138,23 @@ and check also if the 'DOCKER_HOST=unix:///var/run/user/1000/podman/podman.sock' let build_flags = self.package_model.build_flags.get()?.split(';').join(" "); // create new docker container for current build - let build_dir_base = "/var/cache/makepkg/pkg"; - let host_build_path_docker = match get_build_mode() { + let host_build_dir = match get_build_mode() { BuildMode::DinD(cfg) => cfg.build_path, BuildMode::Host(cfg) => cfg.build_artifact_dir_host, }; - let mountpoints = vec![format!("{}:{}", host_build_path_docker, build_dir_base)]; + let container_build_dir = "/build"; + let mountpoints = vec![format!("{}/{name}:{container_build_dir}", host_build_dir)]; let mut mounts = vec![]; // todo allow for custom mirrorlists for other archs if arch == "linux/x86_64" { - let archlinux_mirrorlist_path = "/etc/pacman.d"; + // Mount only the mirrorlist file, not the entire directory + // This preserves other files in /etc/pacman.d (like gnupg keyring) + let archlinux_mirrorlist_path = "/etc/pacman.d/mirrorlist"; let mnt = match get_build_mode() { BuildMode::DinD(cfg) => { - let mirrorlist_path = cfg.mirrorlist_path; + let mirrorlist_path = format!("{}/mirrorlist", cfg.mirrorlist_path); Mount { target: Some(archlinux_mirrorlist_path.to_string()), @@ -163,7 +165,7 @@ and check also if the 'DOCKER_HOST=unix:///var/run/user/1000/podman/podman.sock' } } BuildMode::Host(cfg) => { - let mirrorlist_path = cfg.mirrorlist_path_host; + let mirrorlist_path = format!("{}/mirrorlist", cfg.mirrorlist_path_host); if mirrorlist_path.starts_with('/') { Mount { target: Some(archlinux_mirrorlist_path.to_string()), @@ -183,7 +185,7 @@ and check also if the 'DOCKER_HOST=unix:///var/run/user/1000/podman/podman.sock' typ: Some(MountTypeEnum::VOLUME), read_only: Some(false), volume_options: Some(MountVolumeOptions { - subpath: Some(subpath.to_string()), + subpath: Some(format!("{}/mirrorlist", subpath)), ..Default::default() }), ..Default::default() @@ -194,35 +196,20 @@ and check also if the 'DOCKER_HOST=unix:///var/run/user/1000/podman/podman.sock' mounts.push(mnt); } - let (makepkg_config, makepkg_config_path) = - create_makepkg_config(name.clone(), build_dir_base)?; + let (makepkg_config, makepkg_config_path) = create_makepkg_config(container_build_dir)?; + let self_update = "paru -Syu --noconfirm --noprogressbar --color never"; let source_data = SourceData::from_str(self.package_model.source_data.get()?)?; let build_cmd = match source_data { SourceData::Aur { .. } => { - let build_cmd = format!("paru {build_flags} {name}"); - // first update the package list, then update trustdb and then build cmd - let steps = [ - "sudo pacman -Sy --noconfirm", - "sudo pacman-key --init", - "sudo pacman-key --populate archlinux", - build_cmd.as_str(), - ]; - steps.join(" && ") + format!( + "cd {container_build_dir} && {self_update} && paru -G {name} && paru {build_flags} *" + ) } SourceData::Git { .. } => { - // somehow we need also to `cd` into the repo dir, otherwise builds fail - let build_cmd = format!( - "sudo chmod -R 1777 {GIT_REPO_PATH} && cd {GIT_REPO_PATH} && paru {build_flags} {GIT_REPO_PATH}" - ); - // first update the package list, then update trustdb and then build cmd - let steps = [ - "sudo pacman -Sy --noconfirm", - "sudo pacman-key --init", - "sudo pacman-key --populate archlinux", - build_cmd.as_str(), - ]; - steps.join(" && ") + format!( + "chmod -R 1777 {GIT_REPO_PATH} && {self_update} && cd {GIT_REPO_PATH} && paru {build_flags} *" + ) } SourceData::Upload { .. } => { todo!("unpack zip and store it in build container dir") @@ -239,23 +226,16 @@ and check also if the 'DOCKER_HOST=unix:///var/run/user/1000/podman/podman.sock' let build_id = self.build_model.id.get()?; let container_name = format!("aurcache_build_{filtered_name}_{build_id}"); + let auto_remove = cfg!(not(debug_assertions)); let conf = ContainerCreateBody { image: Some(image_name.to_string()), attach_stdout: Some(true), attach_stderr: Some(true), open_stdin: Some(false), user: Some("ab".to_string()), - cmd: Some(vec![ - "sh".to_string(), - "-l".to_string(), - "-c".to_string(), - cmd, - ]), + cmd: Some(vec!["sh".to_string(), "-lec".to_string(), cmd]), host_config: Some(HostConfig { - #[cfg(debug_assertions)] - auto_remove: Some(false), - #[cfg(not(debug_assertions))] - auto_remove: Some(true), + auto_remove: Some(auto_remove), nano_cpus: Some(cpu_limit as i64), memory_swap: Some(memory_limit), binds: Some(mountpoints), diff --git a/backend/aurcache-builder/src/env.rs b/backend/aurcache-builder/src/env.rs index c13993d4..581ec0fc 100644 --- a/backend/aurcache-builder/src/env.rs +++ b/backend/aurcache-builder/src/env.rs @@ -2,6 +2,8 @@ use std::env; use std::time::Duration; use tracing::debug; +static DEFAULT_BUILDER_IMAGE: &str = "ghcr.io/lukas-heiligenbrunner/aurcache-builder:latest"; + pub fn job_timeout_from_env() -> Duration { let job_timeout = env::var("JOB_TIMEOUT") .ok() @@ -12,13 +14,11 @@ pub fn job_timeout_from_env() -> Duration { } pub fn limits_from_env() -> (u64, i64) { - // cpu_limit in milli cpus let cpu_limit = env::var("CPU_LIMIT") .ok() .and_then(|x| x.parse::().ok()) .map_or(0, |x| x * 1_000_000); debug!("cpu_limit: {cpu_limit} mCPUs"); - // memory_limit in megabytes let memory_limit = env::var("MEMORY_LIMIT") .ok() .and_then(|x| x.parse::().ok()) @@ -26,3 +26,9 @@ pub fn limits_from_env() -> (u64, i64) { debug!("memory_limit: {memory_limit}MB"); (cpu_limit, memory_limit) } + +pub fn builder_image_from_env() -> String { + env::var("BUILDER_IMAGE") + .ok() + .unwrap_or_else(|| DEFAULT_BUILDER_IMAGE.to_string()) +} diff --git a/backend/aurcache-builder/src/makepkg_utils.rs b/backend/aurcache-builder/src/makepkg_utils.rs index 762eee8e..33ac116e 100644 --- a/backend/aurcache-builder/src/makepkg_utils.rs +++ b/backend/aurcache-builder/src/makepkg_utils.rs @@ -1,11 +1,9 @@ -pub fn create_makepkg_config( - name: String, - build_dir_base: &str, -) -> anyhow::Result<(String, String)> { +pub fn create_makepkg_config(build_dir_base: &str) -> anyhow::Result<(String, String)> { let makepkg_config = format!( - "\ + " MAKEFLAGS=-j$(nproc) -PKGDEST={build_dir_base}/{name}" +PKGDEST={build_dir_base} + " ); let makepkg_config_path = "/var/ab/.config/pacman/makepkg.conf"; Ok((makepkg_config, makepkg_config_path.to_string())) diff --git a/backend/aurcache-builder/src/move_location.rs b/backend/aurcache-builder/src/move_location.rs index 568423fa..7cc58982 100644 --- a/backend/aurcache-builder/src/move_location.rs +++ b/backend/aurcache-builder/src/move_location.rs @@ -225,6 +225,9 @@ fn build_output_map( for a in archives { let a = a?; + if a.file_type()?.is_dir() { + continue; + } let name = a.file_name(); let name = name.to_str().ok_or_else(|| anyhow!("Invalid filename"))?; diff --git a/backend/aurcache-builder/src/path_utils.rs b/backend/aurcache-builder/src/path_utils.rs index e2ac2517..121f15d1 100644 --- a/backend/aurcache-builder/src/path_utils.rs +++ b/backend/aurcache-builder/src/path_utils.rs @@ -1,26 +1,13 @@ -use crate::build_mode::{BuildMode, get_build_mode}; +use std::env; use std::fs; use std::fs::Permissions; use std::os::unix::fs::PermissionsExt; use std::path::PathBuf; -/// create the build directory of the package to newly build (as path view of aurcache) -pub fn create_active_build_path(pkg_name: String) -> anyhow::Result { - let path = match get_build_mode() { - BuildMode::DinD(v) => { - let mut build_path = PathBuf::from(v.build_path); - build_path.push(pkg_name); - build_path - } - BuildMode::Host(v) => { - let mut build_path = PathBuf::from(v.build_artifact_dir_aurcache); - build_path.push(pkg_name); - build_path - } - }; - - fs::create_dir_all(path.clone())?; - fs::set_permissions(path.clone(), Permissions::from_mode(0o777))?; +pub fn create_active_build_path(pkg_name: &str) -> anyhow::Result { + let path = env::current_dir()?.join("builds").join(pkg_name); + fs::create_dir_all(&path)?; + fs::set_permissions(&path, Permissions::from_mode(0o777))?; Ok(path) } diff --git a/backend/aurcache-db/src/migration/m20251107_000000_build_flags_no_install.rs b/backend/aurcache-db/src/migration/m20251107_000000_build_flags_no_install.rs new file mode 100644 index 00000000..6bdcef96 --- /dev/null +++ b/backend/aurcache-db/src/migration/m20251107_000000_build_flags_no_install.rs @@ -0,0 +1,69 @@ +use crate::helpers::dbtype::database_type; +use sea_orm::DbBackend; +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let db = manager.get_connection(); + + match database_type() { + DbBackend::Sqlite => { + db.execute_unprepared( + r" +UPDATE packages +SET build_flags = REPLACE(build_flags, '-S', '-B') +WHERE build_flags LIKE '%-S%'; +", + ) + .await?; + } + DbBackend::Postgres => { + db.execute_unprepared( + r" +UPDATE public.packages +SET build_flags = REGEXP_REPLACE(build_flags, '-S', '-B', 'g') +WHERE build_flags ~ '-S'; +", + ) + .await?; + } + _ => Err(DbErr::Migration("Unsupported database type".to_string()))?, + } + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let db = manager.get_connection(); + + match database_type() { + DbBackend::Sqlite => { + db.execute_unprepared( + r" +UPDATE packages +SET build_flags = REPLACE(build_flags, '-B', '-S') +WHERE build_flags LIKE '%-B%'; +", + ) + .await?; + } + DbBackend::Postgres => { + db.execute_unprepared( + r" +UPDATE public.packages +SET build_flags = REGEXP_REPLACE(build_flags, '-B', '-S', 'g') +WHERE build_flags ~ '-B'; +", + ) + .await?; + } + _ => Err(DbErr::Migration("Unsupported database type".to_string()))?, + } + + Ok(()) + } +} diff --git a/backend/aurcache-db/src/migration/mod.rs b/backend/aurcache-db/src/migration/mod.rs index 4469c499..b79fb90c 100644 --- a/backend/aurcache-db/src/migration/mod.rs +++ b/backend/aurcache-db/src/migration/mod.rs @@ -5,6 +5,7 @@ mod m20240907_131839_platform_buildflags; mod m20250213_223900_activity_log; mod m20251015_230000_pkg_sources; mod m20251106_100000_build_version; +mod m20251107_000000_build_flags_no_install; pub struct Migrator; @@ -17,6 +18,7 @@ impl MigratorTrait for Migrator { Box::new(m20250213_223900_activity_log::Migration), Box::new(m20251106_100000_build_version::Migration), Box::new(m20251015_230000_pkg_sources::Migration), + Box::new(m20251107_000000_build_flags_no_install::Migration), ] } } diff --git a/backend/aurcache-utils/src/package/add.rs b/backend/aurcache-utils/src/package/add.rs index a03fa1c0..8c71c380 100644 --- a/backend/aurcache-utils/src/package/add.rs +++ b/backend/aurcache-utils/src/package/add.rs @@ -58,7 +58,7 @@ pub async fn package_add( let build_flags = build_flags.unwrap_or_else(|| { vec![ - "-Syu".to_string(), + "-B".to_string(), "--noconfirm".to_string(), "--noprogressbar".to_string(), "--color never".to_string(), diff --git a/docker-compose.e2e.yaml b/docker-compose.e2e.yaml new file mode 100644 index 00000000..446197fb --- /dev/null +++ b/docker-compose.e2e.yaml @@ -0,0 +1,36 @@ +services: + registry: + image: registry:2 + ports: + - "5000:5000" + environment: + - REGISTRY_HTTP_ADDR=0.0.0.0:5000 + networks: + aurcache_network: + + aurcache: + build: + context: . + dockerfile: docker/Dockerfile + ports: + - "${AURCACHE_PORT:-8080}:8080" + - "${AURCACHE_MIRROR_PORT:-8081}:8081" + environment: + - LOG_LEVEL=debug + - BUILDER_IMAGE=localhost:5000/aurcache-builder:test + - BUILD_ARTIFACT_DIR=${TEMP_DIR}/builds + volumes: + - ${TEMP_DIR}/db:/app/db + - ${TEMP_DIR}/repo:/app/repo + - ${TEMP_DIR}/builds:/app/builds + - ${TEMP_DIR}/config:/app/config + - /var/run/docker.sock:/var/run/docker.sock + networks: + aurcache_network: + depends_on: + - registry + + +networks: + aurcache_network: + driver: bridge diff --git a/docker/add-aur.sh b/docker/add-aur.sh index 84a2e805..70296b87 100644 --- a/docker/add-aur.sh +++ b/docker/add-aur.sh @@ -13,8 +13,11 @@ if [ "$TARGETPLATFORM" = "linux/amd64" ]; then cp /etc/pacman.conf.amd64 /etc/pacman.conf fi +# fix landlock errors in container builds +sed -i '/^\[options\]/a DisableSandbox' /etc/pacman.conf + # we're gonna need sudo to use the helper properly -pacman -Syy --noconfirm +pacman -Syyu --noconfirm pacman --sync --needed --noconfirm --noprogressbar pacman-contrib # repopulate keychain @@ -34,6 +37,7 @@ rankmirrors -n 10 /etc/pacman.d/mirrorlist.backup > /etc/pacman.d/mirrorlist rm /etc/pacman.d/mirrorlist.backup pacman --sync --needed --noconfirm --noprogressbar sudo base-devel git rust || echo "Nothing to do" +git config --global --add safe.directory '*' # create the user AUR_USER_HOME="/var/${AUR_USER}" @@ -60,7 +64,7 @@ sudo -u ${AUR_USER} -D~ bash -c 'echo MAKEFLAGS="-j\$(nproc)" > .config/pacman/m #sudo -u ${AUR_USER} -D~ bash -c 'echo PKGEXT=".pkg.tar" >> .config/pacman/makepkg.conf' # setup storage for AUR packages built -NEW_PKGDEST="/var/cache/makepkg/pkg" +NEW_PKGDEST="/build" NPDP=$(dirname "${NEW_PKGDEST}") mkdir -p "${NPDP}" install -o "${AUR_USER}" -d "${NEW_PKGDEST}" @@ -72,37 +76,12 @@ FPP=$(dirname "${FOREIGN_PKG}") mkdir -p "${FPP}" install -o "${AUR_USER}" -d "${FOREIGN_PKG}" -# arm build doesn't work for some reason: use -bin versrion -if [ "${TARGETARCH}" = "arm" ]; then - HELPER_PKG="paru-bin" -else - HELPER_PKG="paru" -fi - -# get helper pkgbuild -#sudo -u "${AUR_USER}" -D~ bash -c "git clone https://aur.archlinux.org/paru-bin.git" -# use paru instead of paru-bin until their alpm dependency problem is solved -sudo -u "${AUR_USER}" -D~ bash -c "git clone https://aur.archlinux.org/${HELPER_PKG}.git" - -# ---- PATCH FOR RISCV64 ---- -if [ "${TARGETARCH}" = "riscv64" ]; then - echo "Patching paru PKGBUILD to allow riscv64" - sudo -u "${AUR_USER}" -D~ bash -c " - cd ${HELPER_PKG} - sed -i 's/^arch=(/arch=(\"riscv64\" /' PKGBUILD - " - - # Allow installing unsigned local packages (must be root) - grep -q '^LocalFileSigLevel' /etc/pacman.conf || \ - echo 'LocalFileSigLevel = Optional' >> /etc/pacman.conf -fi -# --------------------------- - -# make helper -sudo -u "${AUR_USER}" -D~//${HELPER_PKG} bash -c "makepkg -s --noprogressbar --noconfirm --needed" - -# install helper -pacman --upgrade --needed --noconfirm --noprogressbar "${NEW_PKGDEST}"/*.pkg.* +# Prepare paru helper. +# Currently builds from source, from a fork with some fixes. +# Eventually, we could build from upstream, from crates.io, or even install the paru package itself. +sudo -u "${AUR_USER}" bash -c "cargo install --git https://github.com/gyscos/paru" +cp "${AUR_USER_HOME}/.cargo/bin/paru" /usr/local/bin/ +chmod 755 /usr/local/bin/paru # Remove all pacman caches pacman -Scc --noconfirm || echo "Pacman cache already clean" @@ -115,11 +94,10 @@ pacman -Rns --noconfirm rust || echo "Build dependencies already removed" # cleanup sudo rm -rf "${NEW_PKGDEST}"/* -rm -rf "${AUR_USER_HOME}/${HELPER_PKG}" rm -rf "${AUR_USER_HOME}/.cache/go-build" rm -rf "${AUR_USER_HOME}/.cargo" rm -rf /tmp/* rm -rf /root/.cargo /usr/share/cargo || true rm -rf /var/tmp/* /var/cache/* || true -echo "Cleanup complete" \ No newline at end of file +echo "Cleanup complete" diff --git a/docker/builder.Dockerfile b/docker/builder.Dockerfile index 016873dc..781764a9 100644 --- a/docker/builder.Dockerfile +++ b/docker/builder.Dockerfile @@ -21,4 +21,5 @@ ENV TARGETPLATFORM=${TARGETPLATFORM} ########## Files ########## ADD docker/add-aur.sh /root ADD docker/pacman.conf.amd64 /etc/pacman.conf.amd64 -RUN bash /root/add-aur.sh ab paru \ No newline at end of file +RUN /bin/bash /root/add-aur.sh ab paru +USER ab \ No newline at end of file diff --git a/docker/pacman.conf.amd64 b/docker/pacman.conf.amd64 index 19435bbc..4b916755 100644 --- a/docker/pacman.conf.amd64 +++ b/docker/pacman.conf.amd64 @@ -30,8 +30,6 @@ NoProgressBar CheckSpace VerbosePkgLists ParallelDownloads = 3 -# fix landlock errors -DisableSandbox # By default, pacman accepts packages signed by keys that its local keyring # trusts (see pacman-key and its man page), as well as unsigned packages. diff --git a/scripts/test-builder.sh b/scripts/test-builder.sh new file mode 100755 index 00000000..1e6e827b --- /dev/null +++ b/scripts/test-builder.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -e + +PACKAGE="${1:-hello}" +BUILDER_IMAGE="${2:-aurcache-builder:test}" +BUILD_FLAGS="${3--B --noconfirm --noprogressbar --color never}" + +docker build -f docker/builder.Dockerfile -t $BUILDER_IMAGE . + +TEMP_DIR=$(mktemp -d) +BUILD_DIR="$TEMP_DIR/test_builds" +MAKEPKG_CONF="/var/ab/.config/pacman/makepkg.conf" + +cleanup() { + rm -rf "$BUILD_DIR" 2>/dev/null || true +} +trap cleanup EXIT + +mkdir -p "$BUILD_DIR" +chmod 777 "$BUILD_DIR" + +echo "=== Testing builder image: $BUILDER_IMAGE ===" +echo "Building package: $PACKAGE" +echo "Build flags: $BUILD_FLAGS" + +docker run --rm \ + -v "$BUILD_DIR:/build" \ + --user ab \ + "$BUILDER_IMAGE" sh -c " + cd /build + + # Write makepkg config (same as aurcache) + cat > $MAKEPKG_CONF << EOF +MAKEFLAGS=-j\$(nproc) +PKGDEST=/build +EOF + + # Run exact command that aurcache runs: + # 1. Self-update + # 2. Download PKGBUILD + # 3. Build package + paru -Syu --noconfirm --noprogressbar --color never + paru -G $PACKAGE + paru $BUILD_FLAGS * + " + +echo "=== Checking built package ===" +PKGFILE=$(ls -1 "$BUILD_DIR"/*.pkg.tar.* 2>/dev/null | head -1) +if [ -z "$PKGFILE" ]; then + echo "ERROR: No package file found in $BUILD_DIR" + ls -la "$BUILD_DIR" + exit 1 +fi + +echo "Found package: $PKGFILE" + +if tar -tf "$PKGFILE" > /dev/null 2>&1; then + echo "Archive is valid" +else + echo "ERROR: Invalid archive file" + exit 1 +fi + +TEMP_PARENT=$(dirname "$TEMP_DIR") +docker run --rm -v "$TEMP_PARENT:$TEMP_PARENT" archlinux bash -c " rm -rf '$TEMP_DIR' " +echo "=== Builder test complete ===" diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh new file mode 100755 index 00000000..36dd7673 --- /dev/null +++ b/scripts/test-e2e.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${1?Usage: $0 [port] [timeout]}" +PACKAGE="$1" +export AURCACHE_PORT="${2:-8080}" +export AURCACHE_MIRROR_PORT=$((AURCACHE_PORT + 1)) +BUILD_TIMEOUT="${3:-300}" + +# We take security very seriously +AUTH_HEADER="Authorization: Basic $(echo -n 'admin:secret' | base64)" + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +COMPOSE_FILE="$PROJECT_DIR/docker-compose.e2e.yaml" + +# A clean slate for each new test. +export TEMP_DIR=$(mktemp -d) +echo "Using temp dir $TEMP_DIR" + +# These are mounted by docker-compose +BUILD_DIR="$TEMP_DIR/builds" + +# These will be picked up by docker-compose + +# ============================================================================= +# Helper Functions +# ============================================================================= + + +curl_api() { + local path="$1" + shift + curl -s "http://localhost:$AURCACHE_PORT$path" \ + -H "$AUTH_HEADER" \ + -H "Content-Type: application/json" \ + "$@" +} + +wait_for_service() { + echo "=== Waiting for AURCache to be ready ===" + local max_attempts=30 + local delay=2 + + for i in $(seq 1 "$max_attempts"); do + if curl -s "http://localhost:$AURCACHE_PORT/api" > /dev/null 2>&1; then + echo " AURCache is ready" + return 0 + fi + if [ "$i" -eq "$max_attempts" ]; then + return 1 + fi + sleep "$delay" + done +} + +dc() { + docker compose -f docker-compose.e2e.yaml "$@" +} + +# ============================================================================= +# Setup Functions +# ============================================================================= + +setup_directories() { + mkdir -p "$BUILD_DIR"/{builds,repo,db,downloads,config/pacman_x86_64} + chmod 777 "$BUILD_DIR"/{builds,repo,db,downloads} + + # The build config expects mirrorlist at BUILD_DIR/config/pacman_x86_64/mirrorlist + echo "Server = https://mirror.rackspace.com/archlinux/\$repo/os/\$arch" > "$BUILD_DIR/config/pacman_x86_64/mirrorlist" +} + +cleanup() { + if [ "${CLEANUP:-1}" = "1" ]; then + echo "=== Cleaning up ===" + dc down --remove-orphans -t 10 2>/dev/null || true + # Note: some of the files there were written by root in a docker container. + # So we're not legally allowed to touch them. But we can use the same docker trick to do that. + # We need to mount TEMP_DIR's parent to properly remove the folder itself. + TEMP_PARENT=$(dirname "$TEMP_DIR") + docker run --rm -v "$TEMP_PARENT:$TEMP_PARENT" archlinux bash -c " rm -rf '$TEMP_DIR' " + else + echo "=== Skipping cleanup (CLEANUP=0) ===" + fi +} + +start_docker_services() { + echo "=== Starting Docker services ===" + dc up -d registry + sleep 2 + + echo "=== Building and pushing builder image ===" + docker build -q -t localhost:5000/aurcache-builder:test -f docker/builder.Dockerfile --push . + + echo "=== Building and starting AURCache ===" + dc build -q aurcache && dc up -d aurcache +} + +configure_aurcache_registry() { + echo "=== Configuring AURCache registry ===" + echo '[[registry]] +prefix = "localhost" +location = "localhost" +insecure = true' | docker exec -i aurcache-aurcache-1 bash -c "cat > /etc/containers/registries.conf.d/localhost.conf" +} + +prepare() { + start_docker_services + + wait_for_service || { dc logs; exit 1; } + + configure_aurcache_registry +} + +# ============================================================================= +# Build trigger function +# ============================================================================= + +request_package() { + echo "=== Adding package: $PACKAGE ===" + # We're starting from a fresh DB every time, so we know it'll be a new package. + # If we reused the DB test after test we'd need to delete the package before adding it again. + RESPONSE=$(curl_api "/api/package" -X POST -d "{\"source\": {\"type\": \"aur\", \"name\": \"$PACKAGE\"}, \"platforms\": [\"x86_64\"]}") + + # Hofstadter's law: It always takes longer than you expect, even when you take into account Hofstadter's law. + echo "=== Waiting for build to complete (timeout: ${BUILD_TIMEOUT}s) ===" + local START_TIME + START_TIME=$(date +%s) + while true; do + local ELAPSED + ELAPSED=$(($(date +%s) - START_TIME)) + if [ $ELAPSED -gt "$BUILD_TIMEOUT" ]; then + echo "ERROR: Build timed out after ${BUILD_TIMEOUT}s" + # Show what we can to understand what went wrong. + dc logs + exit 1 + fi + + local RESPONSE + RESPONSE=$(curl_api "/api/packages/list?limit=100") + local BUILD_STATUS + BUILD_STATUS=$(echo "$RESPONSE" | jq -r ".[] | select(.name == \"$PACKAGE\") | .status" 2>/dev/null || echo "not_found") + + echo " Build status: $BUILD_STATUS (elapsed: ${ELAPSED}s)" + + case "$BUILD_STATUS" in + 1) echo " Build completed successfully"; break ;; + 2) echo "ERROR: Build failed"; dc logs; exit 1 ;; + null|"") echo "Package not found yet"; sleep 5 ;; + *) sleep 5 ;; + esac + done +} + +# ============================================================================= +# Validation Functions +# ============================================================================= + +validate() { + echo "=== Validating built package ===" + + # Try to install the package just like a user would. + docker run --rm \ + --network aurcache_aurcache_network \ + archlinux:latest \ + sh -e -c ' + # First setup the repo we want to test + cat >> /etc/pacman.conf << EOF +[repo] +SigLevel = Optional TrustAll +Server = http://aurcache-aurcache-1:'${AURCACHE_MIRROR_PORT}'/\$arch +EOF + + echo "Updating test container" + ( + # Just making sure we are up-to-date + pacman-key --init + pacman-key --populate archlinux + # Need to install this first so we can validate other updates + pacman -Syq archlinux-keyring --noconfirm + pacman -Suq --noconfirm + ) 2>/dev/null >/dev/null + + echo "Installing package" + pacman -S --noconfirm '$PACKAGE' + pacman -Qi '$PACKAGE' + ' + + echo "=== End-to-end test complete ===" +} + +# ============================================================================= +# Main +# ============================================================================= + +trap cleanup EXIT + +setup_directories +prepare +request_package +validate