From ed8d85a777fec633a065b2cf91e086378666edac Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 25 Jun 2026 14:35:25 +0200 Subject: [PATCH 01/25] Create bootc-image-builder directory in the repo root Create a new top-level directory called bootc-image-builder where we will put all data and configuration files related to the tool when possible. Move the Containerfile for bootc-image-builder there and rename it. --- .../Containerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Containerfile.bootc-image-builder => bootc-image-builder/Containerfile (100%) diff --git a/Containerfile.bootc-image-builder b/bootc-image-builder/Containerfile similarity index 100% rename from Containerfile.bootc-image-builder rename to bootc-image-builder/Containerfile From 56c0531de3b12f3ffbd4a60504d10a6bbeed0c8d Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 25 Jun 2026 14:22:55 +0200 Subject: [PATCH 02/25] Move bootc-image-builder tests under the new directory Move the bootc-image-builder tests under the new top-level directory for the tool and rename the files to enable the tests. --- test/bib/_conftest.py => bootc-image-builder/test/conftest.py | 0 .../test/containerbuild.py | 0 .../test/requirements.txt | 0 .../test/test_build_cross.py | 0 .../test/test_build_disk.py | 0 .../test/test_build_iso.py | 0 .../test/test_build_pxe.py | 0 .../_test_flake8.py => bootc-image-builder/test/test_flake8.py | 0 .../test/test_manifest.py | 0 test/bib/_test_opts.py => bootc-image-builder/test/test_opts.py | 0 .../test/test_progress.py | 0 .../_test_pylint.py => bootc-image-builder/test/test_pylint.py | 0 test/bib/_testcases.py => bootc-image-builder/test/testcases.py | 0 test/bib/_testutil.py => bootc-image-builder/test/testutil.py | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename test/bib/_conftest.py => bootc-image-builder/test/conftest.py (100%) rename test/bib/_containerbuild.py => bootc-image-builder/test/containerbuild.py (100%) rename test/bib/_requirements.txt => bootc-image-builder/test/requirements.txt (100%) rename test/bib/_test_build_cross.py => bootc-image-builder/test/test_build_cross.py (100%) rename test/bib/_test_build_disk.py => bootc-image-builder/test/test_build_disk.py (100%) rename test/bib/_test_build_iso.py => bootc-image-builder/test/test_build_iso.py (100%) rename test/bib/_test_build_pxe.py => bootc-image-builder/test/test_build_pxe.py (100%) rename test/bib/_test_flake8.py => bootc-image-builder/test/test_flake8.py (100%) rename test/bib/_test_manifest.py => bootc-image-builder/test/test_manifest.py (100%) rename test/bib/_test_opts.py => bootc-image-builder/test/test_opts.py (100%) rename test/bib/_test_progress.py => bootc-image-builder/test/test_progress.py (100%) rename test/bib/_test_pylint.py => bootc-image-builder/test/test_pylint.py (100%) rename test/bib/_testcases.py => bootc-image-builder/test/testcases.py (100%) rename test/bib/_testutil.py => bootc-image-builder/test/testutil.py (100%) diff --git a/test/bib/_conftest.py b/bootc-image-builder/test/conftest.py similarity index 100% rename from test/bib/_conftest.py rename to bootc-image-builder/test/conftest.py diff --git a/test/bib/_containerbuild.py b/bootc-image-builder/test/containerbuild.py similarity index 100% rename from test/bib/_containerbuild.py rename to bootc-image-builder/test/containerbuild.py diff --git a/test/bib/_requirements.txt b/bootc-image-builder/test/requirements.txt similarity index 100% rename from test/bib/_requirements.txt rename to bootc-image-builder/test/requirements.txt diff --git a/test/bib/_test_build_cross.py b/bootc-image-builder/test/test_build_cross.py similarity index 100% rename from test/bib/_test_build_cross.py rename to bootc-image-builder/test/test_build_cross.py diff --git a/test/bib/_test_build_disk.py b/bootc-image-builder/test/test_build_disk.py similarity index 100% rename from test/bib/_test_build_disk.py rename to bootc-image-builder/test/test_build_disk.py diff --git a/test/bib/_test_build_iso.py b/bootc-image-builder/test/test_build_iso.py similarity index 100% rename from test/bib/_test_build_iso.py rename to bootc-image-builder/test/test_build_iso.py diff --git a/test/bib/_test_build_pxe.py b/bootc-image-builder/test/test_build_pxe.py similarity index 100% rename from test/bib/_test_build_pxe.py rename to bootc-image-builder/test/test_build_pxe.py diff --git a/test/bib/_test_flake8.py b/bootc-image-builder/test/test_flake8.py similarity index 100% rename from test/bib/_test_flake8.py rename to bootc-image-builder/test/test_flake8.py diff --git a/test/bib/_test_manifest.py b/bootc-image-builder/test/test_manifest.py similarity index 100% rename from test/bib/_test_manifest.py rename to bootc-image-builder/test/test_manifest.py diff --git a/test/bib/_test_opts.py b/bootc-image-builder/test/test_opts.py similarity index 100% rename from test/bib/_test_opts.py rename to bootc-image-builder/test/test_opts.py diff --git a/test/bib/_test_progress.py b/bootc-image-builder/test/test_progress.py similarity index 100% rename from test/bib/_test_progress.py rename to bootc-image-builder/test/test_progress.py diff --git a/test/bib/_test_pylint.py b/bootc-image-builder/test/test_pylint.py similarity index 100% rename from test/bib/_test_pylint.py rename to bootc-image-builder/test/test_pylint.py diff --git a/test/bib/_testcases.py b/bootc-image-builder/test/testcases.py similarity index 100% rename from test/bib/_testcases.py rename to bootc-image-builder/test/testcases.py diff --git a/test/bib/_testutil.py b/bootc-image-builder/test/testutil.py similarity index 100% rename from test/bib/_testutil.py rename to bootc-image-builder/test/testutil.py From f5cf8bf4d110c68f0cd351bbdb9c2e60257ba68e Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 25 Jun 2026 14:24:33 +0200 Subject: [PATCH 03/25] test/install-dependencies: add new test dependencies Add new test dependencies, pylint and flake8, required by the bib tests. The packages aren't available on CentOS and RHEL, but we don't run them there. Sort the package lists. Sorted lists are nicer. --- test/scripts/install-dependencies | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/scripts/install-dependencies b/test/scripts/install-dependencies index 178f53e7b3..604f7e05ff 100755 --- a/test/scripts/install-dependencies +++ b/test/scripts/install-dependencies @@ -10,11 +10,12 @@ install_deps() { btrfs-progs-devel \ cloud-utils \ device-mapper-devel \ - libvirt-devel \ + flake8 \ gcc \ go \ gpgme-devel \ krb5-devel \ + libvirt-devel \ lorax \ osbuild \ osbuild-depsolve-dnf \ @@ -23,25 +24,26 @@ install_deps() { osbuild-ostree \ osbuild-selinux \ podman \ + pylint \ python3 \ + python3-boto3 \ python3-pip \ python3-pytest \ - python3-boto3 \ python3-qemu-qmp \ python3-scp \ - yamllint \ - xz + xz \ + yamllint } install_deps_el() { dnf -y install \ awscli2 \ device-mapper-devel \ - libvirt-devel \ gcc \ go \ gpgme-devel \ krb5-devel \ + libvirt-devel \ lorax \ osbuild \ osbuild-depsolve-dnf \ From f191e2bf033ca78026c9b8509fb8f8527b619cab Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 10:42:50 +0200 Subject: [PATCH 04/25] bootc-image-builder/test: update container build fixtures Update the Containerfile path for the bootc-image-builder container fixture to the new location. Add cleanups to the container fixtures. Delete copies of the fixtures from test/conftest.py. These were copied from bootc-image-builder into image-builder-cli but now we have the bootc-image-builder tests entirely in this repository so they were duplicates. --- bootc-image-builder/test/containerbuild.py | 12 +++-- test/conftest.py | 56 ---------------------- 2 files changed, 8 insertions(+), 60 deletions(-) diff --git a/bootc-image-builder/test/containerbuild.py b/bootc-image-builder/test/containerbuild.py index 02d5d1d209..7b818af2b1 100644 --- a/bootc-image-builder/test/containerbuild.py +++ b/bootc-image-builder/test/containerbuild.py @@ -43,10 +43,12 @@ def build_container_fixture(): subprocess.check_call([ "podman", "build", "--cache-ttl=1h", - "-f", "Containerfile.bib", + "-f", "bootc-image-builder/Containerfile", "-t", container_tag, + ".", ]) - return container_tag + yield container_tag + subprocess.check_call(["podman", "rmi", container_tag]) @pytest.fixture(name="build_fake_container", scope="session") @@ -98,7 +100,8 @@ def build_fake_container_fixture(tmpdir_factory, build_container): "-t", container_tag, tmp_path, ]) - return container_tag + yield container_tag + subprocess.check_call(["podman", "rmi", container_tag]) @pytest.fixture(name="build_erroring_container", scope="session") @@ -145,4 +148,5 @@ def build_erroring_container_fixture(tmpdir_factory, build_container): "-t", container_tag, tmp_path, ]) - return container_tag + yield container_tag + subprocess.check_call(["podman", "rmi", container_tag]) diff --git a/test/conftest.py b/test/conftest.py index 4b1c84d5ca..3335d15542 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -10,59 +10,3 @@ def pytest_configure(config): config.addinivalue_line( "markers", "images_integration" ) - - -# XXX: copied from bib -@pytest.fixture(name="build_container", scope="session") -def build_container_fixture(): - """Build a container from the Containerfile and returns the name""" - - container_tag = "ibcli-test-" + "".join(random.choices( - string.ascii_lowercase + string.digits, k=4)) - - subprocess.check_call([ - "podman", "build", - "-f", "Containerfile", - "-t", container_tag, - ]) - yield container_tag - subprocess.check_call(["podman", "rmi", container_tag]) - - -# XXX: copied from bib -@pytest.fixture(name="build_fake_container", scope="session") -def build_fake_container_fixture(tmpdir_factory, build_container): - """Build a container with a fake osbuild and returns the name""" - tmp_path = tmpdir_factory.mktemp("build-fake-container") - - fake_osbuild_path = tmp_path / "fake-osbuild" - fake_osbuild_path.write_text(textwrap.dedent("""\ - #!/bin/bash -e - - # injest generated manifest from the images library, if we do not - # do this images may fail with "broken" pipe errors - cat - >/dev/null - - echo "osbuild-stdout-output" - mkdir -p /output/qcow2 - echo "fake-disk.qcow2" > /output/qcow2/disk.qcow2 - - """), encoding="utf8") - - cntf_path = tmp_path / "Containerfile" - - cntf_path.write_text(textwrap.dedent(f"""\n - FROM {build_container} - COPY fake-osbuild /usr/bin/osbuild - RUN chmod 755 /usr/bin/osbuild - """), encoding="utf8") - - container_tag = "ibcli-test-faked-osbuild-" + "".join(random.choices( - string.ascii_lowercase + string.digits, k=4)) - subprocess.check_call([ - "podman", "build", - "-t", container_tag, - tmp_path, - ]) - yield container_tag - subprocess.check_call(["podman", "rmi", container_tag]) From 2805eb9c68fdbcab506f1c006389030a1a7a72c5 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 10:47:24 +0200 Subject: [PATCH 05/25] bootc-image-builder/test: isort all test scripts Codestyle fixes: Run isort over all files in bootc-image-builder/test. --- bootc-image-builder/test/conftest.py | 1 - bootc-image-builder/test/test_build_cross.py | 13 +++--------- bootc-image-builder/test/test_build_disk.py | 11 +++++----- bootc-image-builder/test/test_build_iso.py | 21 ++++++++------------ bootc-image-builder/test/test_build_pxe.py | 10 +++++----- bootc-image-builder/test/test_manifest.py | 1 - bootc-image-builder/test/test_opts.py | 3 ++- bootc-image-builder/test/test_progress.py | 9 +++------ 8 files changed, 27 insertions(+), 42 deletions(-) diff --git a/bootc-image-builder/test/conftest.py b/bootc-image-builder/test/conftest.py index 53285426fb..28e3094351 100644 --- a/bootc-image-builder/test/conftest.py +++ b/bootc-image-builder/test/conftest.py @@ -1,5 +1,4 @@ import pytest - # pylint: disable=wrong-import-order from testcases import TestCase from vmtest.util import get_free_port diff --git a/bootc-image-builder/test/test_build_cross.py b/bootc-image-builder/test/test_build_cross.py index 12b89eebd7..8b43e8c6dd 100644 --- a/bootc-image-builder/test/test_build_cross.py +++ b/bootc-image-builder/test/test_build_cross.py @@ -1,17 +1,10 @@ import platform import pytest - -from testcases import gen_testcases - from test_build_disk import ( # pylint: disable=unused-import - assert_disk_image_boots, - build_container_fixture, - gpg_conf_fixture, - image_type_fixture, - registry_conf_fixture, - shared_tmpdir_fixture, -) + assert_disk_image_boots, build_container_fixture, gpg_conf_fixture, + image_type_fixture, registry_conf_fixture, shared_tmpdir_fixture) +from testcases import gen_testcases # This testcase is not part of "test_build_disk.py:test_image_boots" diff --git a/bootc-image-builder/test/test_build_disk.py b/bootc-image-builder/test/test_build_disk.py index 9aaadcd49d..67eece1fc8 100644 --- a/bootc-image-builder/test/test_build_disk.py +++ b/bootc-image-builder/test/test_build_disk.py @@ -9,17 +9,18 @@ import subprocess import tempfile import uuid -from contextlib import contextmanager, ExitStack -from typing import NamedTuple +from contextlib import ExitStack, contextmanager from dataclasses import dataclass +from typing import NamedTuple import pytest # local test utils import testutil -from containerbuild import build_container_fixture # pylint: disable=unused-import -from testcases import CLOUD_BOOT_IMAGE_TYPES, DISK_IMAGE_TYPES, gen_testcases import vmtest.util -from vmtest.vm import AWS_REGION, AWS, QEMU +from containerbuild import \ + build_container_fixture # pylint: disable=unused-import +from testcases import CLOUD_BOOT_IMAGE_TYPES, DISK_IMAGE_TYPES, gen_testcases +from vmtest.vm import AWS, AWS_REGION, QEMU if not testutil.has_executable("podman"): pytest.skip("no podman, skipping integration tests that required podman", allow_module_level=True) diff --git a/bootc-image-builder/test/test_build_iso.py b/bootc-image-builder/test/test_build_iso.py index c2f3998cd3..0bddc3ef86 100644 --- a/bootc-image-builder/test/test_build_iso.py +++ b/bootc-image-builder/test/test_build_iso.py @@ -1,8 +1,8 @@ -import os -import random import json +import os import pathlib import platform +import random import string import subprocess import textwrap @@ -11,18 +11,13 @@ import pytest # local test utils import testutil -from containerbuild import build_container_fixture, make_container # pylint: disable=unused-import +from containerbuild import ( # pylint: disable=unused-import + build_container_fixture, make_container) +from test_build_disk import ImageBuildResult # pylint: disable=unused-import +from test_build_disk import (assert_kernel_args, gpg_conf_fixture, + image_type_fixture, registry_conf_fixture, + shared_tmpdir_fixture) from testcases import gen_testcases -from test_build_disk import ( - assert_kernel_args, - ImageBuildResult, -) -from test_build_disk import ( # pylint: disable=unused-import - gpg_conf_fixture, - image_type_fixture, - registry_conf_fixture, - shared_tmpdir_fixture, -) from vmtest.vm import QEMU ISO_BOOT_TIMEOUT = 1800 diff --git a/bootc-image-builder/test/test_build_pxe.py b/bootc-image-builder/test/test_build_pxe.py index 7fba0ae341..13731a8a4a 100644 --- a/bootc-image-builder/test/test_build_pxe.py +++ b/bootc-image-builder/test/test_build_pxe.py @@ -1,21 +1,21 @@ -import os -import random import json +import os import pathlib import platform +import random import string import subprocess import textwrap import threading - from contextlib import ExitStack -from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from tempfile import TemporaryDirectory import pytest # local test utils import testutil -from containerbuild import build_container_fixture, make_container # pylint: disable=unused-import +from containerbuild import ( # pylint: disable=unused-import + build_container_fixture, make_container) from vmtest.vm import QEMU diff --git a/bootc-image-builder/test/test_manifest.py b/bootc-image-builder/test/test_manifest.py index fee8acaa06..f75a7aa7e0 100644 --- a/bootc-image-builder/test/test_manifest.py +++ b/bootc-image-builder/test/test_manifest.py @@ -9,7 +9,6 @@ import textwrap import pytest - import testutil from containerbuild import build_container_fixture as _ from containerbuild import make_container diff --git a/bootc-image-builder/test/test_opts.py b/bootc-image-builder/test/test_opts.py index c12a8a7631..bd5e1311d2 100644 --- a/bootc-image-builder/test/test_opts.py +++ b/bootc-image-builder/test/test_opts.py @@ -5,7 +5,8 @@ import pytest import testutil # pylint: disable=unused-import -from containerbuild import build_container_fixture, build_fake_container_fixture +from containerbuild import (build_container_fixture, + build_fake_container_fixture) @pytest.fixture(name="container_storage", scope="session") diff --git a/bootc-image-builder/test/test_progress.py b/bootc-image-builder/test/test_progress.py index 559f5026e4..383c743096 100644 --- a/bootc-image-builder/test/test_progress.py +++ b/bootc-image-builder/test/test_progress.py @@ -1,15 +1,12 @@ import subprocess import pytest - import testutil +from containerbuild import (build_container_fixture, + build_erroring_container_fixture, + build_fake_container_fixture) # pylint: disable=unused-import,duplicate-code from test_opts import container_storage_fixture -from containerbuild import ( - build_container_fixture, - build_erroring_container_fixture, - build_fake_container_fixture, -) def test_progress_debug(tmp_path, build_fake_container): From b1c3f2f3e1d110db04e5e07876a7abaae569ec63 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 10:51:48 +0200 Subject: [PATCH 06/25] bootc-image-builder: copy README from old repository Copy the old bootc-image-builder README into the bootc-image-builder subdirectory. This will still be useful for as long as we produce the container. We might copy it to our online docs (osbuild.org) and also use it later to provide instructions for migrating builds to image-builder itself. --- bootc-image-builder/README.md | 666 ++++++++++++++++++++++++++++++++++ 1 file changed, 666 insertions(+) create mode 100644 bootc-image-builder/README.md diff --git a/bootc-image-builder/README.md b/bootc-image-builder/README.md new file mode 100644 index 0000000000..e7854c1cbc --- /dev/null +++ b/bootc-image-builder/README.md @@ -0,0 +1,666 @@ +# bootc-image-builder + +A container to create disk images from [bootc](https://github.com/containers/bootc) container inputs, +especially oriented towards [Fedora/CentOS bootc](https://docs.fedoraproject.org/en-US/bootc/) or +derivatives. + +## πŸ”¨ Installation + +Have [podman](https://podman.io/) installed on your system. Either through your systems package manager if you're on +Linux or through [Podman Desktop](https://podman.io/) if you are on macOS or Windows. If you want to run the resulting +virtual machine(s) or installer media you can use [qemu](https://www.qemu.org/). + +A very nice GUI extension for Podman Desktop is also +[available](https://github.com/containers/podman-desktop-extension-bootc). +The command line examples below can be all handled by +Podman Desktop. + +On macOS, the podman machine must be running in rootful mode: + +```bash +$ podman machine stop # if already running +Waiting for VM to exit... +Machine "podman-machine-default" stopped successfully +$ podman machine set --rootful +$ podman machine start +``` + +## ✍ Prerequisites + +If you are on a system with SELinux enforced: The package `osbuild-selinux` or equivalent osbuild SELinux policies must be installed in the system running +`bootc-image-builder`. + +## πŸš€ Examples + +The following example builds a `centos-bootc:stream9` bootable container into a QCOW2 image for the architecture you're running +the command on. However, be sure to see the [upstream documentation](https://docs.fedoraproject.org/en-US/bootc/) +for more general information! Note that outside of initial experimentation, it's recommended to build a *derived* container image +(or reuse a derived image built via someone else) and then use this project to make a disk image from your custom image. + +The generic base images do not include a default user. This example injects a [user configuration file](#-build-config) +by adding a volume-mount for the local file to the bootc-image-builder container. + +The following command will create a QCOW2 disk image. First, create `./config.toml` as described above to configure user access. + +```bash +# Ensure the image is fetched +sudo podman pull quay.io/centos-bootc/centos-bootc:stream9 +mkdir output +sudo podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v ./config.toml:/config.toml:ro \ + -v ./output:/output \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + quay.io/centos-bootc/bootc-image-builder:latest \ + --type qcow2 \ + --use-librepo=True \ + quay.io/centos-bootc/centos-bootc:stream9 +``` + +Note that some images (like fedora) do not have a default root +filesystem type. In this case adds the switch `--rootfs `, +e.g. `--rootfs btrfs`. + +### Rootless + +There is *experimental* support for rootless builds in `bootc-image-builder`. To perform a rootless build KVM is used. The above example can be tried like so: + +```bash +# Ensure the image is fetched +podman pull quay.io/fedora/fedora-bootc:latest +mkdir output +podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v ./config.toml:/config.toml:ro \ + -v ./output:/output \ + -v ~/.local/share/containers/storage:/var/lib/containers/storage \ + quay.io/centos-bootc/bootc-image-builder:latest \ + --in-vm \ + --type qcow2 \ + --use-librepo=True \ + --rootfs ext4 \ + quay.io/fedora/fedora-bootc:latest +``` + +Note the mounting of the users container storage, addition of the `--in-vm` argument and the removal of `sudo` in the commands. + +### Running the resulting QCOW2 file on Linux (x86_64) + +A virtual machine can be launched using `qemu-system-x86_64` or with `virt-install` as shown below; +however there is more information about virtualization and other +choices in the [Fedora/CentOS bootc documentation](https://docs.fedoraproject.org/en-US/bootc/). + +#### qemu-system-x86_64 + +```bash +qemu-system-x86_64 \ + -M accel=kvm \ + -cpu host \ + -smp 2 \ + -m 4096 \ + -bios /usr/share/OVMF/OVMF_CODE.fd \ + -serial stdio \ + -snapshot output/qcow2/disk.qcow2 +``` + +#### virt-install + +```bash +sudo virt-install \ + --name fedora-bootc \ + --cpu host \ + --vcpus 4 \ + --memory 4096 \ + --import --disk ./output/qcow2/disk.qcow2,format=qcow2 \ + --os-variant fedora-eln +``` + +### Running the resulting QCOW2 file on macOS (aarch64) + +This assumes qemu was installed through [homebrew](https://brew.sh/). + +```bash +qemu-system-aarch64 \ + -M accel=hvf \ + -cpu host \ + -smp 2 \ + -m 4096 \ + -bios /opt/homebrew/Cellar/qemu/8.1.3_2/share/qemu/edk2-aarch64-code.fd \ + -serial stdio \ + -machine virt \ + -snapshot output/qcow2/disk.qcow2 +``` + +## πŸ“ Arguments + +```bash +Usage: + sudo podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v ./output:/output \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + quay.io/centos-bootc/bootc-image-builder:latest \ + + +Flags: + --chown string chown the ouput directory to match the specified UID:GID + --output string artifact output directory (default ".") + --progress string type of progress bar to use (e.g. verbose,term) (default "auto") + --rootfs string Root filesystem type. If not given, the default configured in the source container image is used. + --target-arch string build for the given target architecture (experimental) + --type stringArray image types to build [ami, anaconda-iso, bootc-installer, gce, iso, qcow2, raw, vhd, vmdk] (default [qcow2]) + --version version for bootc-image-builder + +Global Flags: + --log-level string logging level (debug, info, error); default error + -v, --verbose Switch to verbose mode +``` + +### Detailed description of optional flags + +| Argument | Description | Default Value | +|-------------------|-----------------------------------------------------------------------------------------------------------|:-------------:| +| --chown | chown the output directory to match the specified UID:GID | ❌ | +| --output | output the artifact into the given output directory | `.` | +| --progress | Show progress in the given format, supported: verbose,term,debug. If empty it is auto-detected | `auto` | +| **--rootfs** | Root filesystem type. Overrides the default from the source container. Supported values: ext4, xfs, btrfs | ❌ | +| **--type** | [Image type](#-image-types) to build (can be passed multiple times) | `qcow2` | +| --target-arch | [Target arch](#-target-architecture) to build | ❌ | +| --log-level | Change log level (debug, info, error) | `error` | +| -v,--verbose | Switch output/progress to verbose mode (implies --log-level=info) | `false` | +| --use-librepo | Download rpms using librepo (faster and more robust) | `false` | + +The `--type` parameter can be given multiple times and multiple +outputs will be produced. Note that comma or space separating the +`image-types`will not work, but this example will: `--type qcow2 +--type ami`. + + +*πŸ’‘ Tip: Flags in **bold** are the most important ones.* + +## πŸ’Ύ Image types + +The following image types are currently available via the `--type` argument: + +| Image type | Target environment | +|-----------------------|-------------------------------------------------------------------------------------------| +| `ami` | [Amazon Machine Image](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) | +| `qcow2` **(default)** | [QEMU](https://www.qemu.org/) | +| `vmdk` | [VMDK](https://en.wikipedia.org/wiki/VMDK) usable in vSphere, among others | +| `bootc-installer` | An installer ISO image based on the specified bootc container image. | +| `anaconda-iso` | An unattended Anaconda installer that installs to the first disk found. Built from RPMs. | +| `raw` | Unformatted [raw disk](https://en.wikipedia.org/wiki/Rawdisk). | +| `vhd` | [vhd](https://en.wikipedia.org/wiki/VHD_(file_format)) usable in Virtual PC, among others | +| `gce` | [GCE](https://cloud.google.com/compute/docs/images#custom_images) | +| `pxe-tar-xz` | A stateless image useful in PXE network boot environments | + + +## πŸ’Ύ Image Type Requirements + +### pxe-tar-xz + +The container image being built must have the `dracut-live` and `squashfs-tools` packages installed as well as a rebuilding the initramfs with the 'dmsquash-live' module. See [osbuild documentation](https://github.com/osbuild/image-builder/blob/main/data/files/pxetree/README) for more information and a sample Containerfile. + +### bootc-installer + +When building `bootc-installer` the positional container argument is expected to be a container that has Anaconda inside it; an example `Containerfile` for such a container is: + +``` +FROM your-favorite-bootc-container:latest +RUN dnf install -y \ + anaconda \ + anaconda-install-env-deps \ + anaconda-dracut \ + dracut-config-generic \ + dracut-network \ + net-tools \ + squashfs-tools \ + grub2-efi-x64-cdboot \ + python3-mako \ + lorax-templates-* \ + biosdevname \ + prefixdevname \ + && dnf clean all + +# On Fedora 42 this is necessary to get files in the right places +# RUN dnf reinstall -y shim-x64 + +# On Fedora 43 and up this is necessary to get files in the right +# places +RUN mkdir -p /boot/efi && cp -ra /usr/lib/efi/*/*/EFI /boot/efi + +# lorax wants to create a symlink in /mnt which points to /var/mnt +# on bootc but /var/mnt does not exist on some images. +# +# If https://gitlab.com/fedora/bootc/base-images/-/merge_requests/294 +# gets merged this will be no longer needed +RUN mkdir /var/mnt +``` + +You must also pass the `--bootc-installer-payload-ref` argument. This is a container reference to the payload to be installed by Anaconda. It will be embedded inside the installer and Anaconda will be configured to install it. + +## πŸ’Ύ Target architecture + +Specify the target architecture of the system on which the disk image will be installed on. By default, +`bootc-image-builder` will build for the native host architecture. The target architecture +must match an available architecture of the `bootc-image-builder` image you are using to build the disk image. +Navigate to the [centos-image-builder repository tags page](https://quay.io/repository/centos-bootc/bootc-image-builder?tab=tags) +and hover over the Tux icons to see the supported target architectures. +The architecture of the bootc OCI image and the bootc-image-builder image must match. For example, when building +a non-native architecture bootc OCI image, say, building for x86_64 from an arm-based Mac, it is possible to run +`podman build` with the `--platform linux/amd64` flag. In this case, to then build a disk image from the same arm-based Mac, +you should provide `--target-arch amd64` when running the `bootc-image-builder` command. + +## Progress types + +The following progress types are supported: + +* verbose: No spinners or progress bar, just information and full osbuild output +* term: Terminal based output, spinner, progressbar and most details of osbuild are hidden +* debug: Details how the progress is called, mostly useful for bugreports + +Note that when no value is given the progress is auto-detected baed on the environment. When `stdin` is a terminal the "term" progress is used, otherwise "verbose". The output of `verbose` is exactaly the same as it was before progress reporting was implemented. + +## ☁️ Cloud uploaders + +### Amazon Machine Images (AMIs) + +#### Prerequisites + +In order to successfully import an AMI into your AWS account, you need to have the [vmimport service role](https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html) configured on your account with the following additional permissions: + +``` +{ + "Effect": "Allow", + "Action": [ + "s3:ListAllMyBuckets", + "s3:GetBucketAcl", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::amzn-s3-demo-import-bucket", + "arn:aws:s3:::amzn-s3-demo-import-bucket/*", + "arn:aws:s3:::amzn-s3-demo-export-bucket", + "arn:aws:s3:::amzn-s3-demo-export-bucket/*" + ] +}, +``` + +Replace `amzn-s3-demo-import-bucket` in the ARN with the bucket name. + +#### Flags + +AMIs can be automatically uploaded to AWS by specifying the following flags: + +| Argument | Description | +|----------------|------------------------------------------------------------------| +| --aws-ami-name | Name for the AMI in AWS | +| --aws-bucket | Target S3 bucket name for intermediate storage when creating AMI | +| --aws-region | Target region for AWS uploads | + +*Notes:* + +- *These flags must all be specified together. If none are specified, the AMI is exported to the output directory.* +- *The bucket must already exist in the selected region, bootc-image-builder will not create it if it is missing.* +- *The output volume is not needed in this case. The image is uploaded to AWS and not exported.* + +#### AWS credentials file + +If you already have a credentials file (usually in `$HOME/.aws/credentials`) you need to forward the +directory to the container + +For example: + +```bash +$ sudo podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v $HOME/.aws:/root/.aws:ro \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + --env AWS_PROFILE=default \ + quay.io/centos-bootc/bootc-image-builder:latest \ + --type ami \ + --aws-ami-name centos-bootc-ami \ + --aws-bucket fedora-bootc-bucket \ + --aws-region us-east-1 \ + quay.io/centos-bootc/centos-bootc:stream9 +``` + +Notes: + +- *you can also inject **ALL** your AWS configuration parameters with `--env AWS_*`* + +see the [AWS CLI documentation](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) for more information about other environment variables + +#### AWS credentials via environment + +AWS credentials can be specified through two environment variables: +| Variable name | Description | +|-----------------------|---------------------------------------------------------------------------------------------------------------------| +| AWS_ACCESS_KEY_ID | AWS access key associated with an IAM account. | +| AWS_SECRET_ACCESS_KEY | Specifies the secret key associated with the access key. This is essentially the "password" for the access key. | + +Those **should not** be specified with `--env` as plain value, but you can silently hand them over with `--env AWS_*` or +save these variables in a file and pass them using the `--env-file` flag for `podman run`. + +For example: + +```bash +$ cat aws.secrets +AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +$ sudo podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + --env-file=aws.secrets \ + quay.io/centos-bootc/bootc-image-builder:latest \ + --type ami \ + --aws-ami-name centos-bootc-ami \ + --aws-bucket centos-bootc-bucket \ + --aws-region us-east-1 \ + quay.io/centos-bootc/centos-bootc:stream9 +``` + +## πŸ’½ Volumes + +The following volumes can be mounted inside the container: + +| Volume | Purpose | Required | +|-----------|--------------------------------------------------------|:--------:| +| `/output` | Used for storing the resulting artifacts | βœ… | +| `/store` | Used for the [osbuild store](https://www.osbuild.org/) | No | +| `/rpmmd` | Used for the DNF cache | No | + +## πŸ“ Build config + +A build config is a TOML (or JSON) file with customizations for the resulting image. The config file is mapped into the container directory to `/config.toml`. The customizations are specified under a `customizations` object. + +The build config is a [Blueprint file](https://github.com/osbuild/blueprint), documented in the [osbuild.org User Guide](https://osbuild.org/docs/user-guide/blueprint-reference/). Note that not all Blueprint options are supported in bootc-image-builder. Refer to the **bootc** tab for information on whether a specific customization is supported. + +As an example, let's show how you can add a user to the image: + +Firstly create a file `./config.toml` and put the following content into it: + +```toml +[[customizations.user]] +name = "alice" +password = "bob" +key = "ssh-rsa AAA ... user@email.com" +groups = ["wheel"] +``` + +Then, run `bootc-image-builder` with the following arguments: + +```bash +sudo podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v ./config.toml:/config.toml:ro \ + -v ./output:/output \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + quay.io/centos-bootc/bootc-image-builder:latest \ + --type qcow2 \ + quay.io/centos-bootc/centos-bootc:stream9 +``` + +The configuration can also be passed in via stdin when `--config -` +is used. Only JSON configuration is supported in this mode. + +Additionally, images can embed a build config file, either as +`config.json` or `config.toml` in the `/usr/lib/bootc-image-builder` +directory. If this exist, and contains filesystem or disk +customizations, then these are used by default if no such +customization are specified in the regular build config. + +### Users (`user`, array) + +Possible fields: + +| Field | Use | Required | +|------------|--------------------------------------------|:--------:| +| `name` | Name of the user | βœ… | +| `password` | Unencrypted password | No | +| `key` | Public SSH key contents | No | +| `groups` | An array of secondary to put the user into | No | + +Example: + +```json +{ + "customizations": { + "user": [ + { + "name": "alice", + "password": "bob", + "key": "ssh-rsa AAA ... user@email.com", + "groups": [ + "wheel", + "admins" + ] + } + ] + } +} +``` + +### Kernel Arguments (`kernel`, mapping) + +```json +{ + "customizations": { + "kernel": { + "append": "mitigations=auto,nosmt" + } + } +} + +``` + +### Filesystems (`filesystem`, array) + +The filesystem section of the customizations can be used to set the minimum size of the base partitions (`/` and `/boot`) as well as to create extra partitions with mountpoints under `/var`. + +```toml +[[customizations.filesystem]] +mountpoint = "/" +minsize = "10 GiB" + +[[customizations.filesystem]] +mountpoint = "/var/data" +minsize = "20 GiB" +``` + +```json +{ + "customizations": { + "filesystem": [ + { + "mountpoint": "/", + "minsize": "10 GiB" + }, + { + "mountpoint": "/var/data", + "minsize": "20 GiB" + } + ] + } +} +``` + +#### Interaction with `rootfs` + +#### Filesystem types + +The `--rootfs` option also sets the filesystem types for all additional mountpoints, where appropriate. See the see [Detailed description of optional flags](#detailed-description-of-optional-flags). + + +#### Allowed mountpoints and sizes + +The following restrictions and rules apply, unless the rootfs is `btrfs`: +- `/` can be specified to set the desired (minimum) size of the root filesystem. The final size of the filesystem, mounted at `/sysroot` on a booted system, is the value specified in this configuration or 2x the size of the base container, whichever is largest. +- `/boot`can be specified to set the desired size of the boot partition. +- Subdirectories of `/var` are supported, but symlinks in `/var` are not. For example, `/var/home` and `/var/run` are symlinks and cannot be filesystems on their own. +- `/var` itself cannot be a mountpoint. + +The `rootfs` option (or source container config, see [Detailed description of optional flags](#detailed-description-of-optional-flags) section) defines the filesystem type for the root filesystem. Currently, creation of btrfs subvolumes at build time is not supported. Therefore, if the `rootfs` is `btrfs`, no custom mountpoints are supported under `/var`. Only `/` and `/boot` can be configured. + + +### Anaconda ISO (installer) options (`installer`, mapping) + +Users can include kickstart file content that will be added to an ISO build to configure the installation process. When using custom kickstart scripts the customization needs to be done via the custom kickstart script. For example using a `[customizations.user]` block alongside a `[customizations.installer.kickstart]` block is not supported. See this issue [https://github.com/osbuild/bootc-image-builder/issues/528] for additional detail. + +Since multi-line strings are difficult to write and read in json, it's easier to use the toml format when adding kickstart contents: + +```toml +[customizations.installer.kickstart] +contents = """ +text --non-interactive +zerombr +clearpart --all --initlabel --disklabel=gpt +autopart --noswap --type=lvm +network --bootproto=dhcp --device=link --activate --onboot=on +""" +``` + +The equivalent in json would be: +```json +{ + "customizations": { + "installer": { + "kickstart": { + "contents": "text --non-interactive\nzerombr\nclearpart --all --initlabel --disklabel=gpt\nautopart --noswap --type=lvm\nnetwork --bootproto=dhcp --device=link --activate --onboot=on" + } + } + } +} +``` + +Note that bootc-image-builder will automatically add the command that installs the container image (`ostreecontainer ...`), so this line or any line that conflicts with it should not be included. See the relevant [Kickstart documentation](https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#ostreecontainer) for more information. +No other kickstart commands are added by bootc-image-builder in this case, so it is the responsibility of the user to provide all other commands (for example, for partitioning, network, language, etc). + +#### Anaconda ISO (installer) Modules + +The Anaconda installer can be configured by enabling or disabling its dbus modules. + +```toml +[customizations.installer.modules] +enable = [ + "org.fedoraproject.Anaconda.Modules.Localization" +] +disable = [ + "org.fedoraproject.Anaconda.Modules.Users" +] +``` + +```json +{ + "customizations": { + "installer": { + "modules": { + "enable": [ + "org.fedoraproject.Anaconda.Modules.Localization" + ], + "disable": [ + "org.fedoraproject.Anaconda.Modules.Users" + ] + } + } + } +} +``` + +The following module names are known and supported: +- `org.fedoraproject.Anaconda.Modules.Localization` +- `org.fedoraproject.Anaconda.Modules.Network` +- `org.fedoraproject.Anaconda.Modules.Payloads` +- `org.fedoraproject.Anaconda.Modules.Runtime` +- `org.fedoraproject.Anaconda.Modules.Security` +- `org.fedoraproject.Anaconda.Modules.Services` +- `org.fedoraproject.Anaconda.Modules.Storage` +- `org.fedoraproject.Anaconda.Modules.Subscription` +- `org.fedoraproject.Anaconda.Modules.Timezone` +- `org.fedoraproject.Anaconda.Modules.Users` + +*Note: The values are not validated. Any name listed under `enable` will be added to the Anaconda configuration. This way, new or unknown modules can be enabled. However, it also means that mistyped or incorrect values may cause Anaconda to fail to start.* + +By default, the following modules are enabled for all Anaconda ISOs: +- `org.fedoraproject.Anaconda.Modules.Network` +- `org.fedoraproject.Anaconda.Modules.Payloads` +- `org.fedoraproject.Anaconda.Modules.Security` +- `org.fedoraproject.Anaconda.Modules.Services` +- `org.fedoraproject.Anaconda.Modules.Storage` +- `org.fedoraproject.Anaconda.Modules.Users` + +### Anaconda ISO (media) options (`iso`, mapping) + +Users can customize the volume_id (which will be the ISO's label, used also in boot/grub.cfg). + + +```toml +[customizations.iso] +volume_id = "TheISOLabel" +application_id = "MyFancyAPP" +publisher = "ThePublisher" +``` + +##### Enable vs Disable priority + +The `disable` list is processed after the `enable` list and therefore takes priority. In other words, adding the same module in both `enable` and `disable` will result in the module being **disabled**. +Furthermore, adding a module that is enabled by default to `disable` will result in the module being **disabled**. + +## Building + +To build the container locally you can run + +```shell +sudo podman build --tag bootc-image-builder . +``` + +NOTE: running already the `podman build` as root avoids problems later as we need to run the building +of the image as root anyway + +### Accessing the system + +With a virtual machine launched with the above [virt-install](#virt-install) example, access the system with + +```shell +ssh -i /path/to/private/ssh-key alice@ip-address +``` + +Note that if you do not provide a password for the provided user, `sudo` will not work unless passwordless sudo +is configured. The base image `quay.io/centos-bootc/centos-bootc:stream9` does not configure passwordless sudo. +This can be configured in a derived bootc container by including the following in a Containerfile. + +```dockerfile +FROM quay.io/centos-bootc/centos-bootc:stream9 +ADD wheel-passwordless-sudo /etc/sudoers.d/wheel-passwordless-sudo +``` + +The contents of the file `wheel-passwordless-sudo` should be + +```text +%wheel ALL=(ALL) NOPASSWD: ALL +``` From 30bd0a3efb84a183ab0cd7d22181b0a87a40eacc Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 11:28:49 +0200 Subject: [PATCH 07/25] test: move vmtest under imgtestlib as .vm Let's integrate vmtest into imgtestlib. There's no need to have two separate modules for testing things. imgtestlib already imports vmtest and this makes things simpler. --- test/scripts/imgtestlib/__init__.py | 1 + test/scripts/{vmtest => imgtestlib/vm}/__init__.py | 0 test/scripts/{vmtest => imgtestlib/vm}/test/test_ssh.py | 0 test/scripts/{vmtest => imgtestlib/vm}/util.py | 0 test/scripts/{vmtest => imgtestlib/vm}/util_test.py | 0 test/scripts/{vmtest => imgtestlib/vm}/vm.py | 0 6 files changed, 1 insertion(+) rename test/scripts/{vmtest => imgtestlib/vm}/__init__.py (100%) rename test/scripts/{vmtest => imgtestlib/vm}/test/test_ssh.py (100%) rename test/scripts/{vmtest => imgtestlib/vm}/util.py (100%) rename test/scripts/{vmtest => imgtestlib/vm}/util_test.py (100%) rename test/scripts/{vmtest => imgtestlib/vm}/vm.py (100%) diff --git a/test/scripts/imgtestlib/__init__.py b/test/scripts/imgtestlib/__init__.py index fe1d86bfbb..3a6457a7ec 100644 --- a/test/scripts/imgtestlib/__init__.py +++ b/test/scripts/imgtestlib/__init__.py @@ -5,3 +5,4 @@ from .gitlab import * from .run import * from .testenv import * +from .vm import * diff --git a/test/scripts/vmtest/__init__.py b/test/scripts/imgtestlib/vm/__init__.py similarity index 100% rename from test/scripts/vmtest/__init__.py rename to test/scripts/imgtestlib/vm/__init__.py diff --git a/test/scripts/vmtest/test/test_ssh.py b/test/scripts/imgtestlib/vm/test/test_ssh.py similarity index 100% rename from test/scripts/vmtest/test/test_ssh.py rename to test/scripts/imgtestlib/vm/test/test_ssh.py diff --git a/test/scripts/vmtest/util.py b/test/scripts/imgtestlib/vm/util.py similarity index 100% rename from test/scripts/vmtest/util.py rename to test/scripts/imgtestlib/vm/util.py diff --git a/test/scripts/vmtest/util_test.py b/test/scripts/imgtestlib/vm/util_test.py similarity index 100% rename from test/scripts/vmtest/util_test.py rename to test/scripts/imgtestlib/vm/util_test.py diff --git a/test/scripts/vmtest/vm.py b/test/scripts/imgtestlib/vm/vm.py similarity index 100% rename from test/scripts/vmtest/vm.py rename to test/scripts/imgtestlib/vm/vm.py From 191833020d5dcec21dc76b22ea436de12a459cf2 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 12:07:35 +0200 Subject: [PATCH 08/25] test: flatten vm submodule under imgtestlib Flatten the vm module (previously vmtest) under imgtestlib. All functions are available under imgtestlib.vm now. Move all vm tests (previously vmtest/test) to test_imgtestlib. Update import paths in the rest of the imgtestlib module to import the vm submodule. --- test/scripts/imgtestlib/boot.py | 4 +- test/scripts/imgtestlib/{vm => }/vm.py | 38 ++++-- test/scripts/imgtestlib/vm/__init__.py | 0 test/scripts/imgtestlib/vm/test/test_ssh.py | 95 -------------- test/scripts/imgtestlib/vm/util.py | 24 ---- test/scripts/imgtestlib/vm/util_test.py | 47 ------- test/scripts/test_imgtestlib.py | 132 +++++++++++++++++++- 7 files changed, 161 insertions(+), 179 deletions(-) rename test/scripts/imgtestlib/{vm => }/vm.py (93%) delete mode 100644 test/scripts/imgtestlib/vm/__init__.py delete mode 100644 test/scripts/imgtestlib/vm/test/test_ssh.py delete mode 100644 test/scripts/imgtestlib/vm/util.py delete mode 100644 test/scripts/imgtestlib/vm/util_test.py diff --git a/test/scripts/imgtestlib/boot.py b/test/scripts/imgtestlib/boot.py index 1ddaed46a1..9663f0aa39 100644 --- a/test/scripts/imgtestlib/boot.py +++ b/test/scripts/imgtestlib/boot.py @@ -12,15 +12,13 @@ from tempfile import TemporaryDirectory from typing import Generator -from vmtest.util import get_free_port -from vmtest.vm import QEMU - from .build import read_build_info, write_build_info from .core import (can_boot_test, find_image_file, read_manifest, skopeo_inspect_id) from .gitlab import log_section from .run import runcmd, runcmd_nc from .testenv import get_bib_ref, host_container_arch +from .vm import QEMU, get_free_port BASE_TEST_EXEC = "check-host-config-" # + arch WSL_TEST_SCRIPT = "test/scripts/wsl-entrypoint.bat" diff --git a/test/scripts/imgtestlib/vm/vm.py b/test/scripts/imgtestlib/vm.py similarity index 93% rename from test/scripts/imgtestlib/vm/vm.py rename to test/scripts/imgtestlib/vm.py index 5c3a2eb88a..31ff228146 100644 --- a/test/scripts/imgtestlib/vm/vm.py +++ b/test/scripts/imgtestlib/vm.py @@ -4,7 +4,8 @@ import platform import shlex import shutil -import subprocess +import socket +import subprocess as sp import sys import tempfile import time @@ -12,7 +13,6 @@ from io import StringIO try: - # The vmtest module is imported by imgtestlib. # Allow importing the module without boto3 since most times it's not needed. import boto3 from botocore.exceptions import ClientError @@ -20,8 +20,6 @@ boto3 = None ClientError = None -from vmtest.util import get_free_port, wait_ssh_ready - AWS_REGION = "us-east-1" @@ -104,15 +102,15 @@ def _run(self, args, user, password="", keyfile=None): ssh_cmd.append(f"{user}@{self._address}") ssh_cmd.append(run_cmd) output = StringIO() - with subprocess.Popen( + with sp.Popen( ssh_cmd, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + stdout=sp.PIPE, stderr=sp.STDOUT, text=True, bufsize=1, ) as p: for out in p.stdout: self._log(out) output.write(out) - ret = subprocess.CompletedProcess(run_cmd, p.returncode) + ret = sp.CompletedProcess(run_cmd, p.returncode) ret.stdout = output.getvalue() # this will raise an CalledProcessError on error ret.check_returncode() @@ -127,7 +125,7 @@ def scp(self, src, dst, user, password="", keyfile=None): scp_cmd.extend(["-i", keyfile]) scp_cmd.append(src) scp_cmd.append(f"{user}@{self._address}:{dst}") - subprocess.check_call(scp_cmd) + sp.check_call(scp_cmd) @property def ssh_port(self): @@ -270,7 +268,7 @@ def start(self, wait_event="ssh", snapshot=True, use_ovmf=False, timeout_sec=180 # XXX: use systemd-run to ensure cleanup? # pylint: disable=consider-using-with - self._qemu_p = subprocess.Popen( + self._qemu_p = sp.Popen( self._gen_qemu_cmdline(snapshot, use_ovmf), stdout=sys.stdout, stderr=sys.stderr, @@ -424,3 +422,25 @@ def force_stop(self): def running(self): return self._ec2_instance is not None + + +def get_free_port() -> int: + # this is racy but there is no race-free way to do better with the qemu CLI + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("localhost", 0)) + return s.getsockname()[1] + + +def wait_ssh_ready(address, port, sleep, max_wait_sec): + for _ in range(int(max_wait_sec / sleep)): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(sleep) + try: + s.connect((address, port)) + data = s.recv(256) + if b"OpenSSH" in data: + return + except (ConnectionRefusedError, ConnectionResetError, TimeoutError): + pass + time.sleep(sleep) + raise ConnectionRefusedError(f"cannot connect to port {port} after {max_wait_sec}s") diff --git a/test/scripts/imgtestlib/vm/__init__.py b/test/scripts/imgtestlib/vm/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/scripts/imgtestlib/vm/test/test_ssh.py b/test/scripts/imgtestlib/vm/test/test_ssh.py deleted file mode 100644 index 739787d9d7..0000000000 --- a/test/scripts/imgtestlib/vm/test/test_ssh.py +++ /dev/null @@ -1,95 +0,0 @@ -import os -import subprocess -import textwrap -import time -from unittest.mock import call, patch - -import pytest - -import vmtest.vm -from vmtest.vm import VM -from vmtest.util import get_free_port - - -class MockVM(VM): - _address = None - _ssh_port = None - def start(self): - pass - def force_stop(self): - pass - def running(self): - return True - def set_ssh(self, address, port): - self._address = address - self._ssh_port = port - - - -def make_fake_ssh(fake_bin_path, extra_script=""): - mock_ssh = fake_bin_path / "ssh" - mock_ssh.write_text(textwrap.dedent(f"""\ - #!/bin/bash -e - echo "calling $0 with: $@" - {extra_script} - """)) - mock_ssh.chmod(0o755) - return mock_ssh - - -def test_ssh_calls_cmd_happy(tmp_path, monkeypatch): - monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) - make_fake_ssh(tmp_path) - vm = MockVM() - res = vm.run(["cmd1", "arg1", "arg2"], user="user1", keyfile="keyfile1") - assert res.returncode == 0 - assert res.stdout.endswith("cmd1 arg1 arg2\n") - - -def test_ssh_calls_cmd_happy_single_cmd(tmp_path, monkeypatch): - monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) - make_fake_ssh(tmp_path) - vm = MockVM() - res = vm.run("true", user="user1", keyfile="keyfile1") - assert res.returncode == 0 - assert res.stdout.endswith("true\n") - - -def test_ssh_calls_cmd_happy_quoting_works(tmp_path, monkeypatch): - monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) - make_fake_ssh(tmp_path) - vm = MockVM() - res = vm.run("this needs quoting", user="user1", keyfile="keyfile1") - assert res.returncode == 0 - assert res.stdout.endswith("'this needs quoting'\n") - -def test_ssh_calls_cmd_sad(tmp_path, monkeypatch): - monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) - make_fake_ssh(tmp_path, """ echo bad-output ; if [ "${@: -1}" = "bad-cmd" ]; then exit 42; fi """) - vm = MockVM() - with pytest.raises(subprocess.CalledProcessError) as e: - vm.run("bad-cmd", user="user1", keyfile="keyfile1") - assert e.value.returncode == 42 - assert e.value.stdout.endswith("bad-output\n") - - -@patch("time.sleep") -def test_ssh_calls_retries(mocked_sleep, tmp_path, monkeypatch, capsys): - monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) - make_fake_ssh(tmp_path, "echo ssh-very-sad; exit 21") - vm = MockVM() - with pytest.raises(RuntimeError) as e: - res = vm.run(["cmd1", "arg1", "arg2"], user="user1", keyfile="keyfile1") - assert str(e.value) == "no ssh after 30 retries of 10s" - assert mocked_sleep.call_args_list == 30 * [ call(10) ] - assert capsys.readouterr().err == "\n".join([ - f"ssh not ready {i+1}/30: Command 'true' returned non-zero exit status 21." - for i in range(30) - ]) + "\n" - -def test_wait_ssh_ready_timeout(): - vm = MockVM() - vm.set_ssh("localhost", get_free_port()) - with pytest.raises(ConnectionRefusedError) as e: - vm.wait_ssh_ready(timeout_sec=3) - assert "after 3s" in str(e.value) diff --git a/test/scripts/imgtestlib/vm/util.py b/test/scripts/imgtestlib/vm/util.py deleted file mode 100644 index 195f52134a..0000000000 --- a/test/scripts/imgtestlib/vm/util.py +++ /dev/null @@ -1,24 +0,0 @@ -import socket -import time - - -def get_free_port() -> int: - # this is racy but there is no race-free way to do better with the qemu CLI - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("localhost", 0)) - return s.getsockname()[1] - - -def wait_ssh_ready(address, port, sleep, max_wait_sec): - for _ in range(int(max_wait_sec / sleep)): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(sleep) - try: - s.connect((address, port)) - data = s.recv(256) - if b"OpenSSH" in data: - return - except (ConnectionRefusedError, ConnectionResetError, TimeoutError): - pass - time.sleep(sleep) - raise ConnectionRefusedError(f"cannot connect to port {port} after {max_wait_sec}s") diff --git a/test/scripts/imgtestlib/vm/util_test.py b/test/scripts/imgtestlib/vm/util_test.py deleted file mode 100644 index 6d91720e13..0000000000 --- a/test/scripts/imgtestlib/vm/util_test.py +++ /dev/null @@ -1,47 +0,0 @@ -import contextlib -import shutil -import subprocess -from unittest.mock import call, patch - -import pytest - -from vmtest.util import get_free_port, wait_ssh_ready - - -def test_get_free_port(): - port_nr = get_free_port() - assert 1024 < port_nr < 65535 - - -@patch("time.sleep") -def test_wait_ssh_ready_sleeps_no_connection(mocked_sleep): - free_port = get_free_port() - with pytest.raises(ConnectionRefusedError): - wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.35) - assert mocked_sleep.call_args_list == [call(0.1), call(0.1), call(0.1)] - - -@pytest.mark.skipif(not shutil.which("nc"), reason="needs nc") -def test_wait_ssh_ready_sleeps_wrong_reply(): - free_port = get_free_port() - with contextlib.ExitStack() as cm: - with subprocess.Popen( - f"echo not-ssh | nc -vv -l -p {free_port}", - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - encoding="utf-8", - ) as p: - cm.callback(p.kill) - # wait for nc to be ready - while True: - # netcat tranditional uses "listening", others "Listening" - # so just omit the first char - if "istening " in p.stdout.readline(): - break - # now connect - with patch("time.sleep") as mocked_sleep: - with pytest.raises(ConnectionRefusedError): - wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.55) - assert mocked_sleep.call_args_list == [ - call(0.1), call(0.1), call(0.1), call(0.1), call(0.1)] diff --git a/test/scripts/test_imgtestlib.py b/test/scripts/test_imgtestlib.py index 3115a6ff13..69bf046f4b 100644 --- a/test/scripts/test_imgtestlib.py +++ b/test/scripts/test_imgtestlib.py @@ -1,8 +1,11 @@ +import contextlib import json import os +import shutil import subprocess as sp import tempfile -from unittest.mock import patch +import textwrap +from unittest.mock import call, patch import pytest @@ -409,3 +412,130 @@ def test_find_image_file_multiple_files_in_export(): # Should raise error about multiple files with pytest.raises(RuntimeError, match="Expected exactly one file in export directory"): testlib.find_image_file(tmpdir) + + +def test_get_free_port(): + port_nr = testlib.vm.get_free_port() + assert 1024 < port_nr < 65535 + + +@patch("time.sleep") +def test_wait_ssh_ready_sleeps_no_connection(mocked_sleep): + free_port = testlib.vm.get_free_port() + with pytest.raises(ConnectionRefusedError): + testlib.vm.wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.35) + assert mocked_sleep.call_args_list == [call(0.1), call(0.1), call(0.1)] + + +@pytest.mark.skipif(not shutil.which("nc"), reason="needs nc") +def test_wait_ssh_ready_sleeps_wrong_reply(): + free_port = testlib.vm.get_free_port() + with contextlib.ExitStack() as cm: + with sp.Popen( + f"echo not-ssh | nc -vv -l -p {free_port}", + shell=True, + stdout=sp.PIPE, + stderr=sp.STDOUT, + encoding="utf-8", + ) as p: + cm.callback(p.kill) + # wait for nc to be ready + while True: + # netcat tranditional uses "listening", others "Listening" + # so just omit the first char + if "istening " in p.stdout.readline(): + break + # now connect + with patch("time.sleep") as mocked_sleep: + with pytest.raises(ConnectionRefusedError): + testlib.vm.wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.55) + assert mocked_sleep.call_args_list == [ + call(0.1), call(0.1), call(0.1), call(0.1), call(0.1)] + + +class MockVM(testlib.vm.VM): + _address = None + _ssh_port = None + + def start(self): + pass + + def force_stop(self): + pass + + def running(self): + return True + + def set_ssh(self, address, port): + self._address = address + self._ssh_port = port + + +def make_fake_ssh(fake_bin_path, extra_script=""): + mock_ssh = fake_bin_path / "ssh" + mock_ssh.write_text(textwrap.dedent(f"""\ + #!/bin/bash -e + echo "calling $0 with: $@" + {extra_script} + """)) + mock_ssh.chmod(0o755) + return mock_ssh + + +def test_ssh_calls_cmd_happy(tmp_path, monkeypatch): + monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) + make_fake_ssh(tmp_path) + vm = MockVM() + res = vm.run(["cmd1", "arg1", "arg2"], user="user1", keyfile="keyfile1") + assert res.returncode == 0 + assert res.stdout.endswith("cmd1 arg1 arg2\n") + + +def test_ssh_calls_cmd_happy_single_cmd(tmp_path, monkeypatch): + monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) + make_fake_ssh(tmp_path) + vm = MockVM() + res = vm.run("true", user="user1", keyfile="keyfile1") + assert res.returncode == 0 + assert res.stdout.endswith("true\n") + + +def test_ssh_calls_cmd_happy_quoting_works(tmp_path, monkeypatch): + monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) + make_fake_ssh(tmp_path) + vm = MockVM() + res = vm.run("this needs quoting", user="user1", keyfile="keyfile1") + assert res.returncode == 0 + assert res.stdout.endswith("'this needs quoting'\n") + + +def test_ssh_calls_cmd_sad(tmp_path, monkeypatch): + monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) + make_fake_ssh(tmp_path, """ echo bad-output ; if [ "${@: -1}" = "bad-cmd" ]; then exit 42; fi """) + vm = MockVM() + with pytest.raises(sp.CalledProcessError) as e: + vm.run("bad-cmd", user="user1", keyfile="keyfile1") + assert e.value.returncode == 42 + assert e.value.stdout.endswith("bad-output\n") + + +@patch("time.sleep") +def test_ssh_calls_retries(mocked_sleep, tmp_path, monkeypatch, capsys): + monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) + make_fake_ssh(tmp_path, "echo ssh-very-sad; exit 21") + vm = MockVM() + with pytest.raises(RuntimeError) as e: + vm.run(["cmd1", "arg1", "arg2"], user="user1", keyfile="keyfile1") + assert str(e.value) == "no ssh after 30 retries of 10s" + assert mocked_sleep.call_args_list == 30 * [call(10)] + assert capsys.readouterr().err == "\n".join([ + f"ssh not ready {i+1}/30: Command 'true' returned non-zero exit status 21." + for i in range(30) + ]) + "\n" + +def test_wait_ssh_ready_timeout(): + vm = MockVM() + vm.set_ssh("localhost", testlib.vm.get_free_port()) + with pytest.raises(ConnectionRefusedError) as e: + vm.wait_ssh_ready(timeout_sec=3) + assert "after 3s" in str(e.value) From a3913a75fc126155378e0acf4e0c6008373094b9 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 12:10:22 +0200 Subject: [PATCH 09/25] Make imgtestlib importable by bootc-image-builder tests Add the path to imgtestlib (parent) to the pytest config in pyproject.toml so that tests under bootc-image-builder/test can import it. Update import paths in the bootc-image-builder tests. --- bootc-image-builder/test/conftest.py | 7 +------ bootc-image-builder/test/test_build_disk.py | 14 +++++++------- bootc-image-builder/test/test_build_iso.py | 11 ++++++----- bootc-image-builder/test/test_build_pxe.py | 5 +++-- pyproject.toml | 6 +++--- 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/bootc-image-builder/test/conftest.py b/bootc-image-builder/test/conftest.py index 28e3094351..d473ff5e2c 100644 --- a/bootc-image-builder/test/conftest.py +++ b/bootc-image-builder/test/conftest.py @@ -1,7 +1,7 @@ import pytest + # pylint: disable=wrong-import-order from testcases import TestCase -from vmtest.util import get_free_port def pytest_addoption(parser): @@ -29,8 +29,3 @@ def pytest_make_parametrize_id(config, val): # pylint: disable=W0613 if isinstance(val, TestCase): return f"{val}" return None - - -@pytest.fixture(name="free_port") -def free_port_fixture(): - return get_free_port() diff --git a/bootc-image-builder/test/test_build_disk.py b/bootc-image-builder/test/test_build_disk.py index 67eece1fc8..f1d11b70b6 100644 --- a/bootc-image-builder/test/test_build_disk.py +++ b/bootc-image-builder/test/test_build_disk.py @@ -13,14 +13,14 @@ from dataclasses import dataclass from typing import NamedTuple +import imgtestlib as testlib import pytest + # local test utils import testutil -import vmtest.util from containerbuild import \ build_container_fixture # pylint: disable=unused-import from testcases import CLOUD_BOOT_IMAGE_TYPES, DISK_IMAGE_TYPES, gen_testcases -from vmtest.vm import AWS, AWS_REGION, QEMU if not testutil.has_executable("podman"): pytest.skip("no podman, skipping integration tests that required podman", allow_module_level=True) @@ -115,7 +115,7 @@ def registry_conf_fixture(shared_tmpdir, request): {local_registry}: lookaside: file:///{sigstore_dir} """ - registry_port = vmtest.util.get_free_port() + registry_port = testlib.vm.get_free_port() # We cannot use localhost as we need to access the registry from both # the host system and the bootc-image-builder container. default_ip = testutil.get_ip_from_default_route() @@ -412,7 +412,7 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_ upload_args = [ f"--aws-ami-name=bootc-image-builder-test-{str(uuid.uuid4())}", - f"--aws-region={AWS_REGION}", + f"--aws-region={testlib.vm.AWS_REGION}", "--aws-bucket=bootc-image-builder-ci", ] elif force_aws_upload: @@ -494,7 +494,7 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_ metadata["ami_id"] = parse_ami_id_from_log(journal_output) def del_ami(): - testutil.deregister_ami(metadata["ami_id"], AWS_REGION) + testutil.deregister_ami(metadata["ami_id"], testlib.vm.AWS_REGION) request.addfinalizer(del_ami) journal_log_path.write_text(journal_output, encoding="utf8") @@ -559,7 +559,7 @@ def test_image_boots(image_type): def assert_disk_image_boots(image_type): - with QEMU(image_type.img_path, arch=image_type.img_arch) as test_vm: + with testlib.vm.QEMU(image_type.img_path, arch=image_type.img_arch) as test_vm: # user/password login works test_vm.run("true", user=image_type.username, password=image_type.password) # root/ssh login also works @@ -595,7 +595,7 @@ def test_ami_boots_in_aws(image_type, force_aws_upload): # check that upload progress is in the output log. Uploads looks like: # 4.30 GiB / 10.00 GiB [------------>____________] 43.02% 58.04 MiB p/s assert "] 100.00%" in image_type.bib_output - with AWS(image_type.metadata["ami_id"]) as test_vm: + with testlib.vm.AWS(image_type.metadata["ami_id"]) as test_vm: test_vm.run("true", user=image_type.username, password=image_type.password) ret = test_vm.run(["echo", "hello"], user=image_type.username, password=image_type.password) assert "hello" in ret.stdout diff --git a/bootc-image-builder/test/test_build_iso.py b/bootc-image-builder/test/test_build_iso.py index 0bddc3ef86..ce8ba97f9b 100644 --- a/bootc-image-builder/test/test_build_iso.py +++ b/bootc-image-builder/test/test_build_iso.py @@ -8,7 +8,9 @@ import textwrap from contextlib import ExitStack +import imgtestlib as testlib import pytest + # local test utils import testutil from containerbuild import ( # pylint: disable=unused-import @@ -18,7 +20,6 @@ image_type_fixture, registry_conf_fixture, shared_tmpdir_fixture) from testcases import gen_testcases -from vmtest.vm import QEMU ISO_BOOT_TIMEOUT = 1800 @@ -31,11 +32,11 @@ def test_iso_installs(image_type): with open(test_disk_path, "w", encoding="utf8") as fp: fp.truncate(10_1000_1000_1000) # install to test disk - with QEMU(test_disk_path, cdrom=installer_iso_path) as vm: + with testlib.vm.QEMU(test_disk_path, cdrom=installer_iso_path) as vm: vm.start(wait_event="qmp:RESET", snapshot=False, use_ovmf=True, timeout_sec=ISO_BOOT_TIMEOUT) vm.force_stop() # boot test disk and do extremly simple check - with QEMU(test_disk_path) as vm: + with testlib.vm.QEMU(test_disk_path) as vm: vm.start(use_ovmf=True) vm.run("true", user=image_type.username, password=image_type.password) assert_kernel_args(vm, image_type) @@ -192,11 +193,11 @@ def test_bootc_installer_iso_installs(tmp_path, build_container, container_ref): with open(test_disk_path, "w", encoding="utf8") as fp: fp.truncate(10_1000_1000_1000) # install to test disk - with QEMU(test_disk_path, cdrom=installer_iso_path) as vm: + with testlib.vm.QEMU(test_disk_path, cdrom=installer_iso_path) as vm: vm.start(wait_event="qmp:RESET", snapshot=False, use_ovmf=True, timeout_sec=ISO_BOOT_TIMEOUT) vm.force_stop() # boot test disk and do extremly simple check - with QEMU(test_disk_path) as vm: + with testlib.vm.QEMU(test_disk_path) as vm: vm.start(use_ovmf=True) vm.run("true", user=username, password=password) ret = vm.run(["bootc", "status"], user="root", keyfile=ssh_keyfile_private_path) diff --git a/bootc-image-builder/test/test_build_pxe.py b/bootc-image-builder/test/test_build_pxe.py index 13731a8a4a..eb9632fc36 100644 --- a/bootc-image-builder/test/test_build_pxe.py +++ b/bootc-image-builder/test/test_build_pxe.py @@ -11,12 +11,13 @@ from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from tempfile import TemporaryDirectory +import imgtestlib as testlib import pytest + # local test utils import testutil from containerbuild import ( # pylint: disable=unused-import build_container_fixture, make_container) -from vmtest.vm import QEMU def get_ostree_path(grub_path): @@ -84,7 +85,7 @@ def boot_qemu_pxe(arch, pxe_tar_path, container_ref, username, password, ssh_key "-append", append_arg ] - with QEMU(test_disk_path, memory="4096", arch=arch, extra_args=extra_args) as vm: + with testlib.vm.QEMU(test_disk_path, memory="4096", arch=arch, extra_args=extra_args) as vm: vm.start(use_ovmf=use_ovmf) vm.run("true", user=username, password=password) vm.run("mount", user="root", keyfile=ssh_key_path) diff --git a/pyproject.toml b/pyproject.toml index afe1d26434..95e83b3120 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,6 @@ dependencies = [ "scp==0.15.0", ] -[tool.setuptools.packages.find] -where = ["test/scripts"] -include = ["vmtest"] +# make imgtestlib importable when running pytest +[tool.pytest.ini_options] +pythonpath = ["test/scripts"] From ea3b446e1610e7c110ef46555cd85ed0b1878918 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 12:15:01 +0200 Subject: [PATCH 10/25] test/install-dependencies: drop the pip install This was needed to make the vmtest module available in test environments. We don't need it any more. There are no installable python modules. --- test/scripts/install-dependencies | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/scripts/install-dependencies b/test/scripts/install-dependencies index 604f7e05ff..ec4348eca6 100755 --- a/test/scripts/install-dependencies +++ b/test/scripts/install-dependencies @@ -67,11 +67,3 @@ case "${ID}" in install_deps ;; esac - -# We need this for the qemu boot tests and for any python tests but -# could skip it for other testing. This can be removed once we figure -# out why it actually does not work: -# -# https://github.com/osbuild/images/issues/2288 -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -pip install "${SCRIPT_DIR}/../.." From 76c7c3fe258198d99c73bf59cbd4c18d08b8b474 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 12:38:38 +0200 Subject: [PATCH 11/25] test: drop -p from nc command On Fedora 44, nc prints nc: cannot use -p and -l With the -l option, the -p is not necessary and the port must be specified after the -l. --- test/scripts/test_imgtestlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scripts/test_imgtestlib.py b/test/scripts/test_imgtestlib.py index 69bf046f4b..bfc58a9d3d 100644 --- a/test/scripts/test_imgtestlib.py +++ b/test/scripts/test_imgtestlib.py @@ -432,7 +432,7 @@ def test_wait_ssh_ready_sleeps_wrong_reply(): free_port = testlib.vm.get_free_port() with contextlib.ExitStack() as cm: with sp.Popen( - f"echo not-ssh | nc -vv -l -p {free_port}", + f"echo not-ssh | nc -vv -l {free_port}", shell=True, stdout=sp.PIPE, stderr=sp.STDOUT, From b05b7a3046f336e77d441b1bc5846ea84819cec9 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 15:06:50 +0200 Subject: [PATCH 12/25] github: run pylint on the whole project Run pylint on the whole project. This commit also fixes or disables (mostly disables) all linting issues found. QEMUMonitorProtocol() is now used from the legacy submodule of qmp. Instead of installing qmp from pip, we install python3-qemu-qmp from the repos (using the install-dependencies script). The test_pylint.py and test_flake8.py files are removed since these checks are now handles by GitHub actions and not part of the unit tests. The flake8 dependency is dropped. We only run pylint. --- .github/workflows/tests.yml | 5 ++--- bootc-image-builder/test/containerbuild.py | 3 ++- bootc-image-builder/test/test_build_disk.py | 10 ++++++---- bootc-image-builder/test/test_build_iso.py | 8 +++++--- bootc-image-builder/test/test_build_pxe.py | 4 +++- bootc-image-builder/test/test_flake8.py | 11 ----------- bootc-image-builder/test/test_manifest.py | 3 +++ bootc-image-builder/test/test_progress.py | 7 ++++--- bootc-image-builder/test/test_pylint.py | 18 ------------------ bootc-image-builder/test/testcases.py | 2 ++ bootc-image-builder/test/testutil.py | 2 ++ test/_test_build.py | 3 +++ test/conftest.py | 8 -------- test/scripts/imgtestlib/core.py | 2 +- test/scripts/imgtestlib/vm.py | 18 ++++++++++++++---- test/scripts/install-dependencies | 1 - test/scripts/test_imgtestlib.py | 1 + test/test_build_integration.py | 2 ++ 18 files changed, 50 insertions(+), 58 deletions(-) delete mode 100644 bootc-image-builder/test/test_flake8.py delete mode 100644 bootc-image-builder/test/test_pylint.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9df47be5ca..4459527740 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -263,12 +263,11 @@ jobs: - name: Install build and test dependencies run: | ./test/scripts/install-dependencies - dnf -y install python3-pylint - name: Analysing the code with pylint run: | - python3 -m pylint --version - python3 -m pylint $(grep -l "/usr/bin/env python3" -r test/scripts) test/scripts/*.py + pylint --version + pylint --recursive=y . yaml-checks: name: "YAML checks" diff --git a/bootc-image-builder/test/containerbuild.py b/bootc-image-builder/test/containerbuild.py index 7b818af2b1..856fff18a7 100644 --- a/bootc-image-builder/test/containerbuild.py +++ b/bootc-image-builder/test/containerbuild.py @@ -37,7 +37,8 @@ def make_container(container_path, arch=None): def build_container_fixture(): """Build a container from the Containerfile and returns the name""" if tag_from_env := os.getenv("BIB_TEST_BUILD_CONTAINER_TAG"): - return tag_from_env + yield tag_from_env + return container_tag = "bootc-image-builder-test" subprocess.check_call([ diff --git a/bootc-image-builder/test/test_build_disk.py b/bootc-image-builder/test/test_build_disk.py index f1d11b70b6..d2b2a2a2f4 100644 --- a/bootc-image-builder/test/test_build_disk.py +++ b/bootc-image-builder/test/test_build_disk.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import json import os import pathlib @@ -36,7 +38,7 @@ class ImageBuildResult(NamedTuple): img_type: str - img_path: str + img_path: pathlib.Path img_arch: str container_ref: str build_container_ref: str @@ -212,7 +214,7 @@ def sign_container_image(gpg_conf: GPGConf, registry_conf: RegistryConf, contain @pytest.fixture(name="image_type", scope="session") -# pylint: disable=too-many-arguments +# pylint: disable=too-many-arguments,too-many-positional-arguments def image_type_fixture(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf, registry_conf): """ Build an image inside the passed build_container and return an @@ -228,7 +230,7 @@ def image_type_fixture(shared_tmpdir, build_container, request, force_aws_upload @pytest.fixture(name="images", scope="session") -# pylint: disable=too-many-arguments +# pylint: disable=too-many-arguments,too-many-positional-arguments def images_fixture(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf, registry_conf): """ Build one or more images inside the passed build_container and return an @@ -241,7 +243,7 @@ def images_fixture(shared_tmpdir, build_container, request, force_aws_upload, gp # XXX: refactor -# pylint: disable=too-many-locals,too-many-branches,too-many-statements,too-many-arguments +# pylint: disable=too-many-locals,too-many-branches,too-many-statements,too-many-arguments,too-many-positional-arguments @contextmanager def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf, registry_conf): """ diff --git a/bootc-image-builder/test/test_build_iso.py b/bootc-image-builder/test/test_build_iso.py index ce8ba97f9b..54b21ec144 100644 --- a/bootc-image-builder/test/test_build_iso.py +++ b/bootc-image-builder/test/test_build_iso.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import json import os import pathlib @@ -16,9 +18,9 @@ from containerbuild import ( # pylint: disable=unused-import build_container_fixture, make_container) from test_build_disk import ImageBuildResult # pylint: disable=unused-import -from test_build_disk import (assert_kernel_args, gpg_conf_fixture, - image_type_fixture, registry_conf_fixture, - shared_tmpdir_fixture) +from test_build_disk import ( # pylint: disable=unused-import + assert_kernel_args, gpg_conf_fixture, image_type_fixture, + registry_conf_fixture, shared_tmpdir_fixture) from testcases import gen_testcases ISO_BOOT_TIMEOUT = 1800 diff --git a/bootc-image-builder/test/test_build_pxe.py b/bootc-image-builder/test/test_build_pxe.py index eb9632fc36..cedce04bcf 100644 --- a/bootc-image-builder/test/test_build_pxe.py +++ b/bootc-image-builder/test/test_build_pxe.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import json import os import pathlib @@ -42,7 +44,7 @@ def finish_request(self, request, client_address): SimpleHTTPRequestHandler(request, client_address, self, directory=self.directory) -# pylint: disable=too-many-arguments,too-many-locals,unused-argument +# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals,unused-argument def boot_qemu_pxe(arch, pxe_tar_path, container_ref, username, password, ssh_key_path, keep=False): with ExitStack() as cm: # unpack the tar and create a combined image diff --git a/bootc-image-builder/test/test_flake8.py b/bootc-image-builder/test/test_flake8.py deleted file mode 100644 index bfd79219ee..0000000000 --- a/bootc-image-builder/test/test_flake8.py +++ /dev/null @@ -1,11 +0,0 @@ -import os -import pathlib -import subprocess - - -def test_flake8(): - p = pathlib.Path(__file__).parent - # TODO: use all static checks from osbuild instead - subprocess.check_call( - ["flake8", "--ignore=E402,F811,F401", "--max-line-length=120", - os.fspath(p)]) diff --git a/bootc-image-builder/test/test_manifest.py b/bootc-image-builder/test/test_manifest.py index f75a7aa7e0..85418e42be 100644 --- a/bootc-image-builder/test/test_manifest.py +++ b/bootc-image-builder/test/test_manifest.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme # pylint: disable=too-many-lines import base64 @@ -9,6 +11,7 @@ import textwrap import pytest + import testutil from containerbuild import build_container_fixture as _ from containerbuild import make_container diff --git a/bootc-image-builder/test/test_progress.py b/bootc-image-builder/test/test_progress.py index 383c743096..64cbd054de 100644 --- a/bootc-image-builder/test/test_progress.py +++ b/bootc-image-builder/test/test_progress.py @@ -1,10 +1,11 @@ import subprocess import pytest + import testutil -from containerbuild import (build_container_fixture, - build_erroring_container_fixture, - build_fake_container_fixture) +from containerbuild import ( # pylint: disable=unused-import + build_container_fixture, build_erroring_container_fixture, + build_fake_container_fixture) # pylint: disable=unused-import,duplicate-code from test_opts import container_storage_fixture diff --git a/bootc-image-builder/test/test_pylint.py b/bootc-image-builder/test/test_pylint.py deleted file mode 100644 index eb1c386bba..0000000000 --- a/bootc-image-builder/test/test_pylint.py +++ /dev/null @@ -1,18 +0,0 @@ -import pathlib -import subprocess - - -def test_pylint(): - p = pathlib.Path(__file__).parent - subprocess.check_call( - ["pylint", - "--disable=fixme", - "--disable=missing-class-docstring", - "--disable=missing-module-docstring", - "--disable=missing-function-docstring", - "--disable=too-many-instance-attributes", - # false positive because of "if yield else yield" in - # the "build_container" fixture, see - # https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/contextmanager-generator-missing-cleanup.html - "--disable=contextmanager-generator-missing-cleanup", - "--max-line-length=120"] + list(p.glob("*.py"))) diff --git a/bootc-image-builder/test/testcases.py b/bootc-image-builder/test/testcases.py index 5ef62fea8d..2e8a331552 100644 --- a/bootc-image-builder/test/testcases.py +++ b/bootc-image-builder/test/testcases.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import dataclasses import inspect import os diff --git a/bootc-image-builder/test/testutil.py b/bootc-image-builder/test/testutil.py index 096d8f6610..d385c99444 100644 --- a/bootc-image-builder/test/testutil.py +++ b/bootc-image-builder/test/testutil.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import os import pathlib import platform diff --git a/test/_test_build.py b/test/_test_build.py index ff0f81aa51..e66aa6554a 100644 --- a/test/_test_build.py +++ b/test/_test_build.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import json import os import pathlib @@ -93,6 +95,7 @@ def test_build_generates_rpmlist(tmp_path, build_container, shared_store): ("term", "[|]", "osbuild-stdout-output"), ]) @pytest.mark.skipif(os.getuid() != 0, reason="needs root") +# pylint: disable=too-many-arguments,too-many-positional-arguments def test_build_with_progress(tmp_path, build_fake_container, shared_store, progress, needle, forbidden): output_dir = tmp_path / "output" output_dir.mkdir() diff --git a/test/conftest.py b/test/conftest.py index 3335d15542..f332d21b83 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,11 +1,3 @@ -import random -import string -import subprocess -import textwrap - -import pytest - - def pytest_configure(config): config.addinivalue_line( "markers", "images_integration" diff --git a/test/scripts/imgtestlib/core.py b/test/scripts/imgtestlib/core.py index e0d23a1ee2..650c51db7f 100644 --- a/test/scripts/imgtestlib/core.py +++ b/test/scripts/imgtestlib/core.py @@ -7,7 +7,7 @@ from typing import Dict from .build import get_manifest_id -from .cache import dl_build_info, gen_build_info_dir_path_prefix, touch_s3 +from .cache import dl_build_info, gen_build_info_dir_path_prefix from .gitlab import log_section from .run import runcmd from .testenv import (get_bib_ref, get_ci_runner_for, host_container_arch, diff --git a/test/scripts/imgtestlib/vm.py b/test/scripts/imgtestlib/vm.py index 31ff228146..b768e764da 100644 --- a/test/scripts/imgtestlib/vm.py +++ b/test/scripts/imgtestlib/vm.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import abc import os import pathlib @@ -12,6 +14,12 @@ import uuid from io import StringIO +try: + # Allow importing the module without boto3 since most times it's not needed. + from qemu import qmp +except ImportError: + qmp = None + try: # Allow importing the module without boto3 since most times it's not needed. import boto3 @@ -78,7 +86,7 @@ def _ensure_ssh(self, user, password="", keyfile=None): try: self._run("true", user=user, password=password, keyfile=keyfile) return - except Exception as e: + except Exception as e: # pylint: disable=broad-exception-caught print(f"ssh not ready {i+1}/{ssh_ready_n_retries}: {e}", file=sys.stderr) time.sleep(ssh_ready_wait_sec) raise RuntimeError(f"no ssh after {ssh_ready_n_retries} retries of {ssh_ready_wait_sec}s") @@ -116,6 +124,7 @@ def _run(self, args, user, password="", keyfile=None): ret.check_returncode() return ret + # pylint: disable=too-many-arguments,too-many-positional-arguments def scp(self, src, dst, user, password="", keyfile=None): self._ensure_ssh(user, password, keyfile) scp_cmd = self._sshpass(password) + [ @@ -155,7 +164,9 @@ def find_ovmf(): raise ValueError("cannot find a OVMF bios") +# pylint: disable=too-many-instance-attributes class QEMU(VM): + # pylint: disable=too-many-arguments,too-many-positional-arguments def __init__(self, img, arch="", snapshot=True, cdrom=None, extra_args=None, memory="2048"): super().__init__() self._img = pathlib.Path(img) @@ -181,6 +192,7 @@ def _num_cores(self): """ return os.cpu_count() or 1 + # pylint: disable=too-many-branches def _gen_qemu_cmdline(self, snapshot, use_ovmf): virtio_scsi_hd = [ "-device", "virtio-scsi-pci,id=scsi", @@ -293,10 +305,8 @@ def _wait_qmp_socket(self, timeout_sec): raise TimeoutError(f"no {self._qmp_socket} after {timeout_sec} seconds") def wait_qmp_event(self, qmp_event, timeout_sec=120): - # import lazy to avoid requiring it for all operations - import qmp # pylint: disable=import-outside-toplevel self._wait_qmp_socket(30) - mon = qmp.QEMUMonitorProtocol(os.fspath(self._qmp_socket)) + mon = qmp.legacy.QEMUMonitorProtocol(os.fspath(self._qmp_socket)) mon.connect() start = time.monotonic() while True: diff --git a/test/scripts/install-dependencies b/test/scripts/install-dependencies index ec4348eca6..1cb6cf36d1 100755 --- a/test/scripts/install-dependencies +++ b/test/scripts/install-dependencies @@ -10,7 +10,6 @@ install_deps() { btrfs-progs-devel \ cloud-utils \ device-mapper-devel \ - flake8 \ gcc \ go \ gpgme-devel \ diff --git a/test/scripts/test_imgtestlib.py b/test/scripts/test_imgtestlib.py index bfc58a9d3d..30dd24d3f3 100644 --- a/test/scripts/test_imgtestlib.py +++ b/test/scripts/test_imgtestlib.py @@ -533,6 +533,7 @@ def test_ssh_calls_retries(mocked_sleep, tmp_path, monkeypatch, capsys): for i in range(30) ]) + "\n" + def test_wait_ssh_ready_timeout(): vm = MockVM() vm.set_ssh("localhost", testlib.vm.get_free_port()) diff --git a/test/test_build_integration.py b/test/test_build_integration.py index a1c1fd77b8..4240b0315b 100644 --- a/test/test_build_integration.py +++ b/test/test_build_integration.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import os import platform import subprocess From 96e3a3c831f68030c9471b0d6f88f5af852abd6b Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Wed, 8 Jul 2026 16:36:35 +0200 Subject: [PATCH 13/25] imgtestlib: don't use star imports in __init__.py Make every submodule a relative import and update all external imports. --- test/scripts/boot-image | 2 +- test/scripts/build-image | 2 +- test/scripts/check-build-coverage | 4 +- test/scripts/dl-image-build-cache | 12 +++--- test/scripts/dl-one-image-build-cache | 8 ++-- test/scripts/generate-build-config | 24 +++++------ test/scripts/generate-gitlab-ci | 8 ++-- test/scripts/generate-ostree-build-config | 50 +++++++++++----------- test/scripts/imgtestlib/__init__.py | 9 +--- test/scripts/setup-osbuild-repo | 6 +-- test/scripts/test_imgtestlib.py | 51 ++++++++++++----------- test/scripts/update-schutzfile-bib | 8 ++-- test/scripts/update-schutzfile-osbuild | 4 +- test/scripts/upload-results | 2 +- test/scripts/validate-config-list | 2 +- test/test_build_integration.py | 7 ++-- test/test_cross_arch_integration.py | 2 +- 17 files changed, 98 insertions(+), 103 deletions(-) diff --git a/test/scripts/boot-image b/test/scripts/boot-image index c39d32ae4c..d8b987d488 100755 --- a/test/scripts/boot-image +++ b/test/scripts/boot-image @@ -18,7 +18,7 @@ def main(): build_config_path = args.config keep = args.keep_booted - testlib.boot_image(search_path, build_config_path, keep) + testlib.boot.boot_image(search_path, build_config_path, keep) if __name__ == "__main__": diff --git a/test/scripts/build-image b/test/scripts/build-image index cc997ff491..368cef5cb0 100755 --- a/test/scripts/build-image +++ b/test/scripts/build-image @@ -21,7 +21,7 @@ def main(): arch = args.arch config_path = args.config - testlib.build_image(distro, arch, image_type, config_path) + testlib.build.build_image(distro, arch, image_type, config_path) if __name__ == "__main__": diff --git a/test/scripts/check-build-coverage b/test/scripts/check-build-coverage index c2cec170c8..c39d1fcf14 100755 --- a/test/scripts/check-build-coverage +++ b/test/scripts/check-build-coverage @@ -43,7 +43,7 @@ def check_build_coverage(cachedir): built_pr.add((distro, arch, image, config, commit)) all_combos = set() - for config in testlib.list_images(arches=["x86_64", "aarch64"]): + for config in testlib.core.list_images(arches=["x86_64", "aarch64"]): distro = config["distro"] arch = config["arch"] image = config["image-type"] @@ -60,7 +60,7 @@ def main(): parser.add_argument("cachedir", type=str, help="path to download the build test cache") args = parser.parse_args() cachedir = args.cachedir - _, ok = testlib.dl_build_info(cachedir) + _, ok = testlib.cache.dl_build_info(cachedir) # fail the job if the sync failed if not ok: sys.exit(1) diff --git a/test/scripts/dl-image-build-cache b/test/scripts/dl-image-build-cache index 0219a21ed2..52c7c6627b 100755 --- a/test/scripts/dl-image-build-cache +++ b/test/scripts/dl-image-build-cache @@ -117,8 +117,8 @@ def main(): if args.image_type and args.skip_image_type: parser.error("--image-type and --skip-image-type are mutually exclusive") - runner_distro = testlib.get_common_ci_runner_distro() - osbuild_ref = testlib.get_osbuild_commit(runner_distro) + runner_distro = testlib.testenv.get_common_ci_runner_distro() + osbuild_ref = testlib.testenv.get_osbuild_commit(runner_distro) if osbuild_ref is None: raise RuntimeError(f"Failed to determine osbuild commit for {runner_distro} from the Schutzfile") @@ -128,7 +128,7 @@ def main(): with tempfile.TemporaryDirectory() as tmpdir: print("πŸ“œ Generating current manifests to determine their IDs") - err = testlib.gen_manifests(tmpdir, arches=args.arch, distros=args.distro, images=args.image_type) + err = testlib.core.gen_manifests(tmpdir, arches=args.arch, distros=args.distro, images=args.image_type) # print stderr in case there were errors or warnings about skipped configurations # but filter out the annoying ones stderr = err.decode().splitlines() @@ -138,7 +138,7 @@ def main(): if "Failed to load consumer certs" in line: continue print(line) - manifest_gen_data = testlib.read_manifests(tmpdir) + manifest_gen_data = testlib.core.read_manifests(tmpdir) build_cache_infos = gen_manifest_data_to_build_cache_info(manifest_gen_data, args.config, args.skip_image_type) @@ -162,9 +162,9 @@ def main(): config = build_cache_info["config"] manifest_id = build_cache_info["manifest-id"] - target_dir = os.path.join(output_dir, testlib.gen_build_name(distro, arch, image_type, config)) + target_dir = os.path.join(output_dir, testlib.build.gen_build_name(distro, arch, image_type, config)) - out, dl_ok = testlib.dl_build_cache( + out, dl_ok = testlib.cache.dl_build_cache( target_dir, distro, arch, osbuild_ref, runner_distro, manifest_id, s3_include_only) if not dl_ok: failed_downloads.append(build_cache_info) diff --git a/test/scripts/dl-one-image-build-cache b/test/scripts/dl-one-image-build-cache index c1f4704655..4094b7d0f5 100755 --- a/test/scripts/dl-one-image-build-cache +++ b/test/scripts/dl-one-image-build-cache @@ -39,7 +39,7 @@ def main(): build_info_dir = os.path.dirname(args.build_info) if args.build_info else build_dir print(f"πŸ“œ Reading 'info.json' from {build_info_dir}") - build_info = testlib.read_build_info(build_info_dir) + build_info = testlib.build.read_build_info(build_info_dir) distro = build_info["distro"] arch = build_info["arch"] @@ -48,7 +48,7 @@ def main(): runner_distro = build_info.get("runner-distro") if runner_distro is None: - runner_distro = testlib.get_common_ci_runner_distro() + runner_distro = testlib.testenv.get_common_ci_runner_distro() print("⚠️ Runner distro not found in the build info. " + f"Using the CI runner distro from the current branch: {runner_distro}", file=sys.stderr) @@ -59,14 +59,14 @@ def main(): print(f" osbuild-ref: {osbuild_ref}") print(f" runner-distro: {runner_distro}") - out, dl_ok = testlib.dl_build_cache(build_dir, distro, arch, osbuild_ref, runner_distro, manifest_id) + out, dl_ok = testlib.cache.dl_build_cache(build_dir, distro, arch, osbuild_ref, runner_distro, manifest_id) print(out) if not dl_ok: print("❌ Failed to download the image build cache", file=sys.stderr) sys.exit(1) print("βœ… Successfully downloaded the image build cache") - testlib.touch_s3(distro, arch, manifest_id, osbuild_ref, runner_distro) + testlib.cache.touch_s3(distro, arch, manifest_id, osbuild_ref, runner_distro) if __name__ == "__main__": diff --git a/test/scripts/generate-build-config b/test/scripts/generate-build-config index b0e9b9c87d..b52717723e 100755 --- a/test/scripts/generate-build-config +++ b/test/scripts/generate-build-config @@ -25,7 +25,7 @@ build/{distro}/{arch}/{image_type}/{config_name}: """ -@testlib.log_section("Generating manifests") +@testlib.gitlab.log_section("Generating manifests") def generate_manifests(outputdir, distro, arch): """ Generate all manifest using the default config list and return a dictionary mapping each manifest file to the @@ -38,7 +38,7 @@ def generate_manifests(outputdir, distro, arch): distros = [distro] print(f"πŸ—’οΈ Generating all manifests using the default config list for {target}") - err = testlib.gen_manifests(outputdir, arches=[arch], distros=distros) + err = testlib.core.gen_manifests(outputdir, arches=[arch], distros=distros) # print stderr in case there were errors or warnings about skipped configurations # but filter out the annoying ones @@ -51,7 +51,7 @@ def generate_manifests(outputdir, distro, arch): print(line) print("βœ… Manifest generation done!\n") - return testlib.read_manifests(outputdir) + return testlib.core.read_manifests(outputdir) def generate_configs(build_requests, pipeline_file): @@ -64,13 +64,13 @@ def generate_configs(build_requests, pipeline_file): config_name = config["name"] - build_name = testlib.gen_build_name(distro, arch, image_type, config_name) + build_name = testlib.build.gen_build_name(distro, arch, image_type, config_name) image_path = f"./build/{build_name}" - runner = testlib.get_ci_runner_for(distro, arch, image_type) - tag = testlib.get_tag_for(runner) + runner = testlib.testenv.get_ci_runner_for(distro, arch, image_type) + tag = testlib.core.get_tag_for(runner) - config_path = os.path.join(testlib.CONFIGS_PATH, config_name+".json") + config_path = os.path.join(testlib.core.CONFIGS_PATH, config_name+".json") pipeline_file.write(JOB_TEMPLATE.format(distro=distro, arch=arch, image_type=image_type, runner=runner, tag=tag, config_name=config_name, config=config_path, @@ -80,26 +80,26 @@ def generate_configs(build_requests, pipeline_file): def main(): - parser = testlib.clargs() + parser = testlib.core.clargs() args = parser.parse_args() config_path = args.config distro = args.distro arch = args.arch - testlib.check_config_names() + testlib.core.check_config_names() with TemporaryDirectory() as manifest_dir: manifests = generate_manifests(manifest_dir, distro, arch) - build_requests = testlib.filter_builds(manifests, distro=distro, arch=arch) + build_requests = testlib.core.filter_builds(manifests, distro=distro, arch=arch) with open(config_path, "w", encoding="utf-8") as config_file: if len(build_requests) == 0: print("⚫ No manifest changes detected. Generating null config.") - config_file.write(testlib.NULL_CONFIG) + config_file.write(testlib.core.NULL_CONFIG) return - config_file.write(testlib.BASE_CONFIG) + config_file.write(testlib.core.BASE_CONFIG) generate_configs(build_requests, config_file) diff --git a/test/scripts/generate-gitlab-ci b/test/scripts/generate-gitlab-ci index 070852785c..f41845a857 100755 --- a/test/scripts/generate-gitlab-ci +++ b/test/scripts/generate-gitlab-ci @@ -7,7 +7,7 @@ import imgtestlib as testlib ARCHITECTURES = ["x86_64", "aarch64"] MANIFEST_ONLY_ARCHES = ["ppc64le", "s390x"] -RUNNER = testlib.get_common_ci_runner() +RUNNER = testlib.testenv.get_common_ci_runner() OSTREE_DISTROS = ["fedora", "rhel-8", "rhel-9", "centos-9"] @@ -145,7 +145,7 @@ def sort_configs(configs): def main(): config_path = sys.argv[1] - images = testlib.list_images(arches=ARCHITECTURES) + images = testlib.core.list_images(arches=ARCHITECTURES) combos = set() gen_stage = [] @@ -184,7 +184,7 @@ def main(): arch=img["arch"], runner=RUNNER)) - man_only_images = testlib.list_images(arches=MANIFEST_ONLY_ARCHES) + man_only_images = testlib.core.list_images(arches=MANIFEST_ONLY_ARCHES) man_gen_stage = [] for img in sort_configs(man_only_images): combo = (img["distro"], img["arch"]) @@ -196,7 +196,7 @@ def main(): with open(config_path, "w", encoding="utf-8") as config_file: config_file.write(BASE_CONFIG.format(runner=RUNNER)) - config_file.write(testlib.BASE_CONFIG) + config_file.write(testlib.core.BASE_CONFIG) config_file.write("\n".join(gen_stage)) config_file.write("\n".join(trigger_stage)) config_file.write("\n".join(ostree_gen_stage)) diff --git a/test/scripts/generate-ostree-build-config b/test/scripts/generate-ostree-build-config index e8ec3c512c..f2256ea6d8 100755 --- a/test/scripts/generate-ostree-build-config +++ b/test/scripts/generate-ostree-build-config @@ -29,7 +29,7 @@ build/{distro}/{arch}/{image_type}/{config_name}: def read_config_list(): - with open(testlib.CONFIG_LIST, "r", encoding="utf-8") as config_list_file: + with open(testlib.core.CONFIG_LIST, "r", encoding="utf-8") as config_list_file: return json.load(config_list_file) @@ -43,7 +43,7 @@ def configs_with_deps(configs, distro=None, arch=None): Filter on distro and arch if specified. """ with_deps = [] - config_list_dir = os.path.abspath(os.path.dirname(testlib.CONFIG_LIST)) + config_list_dir = os.path.abspath(os.path.dirname(testlib.core.CONFIG_LIST)) for item in configs: path = item["path"] filters = item["filters"] @@ -105,8 +105,8 @@ def gen_dependency_manifests(config_list, distro, arch, outputdir): with NamedTemporaryFile(mode="w") as tmpfile: json.dump(dep_config_list, tmpfile) tmpfile.flush() - err = testlib.gen_manifests(outputdir, config_list=tmpfile.name, - distros=distros, arches=[arch], images=gen_image_types) + err = testlib.core.gen_manifests(outputdir, config_list=tmpfile.name, + distros=distros, arches=[arch], images=gen_image_types) # print stderr in case there were errors or warnings about skipped configurations # but filter out the annoying ones stderr = err.decode().splitlines() @@ -118,7 +118,7 @@ def gen_dependency_manifests(config_list, distro, arch, outputdir): print(line) print("βœ… Manifest generation done!\n") - return testlib.read_manifests(outputdir) + return testlib.core.read_manifests(outputdir) def gen_image_manifests(config_list, configs, distro, arch, outputdir): @@ -147,9 +147,9 @@ def gen_image_manifests(config_list, configs, distro, arch, outputdir): with open(config_list_path, "w", encoding="utf-8") as config_list_file: json.dump(config_list, config_list_file) - err = testlib.gen_manifests(outputdir, config_list=config_list_path, - arches=[arch], distros=distros, commits=True, - flatpaks=True, skip_no_config=True) + err = testlib.core.gen_manifests(outputdir, config_list=config_list_path, + arches=[arch], distros=distros, commits=True, + flatpaks=True, skip_no_config=True) # print stderr in case there were errors or warnings about skipped configurations # but filter out the annoying ones @@ -162,7 +162,7 @@ def gen_image_manifests(config_list, configs, distro, arch, outputdir): print(line) print("βœ… Manifest generation done!\n") - return testlib.read_manifests(outputdir) + return testlib.core.read_manifests(outputdir) def default_ref(distro, arch): @@ -217,9 +217,9 @@ def setup_dependencies(manifests, config_list, distro, arch): if distro: distros = [distro] arches = [arch] - image_configs = testlib.list_images(distros=distros, - arches=arches, - images=filters.get("image-types")) + image_configs = testlib.core.list_images(distros=distros, + arches=arches, + images=filters.get("image-types")) distros_to_skip = [] for skip in filters.get("skip-distros", []): @@ -234,9 +234,9 @@ def setup_dependencies(manifests, config_list, distro, arch): continue ic_arch = image_config["arch"] ic_image_type = image_config["image-type"] - dep_build_name = testlib.gen_build_name(ic_distro, ic_arch, dep_image_type, dep_config_name) + dep_build_name = testlib.build.gen_build_name(ic_distro, ic_arch, dep_image_type, dep_config_name) manifest_id = manifests[dep_build_name + ".json"]["id"] - container_s3_prefix = testlib.gen_build_info_s3_dir_path(distro, arch, manifest_id) + container_s3_prefix = testlib.cache.gen_build_info_s3_dir_path(distro, arch, manifest_id) container_s3_path = os.path.join(container_s3_prefix, "container", "container.tar") # start each container once on an incremental port @@ -284,19 +284,19 @@ def setup_dependencies(manifests, config_list, distro, arch): with TemporaryDirectory() as container_dir: print(f"⬇️ Downloading container archive from {container_s3_path}") container_archive = os.path.join(container_dir, "container.tar") - testlib.runcmd_nc(["aws", "s3", "cp", "--no-progress", container_s3_path, container_archive]) + testlib.run.runcmd_nc(["aws", "s3", "cp", "--no-progress", container_s3_path, container_archive]) print(f"πŸ“¦ Starting container oci-archive:{container_archive} {port}") # Running podman as root is not necessary, but currently it's failing with permission errors. # Run it with sudo for now until we figure out the issue. - cont_id, _ = testlib.runcmd(["sudo", "podman", "run", "-d", "--rm", f"-p{port}:8080", - f"oci-archive:{container_archive}"]) + cont_id, _ = testlib.run.runcmd(["sudo", "podman", "run", "-d", "--rm", f"-p{port}:8080", + f"oci-archive:{container_archive}"]) container_ids.append(cont_id.strip().decode()) yield new_config_list, new_configs, container_configs finally: if container_ids: print("πŸ“¦ Stopping containers") - out, _ = testlib.runcmd(["sudo", "podman", "stop", *container_ids]) + out, _ = testlib.run.runcmd(["sudo", "podman", "stop", *container_ids]) print(out.decode()) @@ -311,7 +311,7 @@ def generate_configs(build_requests, container_configs, pipeline_file, configs_d config_name = config["name"] - build_name = testlib.gen_build_name(distro, arch, image_type, config_name) + build_name = testlib.build.gen_build_name(distro, arch, image_type, config_name) image_path = f"./build/{build_name}" # generate script line to pull and start container container = container_configs[config_name] @@ -328,7 +328,7 @@ def generate_configs(build_requests, container_configs, pipeline_file, configs_d run_container_cmd = f"sudo podman run -d --rm -p{container_port}:8080 oci-archive:container.tar" pipeline_file.write(JOB_TEMPLATE.format(distro=distro, arch=arch, image_type=image_type, - runner=testlib.get_common_ci_runner(), + runner=testlib.testenv.get_common_ci_runner(), config_name=config_name, config=build_config_path, dl_container=dl_container_cmd, start_container=run_container_cmd, @@ -338,7 +338,7 @@ def generate_configs(build_requests, container_configs, pipeline_file, configs_d def main(): - parser = testlib.clargs() + parser = testlib.core.clargs() parser.add_argument("build_configs", type=str, help="directory to write individual build configs") args = parser.parse_args() @@ -347,7 +347,7 @@ def main(): distro = args.distro arch = args.arch - testlib.check_config_names() + testlib.core.check_config_names() config_list = configs_with_deps(read_config_list(), distro, arch) # filtered config list: only configs with deps @@ -360,15 +360,15 @@ def main(): manifest_dir = os.path.join(cache_root, "manifests") manifests = gen_image_manifests(pull_config_list, pull_configs, distro, arch, manifest_dir) - build_requests = testlib.filter_builds(manifests, distro=distro, arch=arch, skip_ostree_pull=False) + build_requests = testlib.core.filter_builds(manifests, distro=distro, arch=arch, skip_ostree_pull=False) with open(config_path, "w", encoding="utf-8") as config_file: if len(build_requests) == 0: print("⚫ No manifest changes detected. Generating null config.") - config_file.write(testlib.NULL_CONFIG) + config_file.write(testlib.core.NULL_CONFIG) return - config_file.write(testlib.BASE_CONFIG) + config_file.write(testlib.core.BASE_CONFIG) generate_configs(build_requests, containers, config_file, configs_dir) diff --git a/test/scripts/imgtestlib/__init__.py b/test/scripts/imgtestlib/__init__.py index 3a6457a7ec..5ad4b175b5 100644 --- a/test/scripts/imgtestlib/__init__.py +++ b/test/scripts/imgtestlib/__init__.py @@ -1,8 +1 @@ -from .boot import * -from .build import * -from .cache import * -from .core import * -from .gitlab import * -from .run import * -from .testenv import * -from .vm import * +from . import boot, build, cache, core, gitlab, run, testenv, vm diff --git a/test/scripts/setup-osbuild-repo b/test/scripts/setup-osbuild-repo index 859970fe16..a1a219ba28 100755 --- a/test/scripts/setup-osbuild-repo +++ b/test/scripts/setup-osbuild-repo @@ -49,10 +49,10 @@ def write_repo(commit, distro_version): repofile.write(REPO_TEMPLATE.format(commit=commit, baseurl=repo_baseurl)) -@testlib.log_section("Setting up osbuild repo") +@testlib.gitlab.log_section("Setting up osbuild repo") def main(): - distro_version = testlib.get_host_distro() - commit_id = testlib.get_osbuild_commit(distro_version) + distro_version = testlib.testenv.get_host_distro() + commit_id = testlib.testenv.get_osbuild_commit(distro_version) if not commit_id: print(f"Error: {distro_version} does not have the osbuild commit ID defined in the Schutzfile") sys.exit(1) diff --git a/test/scripts/test_imgtestlib.py b/test/scripts/test_imgtestlib.py index 30dd24d3f3..465ce6cfcf 100644 --- a/test/scripts/test_imgtestlib.py +++ b/test/scripts/test_imgtestlib.py @@ -23,14 +23,14 @@ def can_sudo_nopw() -> bool: def test_runcmd(): - stdout, stderr = testlib.runcmd(["/bin/echo", "hello"]) + stdout, stderr = testlib.run.runcmd(["/bin/echo", "hello"]) assert stdout == b"hello\n" assert stderr == b"" def test_runcmd_env(): os.environ["RUNCMD_GLOBAL_TEST_VAR"] = "global test value" - stdout, stderr = testlib.runcmd(["env"], extra_env={"RUNCMD_TEST_VAR": "the test value"}) + stdout, stderr = testlib.run.runcmd(["env"], extra_env={"RUNCMD_TEST_VAR": "the test value"}) assert b"RUNCMD_TEST_VAR=the test value\n" in stdout, "extra env var not set" assert b"RUNCMD_GLOBAL_TEST_VAR=global test value\n" in stdout, "global env vars not preserved" assert stderr == b"" @@ -38,7 +38,7 @@ def test_runcmd_env(): def test_read_seed(): # check that it's read without error - no need to test the value itself - seed_env = testlib.rng_seed_env() + seed_env = testlib.testenv.rng_seed_env() assert "OSBUILD_TESTING_RNG_SEED" in seed_env @@ -121,7 +121,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): # we need to patch the functions that were imported into the cache namespace, not the originals in .testenv with patch("imgtestlib.cache.get_host_distro", return_value="fedora-999"), \ patch("imgtestlib.cache.get_osbuild_commit", return_value="abcdef123456"): - assert testlib.gen_build_info_dir_path_prefix(**kwargs) == expected + assert testlib.cache.gen_build_info_dir_path_prefix(**kwargs) == expected @pytest.mark.parametrize("kwargs,expected", ( @@ -133,7 +133,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "arch": "aarch64", "manifest_id": "abc123" }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/aarch64/manifest-id-abc123/", ), ( @@ -143,7 +143,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "distro": "fedora-41", "arch": "aarch64", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/aarch64/", ), ( @@ -152,7 +152,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "runner_distro": "fedora-41", "distro": "fedora-41", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/", ), ( @@ -160,7 +160,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "osbuild_ref": "abcdef123456", "runner_distro": "fedora-41", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/", ), # Optional arg 'distro' not specified, thus following optional args 'arch' and 'manifest_id' are ignored @@ -171,7 +171,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "arch": "aarch64", "manifest_id": "abc123" }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/", ), # Optional arg 'arch' not specified, thus following optional arg 'manifest_id' is ignored @@ -182,7 +182,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "distro": "fedora-41", "manifest_id": "abc123" }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/", ), # default osbuild_ref @@ -190,26 +190,26 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): { "runner_distro": "fedora-41", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/" + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/" ), # default runner_distro ( { "osbuild_ref": "abc123", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + "/osbuild-ref-abc123/runner-fedora-999/" + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abc123/runner-fedora-999/" ), # default osbuild_ref and runner_distro ( {}, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-999/" + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-999/" ), )) def test_gen_build_info_s3_dir_path(kwargs, expected): # we need to patch the functions that were imported into the cache namespace, not the originals in .testenv with patch("imgtestlib.cache.get_host_distro", return_value="fedora-999"), \ patch("imgtestlib.cache.get_osbuild_commit", return_value="abcdef123456"): - assert testlib.gen_build_info_s3_dir_path(**kwargs) == expected + assert testlib.cache.gen_build_info_s3_dir_path(**kwargs) == expected test_container = "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/manifest-list-test" @@ -239,8 +239,8 @@ def test_gen_build_info_s3_dir_path(kwargs, expected): def test_skopeo_inspect_id_manifest_list(arch): transport = "docker://" image_id = image_ids[arch] - assert testlib.skopeo_inspect_id(f"{transport}{test_container}:latest", arch) == image_id - assert testlib.skopeo_inspect_id(f"{transport}{test_container}@{manifest_list_digest}", arch) == image_id + assert testlib.core.skopeo_inspect_id(f"{transport}{test_container}:latest", arch) == image_id + assert testlib.core.skopeo_inspect_id(f"{transport}{test_container}@{manifest_list_digest}", arch) == image_id @pytest.mark.parametrize("arch", TEST_ARCHES) @@ -249,7 +249,7 @@ def test_skopeo_inspect_image_manifest(arch): manifest_id = manifest_ids[arch] image_id = image_ids[arch] # arch arg to skopeo_inspect_id doesn't matter here - assert testlib.skopeo_inspect_id(f"{transport}{test_container}@{manifest_id}", arch) == image_id + assert testlib.core.skopeo_inspect_id(f"{transport}{test_container}@{manifest_id}", arch) == image_id @pytest.mark.skipif(not can_sudo_nopw(), reason="requires passwordless sudo") @@ -259,10 +259,11 @@ def test_skopeo_inspect_localstore(arch): transport = "containers-storage:" image = "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/manifest-list-test:latest" with tempfile.TemporaryDirectory() as tmpdir: - testlib.runcmd(["sudo", "podman", "pull", f"--arch={arch}", "--storage-driver=vfs", f"--root={tmpdir}", image]) + testlib.run.runcmd(["sudo", "podman", "pull", + f"--arch={arch}", "--storage-driver=vfs", f"--root={tmpdir}", image]) # arch arg to skopeo_inspect_id doesn't matter here - assert testlib.skopeo_inspect_id(f"{transport}[vfs@{tmpdir}]{image}", arch) == image_ids[arch] + assert testlib.core.skopeo_inspect_id(f"{transport}[vfs@{tmpdir}]{image}", arch) == image_ids[arch] def test_find_image_file_single_export(): @@ -287,7 +288,7 @@ def test_find_image_file_single_export(): f.write("fake image") # Test that it finds the correct file - result = testlib.find_image_file(tmpdir) + result = testlib.core.find_image_file(tmpdir) assert result == image_file @@ -315,7 +316,7 @@ def test_find_image_file_multiple_pipelines_one_export(): f.write("fake archive") # Test that it finds the correct file from the exported pipeline - result = testlib.find_image_file(tmpdir) + result = testlib.core.find_image_file(tmpdir) assert result == image_file @@ -335,7 +336,7 @@ def test_find_image_file_no_export_directory(): # Don't create any export directories with pytest.raises(RuntimeError, match="Expected exactly one exported pipeline directory"): - testlib.find_image_file(tmpdir) + testlib.core.find_image_file(tmpdir) def test_find_image_file_multiple_export_directories(): @@ -362,7 +363,7 @@ def test_find_image_file_multiple_export_directories(): # Should raise error about multiple export directories with pytest.raises(RuntimeError, match="Expected exactly one exported pipeline directory"): - testlib.find_image_file(tmpdir) + testlib.core.find_image_file(tmpdir) def test_find_image_file_no_files_in_export(): @@ -385,7 +386,7 @@ def test_find_image_file_no_files_in_export(): # Should raise error about no files with pytest.raises(RuntimeError, match="Expected exactly one file in export directory"): - testlib.find_image_file(tmpdir) + testlib.core.find_image_file(tmpdir) def test_find_image_file_multiple_files_in_export(): @@ -411,7 +412,7 @@ def test_find_image_file_multiple_files_in_export(): # Should raise error about multiple files with pytest.raises(RuntimeError, match="Expected exactly one file in export directory"): - testlib.find_image_file(tmpdir) + testlib.core.find_image_file(tmpdir) def test_get_free_port(): diff --git a/test/scripts/update-schutzfile-bib b/test/scripts/update-schutzfile-bib index f24791ad1d..713e5c06f7 100755 --- a/test/scripts/update-schutzfile-bib +++ b/test/scripts/update-schutzfile-bib @@ -8,9 +8,9 @@ import imgtestlib as testlib def main(): ref = "quay.io/centos-bootc/bootc-image-builder:latest" cmd = ["skopeo", "inspect", "--raw", f"docker://{ref}"] - out, _ = testlib.runcmd(cmd) + out, _ = testlib.run.runcmd(cmd) data = json.loads(out) - if not testlib.is_manifest_list(data): + if not testlib.core.is_manifest_list(data): raise RuntimeError(f"{ref} is not a manifest list") checksum = hashlib.sha256(out).hexdigest() @@ -18,11 +18,11 @@ def main(): ref_no_tag = ":".join(ref.split(":")[:-1]) new_ref = f"{ref_no_tag}@{digest}" - with open(testlib.SCHUTZFILE, encoding="utf-8") as schutzfile: + with open(testlib.testenv.SCHUTZFILE, encoding="utf-8") as schutzfile: data = json.load(schutzfile) data["common"]["dependencies"]["bootc-image-builder"]["ref"] = new_ref - with open(testlib.SCHUTZFILE, "w", encoding="utf-8") as schutzfile: + with open(testlib.testenv.SCHUTZFILE, "w", encoding="utf-8") as schutzfile: json.dump(data, schutzfile, indent=2) diff --git a/test/scripts/update-schutzfile-osbuild b/test/scripts/update-schutzfile-osbuild index 19a59ab193..053a595b8b 100755 --- a/test/scripts/update-schutzfile-osbuild +++ b/test/scripts/update-schutzfile-osbuild @@ -36,7 +36,7 @@ def osbuild_main_commit_id(): def update_osbuild_commit_ids(new): - with open(testlib.SCHUTZFILE, encoding="utf-8") as schutzfile: + with open(testlib.testenv.SCHUTZFILE, encoding="utf-8") as schutzfile: data = json.load(schutzfile) unique_changes = [] @@ -50,7 +50,7 @@ def update_osbuild_commit_ids(new): unique_changes.append(change) data[distro].setdefault("dependencies", {}).setdefault("osbuild", {})["commit"] = new - with open(testlib.SCHUTZFILE, encoding="utf-8", mode="w") as schutzfile: + with open(testlib.testenv.SCHUTZFILE, encoding="utf-8", mode="w") as schutzfile: json.dump(data, schutzfile, indent=" ") with open("github_pr_body.txt", encoding="utf-8", mode="w") as pr_body: diff --git a/test/scripts/upload-results b/test/scripts/upload-results index 4eae11d43e..55803e286a 100755 --- a/test/scripts/upload-results +++ b/test/scripts/upload-results @@ -18,7 +18,7 @@ def main(): config_path = args.config arch = os.uname().machine - testlib.upload_results(distro, arch, image_type, config_path) + testlib.cache.upload_results(distro, arch, image_type, config_path) if __name__ == "__main__": diff --git a/test/scripts/validate-config-list b/test/scripts/validate-config-list index ffacecc4d0..ed982fdaec 100755 --- a/test/scripts/validate-config-list +++ b/test/scripts/validate-config-list @@ -56,7 +56,7 @@ def validate_build_config(config_list): arches = filters.get("arches", ["*"]) image_types = filters.get("image-types", ["*"]) - matches = testlib.list_images(distros=distros, arches=arches, images=image_types) + matches = testlib.core.list_images(distros=distros, arches=arches, images=image_types) if not matches: no_matches.append((config, filters)) diff --git a/test/test_build_integration.py b/test/test_build_integration.py index 4240b0315b..11f0514a7b 100644 --- a/test/test_build_integration.py +++ b/test/test_build_integration.py @@ -24,7 +24,8 @@ def _test_cases(): boot_tests = set() for tcase in all_test_cases: _, arch, image_type, _ = tcase.split(",") - if not (image_type in testlib.CAN_BOOT_TEST["*"] or image_type in testlib.CAN_BOOT_TEST.get(arch, [])): + if not (image_type in testlib.core.CAN_BOOT_TEST["*"] or + image_type in testlib.core.CAN_BOOT_TEST.get(arch, [])): continue # XXX: we need to filter further here, i.e. not all qemu tests can be run currently, e.g. # if sshd is missing (see boot_image.ensure_can_run_qemu_test - however this needs @@ -46,7 +47,7 @@ def test_build_only(distro, arch, image_type, config_name): config_path = f"test/configs/{config_name}.json" subprocess.check_call( ["./test/scripts/build-image", distro, image_type, config_path]) - build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name)) + build_dir = os.path.join("build", testlib.build.gen_build_name(distro, arch, image_type, config_name)) subprocess.check_call( ["./test/scripts/boot-image", build_dir, config_path]) @@ -58,6 +59,6 @@ def test_build_and_boot(distro, arch, image_type, config_name): config_path = f"test/configs/{config_name}.json" subprocess.check_call( ["./test/scripts/build-image", distro, image_type, config_path]) - build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name)) + build_dir = os.path.join("build", testlib.build.gen_build_name(distro, arch, image_type, config_name)) subprocess.check_call( ["./test/scripts/boot-image", build_dir, config_path]) diff --git a/test/test_cross_arch_integration.py b/test/test_cross_arch_integration.py index 04e917ea19..57260fbe6b 100644 --- a/test/test_cross_arch_integration.py +++ b/test/test_cross_arch_integration.py @@ -29,6 +29,6 @@ def test_build_boot_cross_arch_smoke(arch): config_path = f"test/configs/{config_name}.json" subprocess.check_call( ["./test/scripts/build-image", f"--arch={arch}", distro, image_type, config_path]) - build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name)) + build_dir = os.path.join("build", testlib.build.gen_build_name(distro, arch, image_type, config_name)) subprocess.check_call( ["./test/scripts/boot-image", build_dir, config_path]) From 50a65c65b8cb753aac1d1260618e22b48dcda1a9 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 15:13:20 +0200 Subject: [PATCH 14/25] github: pytest only ./test Run pytest only the tests under ./test as part of the main test workflow. The bootc-image-builder tests will be handled separately. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4459527740..b06589f4d2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -241,7 +241,7 @@ jobs: - name: Testing imgtestlib and test scripts run: | - python3 -m pytest -v -k "not images_integration" + python3 -m pytest -v -k "not images_integration" ./test python-lint: name: "🐍 Lint (test scripts)" From 5064cf7ed9616f2fa69145cadf4c8658ca90485c Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 17:48:30 +0200 Subject: [PATCH 15/25] test/install-dependencies: add bootc-image-builder test deps The bootc-image-builder tests call 'ip route list default' to connect to a local container registry for testing. Some environments (Fedora container on GitHub) don't include the tool by default. sshpass is used in vm boot tests. --- test/scripts/install-dependencies | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/scripts/install-dependencies b/test/scripts/install-dependencies index 1cb6cf36d1..93d90e54f7 100755 --- a/test/scripts/install-dependencies +++ b/test/scripts/install-dependencies @@ -13,6 +13,7 @@ install_deps() { gcc \ go \ gpgme-devel \ + iproute \ krb5-devel \ libvirt-devel \ lorax \ @@ -30,6 +31,7 @@ install_deps() { python3-pytest \ python3-qemu-qmp \ python3-scp \ + sshpass \ xz \ yamllint } From 371ebc214ec738e9a101d9f312c643c1abda02d0 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 17:50:32 +0200 Subject: [PATCH 16/25] bootc-image-builder/test: drop memory limits from podman run calls When testing locally, many builds would fail with exit code 137 (sigkill, possibly OOM). --- bootc-image-builder/test/testutil.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bootc-image-builder/test/testutil.py b/bootc-image-builder/test/testutil.py index d385c99444..9817f36b10 100644 --- a/bootc-image-builder/test/testutil.py +++ b/bootc-image-builder/test/testutil.py @@ -178,8 +178,6 @@ def maybe_create_disk_customizations(cfg, tc): "--privileged", "-v", "/var/lib/containers/storage:/var/lib/containers/storage", "--security-opt", "label=type:unconfined_t", - # ensure we run in reasonable memory limits - "--memory=8g", "--memory-swap=8g", ] From 55af73510e9f16ddb2a9b05b9537577c614ee445 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 21:17:36 +0200 Subject: [PATCH 17/25] bootc-image-builder: disable boot tests for now Disable bootc-image-builder boot tests temporarily. I'd like to set these up to run like the other boot tests on rhos, gcp, or the new aws runners with kvm [1]. The current setup on GitHub is having issues and is hard to debug. [1] https://github.com/osbuild/gitlab-ci-terraform/pull/245 --- bootc-image-builder/test/test_build_disk.py | 1 + bootc-image-builder/test/test_build_iso.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/bootc-image-builder/test/test_build_disk.py b/bootc-image-builder/test/test_build_disk.py index d2b2a2a2f4..96aefdeb5b 100644 --- a/bootc-image-builder/test/test_build_disk.py +++ b/bootc-image-builder/test/test_build_disk.py @@ -556,6 +556,7 @@ def assert_kernel_args(test_vm, image_type): @pytest.mark.skipif(platform.system() != "Linux", reason="boot test only runs on linux right now") @pytest.mark.parametrize("image_type", gen_testcases("qemu-boot"), indirect=["image_type"]) +@pytest.mark.skip(reason="kvm boot tests are currently disabled") def test_image_boots(image_type): assert_disk_image_boots(image_type) diff --git a/bootc-image-builder/test/test_build_iso.py b/bootc-image-builder/test/test_build_iso.py index 54b21ec144..f75c4b32f1 100644 --- a/bootc-image-builder/test/test_build_iso.py +++ b/bootc-image-builder/test/test_build_iso.py @@ -28,6 +28,7 @@ @pytest.mark.skipif(platform.system() != "Linux", reason="boot test only runs on linux right now") @pytest.mark.parametrize("image_type", gen_testcases("anaconda-iso"), indirect=["image_type"]) +@pytest.mark.skip(reason="kvm boot tests are currently disabled") def test_iso_installs(image_type): installer_iso_path = image_type.img_path test_disk_path = installer_iso_path.with_name("test-disk.img") @@ -97,6 +98,7 @@ def test_iso_install_img_is_squashfs(tmp_path, image_type): "quay.io/centos-bootc/centos-bootc:stream9", ]) # pylint: disable=too-many-locals +@pytest.mark.skip(reason="kvm boot tests are currently disabled") def test_bootc_installer_iso_installs(tmp_path, build_container, container_ref): # XXX: duplicated from test_build_disk.py username = "test" From 4a5d330464da68d8c4c61da11ed13635e39dae0f Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Mon, 6 Jul 2026 14:06:59 +0200 Subject: [PATCH 18/25] bootc-image-builder: base upstream container on Fedora 44 --- bootc-image-builder/Containerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bootc-image-builder/Containerfile b/bootc-image-builder/Containerfile index dba3cdf4a9..7334a072c0 100644 --- a/bootc-image-builder/Containerfile +++ b/bootc-image-builder/Containerfile @@ -1,4 +1,4 @@ -FROM registry.fedoraproject.org/fedora:43 AS builder +FROM registry.fedoraproject.org/fedora:44 AS builder RUN dnf install -y git-core golang gpgme-devel libassuan-devel libvirt-devel && mkdir -p /build/bib COPY go.mod go.sum /build/bib/ RUN cd /build/bib && go mod download @@ -9,7 +9,7 @@ WORKDIR /build # keep in sync with main Containerfile, this was part of bib:build.sh RUN make build -FROM registry.fedoraproject.org/fedora:43 +FROM registry.fedoraproject.org/fedora:44 # Fast-track osbuild so we don't depend on the "slow" Fedora release process to implement new features in bib RUN dnf install -y dnf-plugins-core \ && dnf copr enable -y @osbuild/osbuild \ @@ -18,6 +18,7 @@ RUN dnf install -y dnf-plugins-core \ # copy as bootc-image-builder COPY --from=builder /build/bin/image-builder /usr/bin/bootc-image-builder +COPY ./data/distrodefs/ /usr/share/bootc-image-builder/defs/ ENTRYPOINT ["/usr/bin/bootc-image-builder"] VOLUME /output @@ -29,5 +30,5 @@ VOLUME /var/lib/containers/storage LABEL description="This tools allows to build and deploy disk-images from bootc container inputs." LABEL io.k8s.description="This tools allows to build and deploy disk-images from bootc container inputs." LABEL io.k8s.display-name="Bootc Image Builder" -LABEL io.openshift.tags="base fedora43" +LABEL io.openshift.tags="base fedora44" LABEL summary="A container to create disk-images from bootc container inputs" From c99fb72932e279296f6104cb55a668bbd1369fc9 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Mon, 6 Jul 2026 14:07:20 +0200 Subject: [PATCH 19/25] bootc-image-builder: test newer bootc containers Replace fedora-bootc:42 with fedora-bootc:44. Another test case still tests fedora-bootc:43. Swap out the CentOS Stream 9 container for CentOS Stream 10 in the multidisk test case. This commit also deletes the test_testcase_nameing() test. It wasn't being run because the filename, testcase.py, doesn't match the test naming convention and was not being discovered by pytest. It's not a very useful test anyway. --- bootc-image-builder/test/testcases.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/bootc-image-builder/test/testcases.py b/bootc-image-builder/test/testcases.py index 2e8a331552..cc916984dc 100644 --- a/bootc-image-builder/test/testcases.py +++ b/bootc-image-builder/test/testcases.py @@ -51,7 +51,7 @@ def __str__(self): @dataclasses.dataclass class TestCaseFedora(TestCase): - container_ref: str = "quay.io/fedora/fedora-bootc:42" + container_ref: str = "quay.io/fedora/fedora-bootc:44" rootfs: str = "btrfs" use_librepo: bool = True @@ -80,16 +80,6 @@ class TestCaseC10S(TestCase): use_librepo: bool = True -def test_testcase_nameing(): - """ - Ensure the testcase naming does not change without us knowing as those - are visible when running "pytest --collect-only" - """ - tc = TestCaseFedora() - expected = "container_ref=quay.io/fedora/fedora-bootc:40,rootfs=btrfs" - assert f"{tc}" == expected, f"{tc} != {expected}" - - def gen_testcases(what): # pylint: disable=too-many-return-statements if what == "manifest": return [TestCaseC9S(), TestCaseFedora(), TestCaseC10S()] @@ -135,7 +125,7 @@ def gen_testcases(what): # pylint: disable=too-many-return-statements # single test that specifies all image types image = "+".join(DISK_IMAGE_TYPES) return [ - TestCaseC9S(image=image), + TestCaseC10S(image=image), TestCaseFedora(image=image), ] # Smoke test that all supported --target-arch architecture can From 737fbfc0983dfbeff5a89eb2ebb9802d54bf93c5 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Mon, 6 Jul 2026 14:21:50 +0200 Subject: [PATCH 20/25] bootc-image-builder: disable anaconda-iso tests The anaconda-iso was never fully supported in image-builder. Disabling the tests so we can make sure everything else is working until we finish the transition to building everything in image-builder. --- bootc-image-builder/test/test_build_disk.py | 1 + bootc-image-builder/test/test_build_iso.py | 3 +++ bootc-image-builder/test/test_manifest.py | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/bootc-image-builder/test/test_build_disk.py b/bootc-image-builder/test/test_build_disk.py index 96aefdeb5b..6b626f4a83 100644 --- a/bootc-image-builder/test/test_build_disk.py +++ b/bootc-image-builder/test/test_build_disk.py @@ -651,6 +651,7 @@ def test_image_build_without_se_linux_denials(image_type): @pytest.mark.skipif(platform.system() != "Linux", reason="osinfo detect test only runs on linux right now") @pytest.mark.skipif(not testutil.has_executable("unsquashfs"), reason="need unsquashfs") @pytest.mark.parametrize("image_type", gen_testcases("anaconda-iso"), indirect=["image_type"]) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_iso_install_img_is_squashfs(tmp_path, image_type): installer_iso_path = image_type.img_path with ExitStack() as cm: diff --git a/bootc-image-builder/test/test_build_iso.py b/bootc-image-builder/test/test_build_iso.py index f75c4b32f1..d15b8d2938 100644 --- a/bootc-image-builder/test/test_build_iso.py +++ b/bootc-image-builder/test/test_build_iso.py @@ -29,6 +29,7 @@ @pytest.mark.skipif(platform.system() != "Linux", reason="boot test only runs on linux right now") @pytest.mark.parametrize("image_type", gen_testcases("anaconda-iso"), indirect=["image_type"]) @pytest.mark.skip(reason="kvm boot tests are currently disabled") +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_iso_installs(image_type): installer_iso_path = image_type.img_path test_disk_path = installer_iso_path.with_name("test-disk.img") @@ -61,6 +62,7 @@ def osinfo_for(it: ImageBuildResult, arch: str) -> str: @pytest.mark.skipif(platform.system() != "Linux", reason="osinfo detect test only runs on linux right now") @pytest.mark.parametrize("image_type", gen_testcases("anaconda-iso"), indirect=["image_type"]) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_iso_os_detection(image_type): installer_iso_path = image_type.img_path arch = image_type.img_arch @@ -78,6 +80,7 @@ def test_iso_os_detection(image_type): @pytest.mark.skipif(platform.system() != "Linux", reason="osinfo detect test only runs on linux right now") @pytest.mark.skipif(not testutil.has_executable("unsquashfs"), reason="need unsquashfs") @pytest.mark.parametrize("image_type", gen_testcases("anaconda-iso"), indirect=["image_type"]) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_iso_install_img_is_squashfs(tmp_path, image_type): installer_iso_path = image_type.img_path with ExitStack() as cm: diff --git a/bootc-image-builder/test/test_manifest.py b/bootc-image-builder/test/test_manifest.py index 85418e42be..249dd7250a 100644 --- a/bootc-image-builder/test/test_manifest.py +++ b/bootc-image-builder/test/test_manifest.py @@ -55,6 +55,7 @@ def test_manifest_smoke(build_container, tc): @pytest.mark.parametrize("tc", gen_testcases("anaconda-iso")) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_rpm_iso_manifest_smoke(build_container, tc): testutil.pull_container(tc.container_ref, tc.target_arch) @@ -264,6 +265,7 @@ def test_manifest_user_customizations_toml(tmp_path, build_container): } +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_manifest_installer_customizations(tmp_path, build_container): container_ref = "quay.io/centos-bootc/centos-bootc:stream9" testutil.pull_container(container_ref) @@ -395,6 +397,7 @@ def find_image_anaconda_stage(manifest_str): @pytest.mark.parametrize("tc", gen_testcases("anaconda-iso")) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_manifest_anaconda_module_customizations(tmpdir_factory, build_container, tc): testutil.pull_container(tc.container_ref, tc.target_arch) @@ -600,6 +603,7 @@ def find_grub2_iso_stage_from(manifest_str): raise ValueError(f"cannot find grub2.iso stage in manifest:\n{manifest_str}") +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_manifest_fips_customization(tmp_path, build_container): container_ref = "quay.io/centos-bootc/centos-bootc:stream9" testutil.pull_container(container_ref) @@ -819,6 +823,7 @@ def test_manifest_disk_customization_lvm_swap(tmp_path, build_container): @pytest.mark.parametrize("use_librepo", [False, True]) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_iso_manifest_use_librepo(build_container, use_librepo): # no need to parameterize this test, --use-librepo behaves same for all containers container_ref = "quay.io/centos-bootc/centos-bootc:stream9" @@ -1061,6 +1066,7 @@ def test_manifest_image_disk_yaml(tmp_path, build_container): @pytest.mark.parametrize("tc", gen_testcases("anaconda-iso")) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_ova_manifest_smoke(build_container, tc): testutil.pull_container(tc.container_ref, tc.target_arch) From c89144c4a2769dac809a849131e8c67c035fb584 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 7 Jul 2026 13:48:38 +0200 Subject: [PATCH 21/25] bootc-image-builder: skip local registry for disk images The local registry is used to test signed containers for ISO installations. It also takes too long to set up and can fail in certain cases (parsing the output of 'ip route' to get the address of the registry container). Since it's unnecessary in disk images, don't use it there and make it optional for the build_images() context. This is a bit dirty, as there's nothing tying the necessity of a registry to the test case that needs it. But all of these tests are due for an overhaul so it's fine for now. --- bootc-image-builder/test/test_build_disk.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bootc-image-builder/test/test_build_disk.py b/bootc-image-builder/test/test_build_disk.py index 6b626f4a83..5461d501d5 100644 --- a/bootc-image-builder/test/test_build_disk.py +++ b/bootc-image-builder/test/test_build_disk.py @@ -215,7 +215,7 @@ def sign_container_image(gpg_conf: GPGConf, registry_conf: RegistryConf, contain @pytest.fixture(name="image_type", scope="session") # pylint: disable=too-many-arguments,too-many-positional-arguments -def image_type_fixture(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf, registry_conf): +def image_type_fixture(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf): """ Build an image inside the passed build_container and return an ImageBuildResult with the resulting image path and user/password @@ -224,8 +224,7 @@ def image_type_fixture(shared_tmpdir, build_container, request, force_aws_upload """ testutil.pull_container(request.param.container_ref, request.param.target_arch) - with build_images(shared_tmpdir, build_container, - request, force_aws_upload, gpg_conf, registry_conf) as build_results: + with build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf) as build_results: return build_results[0] @@ -245,7 +244,7 @@ def images_fixture(shared_tmpdir, build_container, request, force_aws_upload, gp # XXX: refactor # pylint: disable=too-many-locals,too-many-branches,too-many-statements,too-many-arguments,too-many-positional-arguments @contextmanager -def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf, registry_conf): +def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf, registry_conf=None): """ Build all available image types if necessary and return the results for the image types that were requested via :request:. From 4d58a77fd1f6f55e064409189a4a7fb71df64966 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 26 Jun 2026 15:59:31 +0200 Subject: [PATCH 22/25] gitlab: run a simple bootc-image-builder test in CI The environment in GitHub actions needs a lot of fiddling to get the tests working. Let's run the core image generation test for now in GitLab CI. This matches two tests, one for Fedora 44 and one for CentOS Stream 10, which builds all disk image types and checks that the images are generated. For now, this is enough to ensure that bootc-image-builder works as expected for the main task of building disk images. We will expand the matrix later to cover more functionality. The pytest invocation was copied from the bootc-image-builder tests. The basetemp ensures that the tests use /var/tmp instead of /tmp, which would be too small for the artifacts we produce. The unsetting of XDG_RUNTIME_DIR is a workaround for some podman failures that aren't explained or fully understood. I also added the --setup-show option which shows the fixtures that are set up during a run, since a lot of them can take a while and might fail. --- .gitlab-ci.yml | 22 ++++++++++++++++++++++ test/scripts/generate-gitlab-ci | 19 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a5366d732d..1310e1cf35 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1067,6 +1067,28 @@ fail: needs: - "generate-build-config: [rhel-9.9, x86_64]" + +"bootc-image-builder [x86_64]": + stage: gen + extends: .terraform + variables: + RUNNER: aws/fedora-44-x86_64 + script: + - sudo ./test/scripts/install-dependencies + - sudo mkdir -pv /var/tmp/bootc-image-builder-tests + - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests ./bootc-image-builder/test/test_build_disk.py::test_image_is_generated + + +"bootc-image-builder [aarch64]": + stage: gen + extends: .terraform + variables: + RUNNER: aws/fedora-44-aarch64 + script: + - sudo ./test/scripts/install-dependencies + - sudo mkdir -pv /var/tmp/bootc-image-builder-tests + - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests ./bootc-image-builder/test/test_build_disk.py::test_image_is_generated + "generate-ostree-build-config: [centos-9, aarch64]": stage: ostree-gen extends: .terraform diff --git a/test/scripts/generate-gitlab-ci b/test/scripts/generate-gitlab-ci index f41845a857..12450778cb 100755 --- a/test/scripts/generate-gitlab-ci +++ b/test/scripts/generate-gitlab-ci @@ -138,6 +138,18 @@ MANIFEST_GEN_TEMPLATE = """ exit "$errors" """ +BOOTC_IMAGE_BUILDER_TEMPLATE = """ +"bootc-image-builder [{arch}]": + stage: gen + extends: .terraform + variables: + RUNNER: {runner}-{arch} + script: + - sudo ./test/scripts/install-dependencies + - sudo mkdir -pv /var/tmp/bootc-image-builder-tests + - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests ./bootc-image-builder/test/test_build_disk.py::test_image_is_generated +""" + def sort_configs(configs): return sorted(configs, key=lambda img: img["distro"]+img["arch"]+img["image-type"]) @@ -194,6 +206,13 @@ def main(): combos.add(combo) man_gen_stage.append(MANIFEST_GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], runner=RUNNER)) + # Run bootc-image-builder tests on Fedora 44 only for now (both arches) + for arch in ("x86_64", "aarch64"): + trigger_stage.append(BOOTC_IMAGE_BUILDER_TEMPLATE.format( + runner=RUNNER, + arch=arch, + )) + with open(config_path, "w", encoding="utf-8") as config_file: config_file.write(BASE_CONFIG.format(runner=RUNNER)) config_file.write(testlib.core.BASE_CONFIG) From cf2ea01034d8e16d65c6bbd3e0483c47bf98ad74 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 9 Jul 2026 12:22:56 +0200 Subject: [PATCH 23/25] bootc-image-builder: fix typos and link in README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Anna VΓ­tovΓ‘ --- bootc-image-builder/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootc-image-builder/README.md b/bootc-image-builder/README.md index e7854c1cbc..f61a9af13f 100644 --- a/bootc-image-builder/README.md +++ b/bootc-image-builder/README.md @@ -271,7 +271,7 @@ The following progress types are supported: * term: Terminal based output, spinner, progressbar and most details of osbuild are hidden * debug: Details how the progress is called, mostly useful for bugreports -Note that when no value is given the progress is auto-detected baed on the environment. When `stdin` is a terminal the "term" progress is used, otherwise "verbose". The output of `verbose` is exactaly the same as it was before progress reporting was implemented. +Note that when no value is given the progress is auto-detected based on the environment. When `stdin` is a terminal the "term" progress is used, otherwise "verbose". The output of `verbose` is exactly the same as it was before progress reporting was implemented. ## ☁️ Cloud uploaders @@ -530,7 +530,7 @@ The `rootfs` option (or source container config, see [Detailed description of op ### Anaconda ISO (installer) options (`installer`, mapping) -Users can include kickstart file content that will be added to an ISO build to configure the installation process. When using custom kickstart scripts the customization needs to be done via the custom kickstart script. For example using a `[customizations.user]` block alongside a `[customizations.installer.kickstart]` block is not supported. See this issue [https://github.com/osbuild/bootc-image-builder/issues/528] for additional detail. +Users can include kickstart file content that will be added to an ISO build to configure the installation process. When using custom kickstart scripts the customization needs to be done via the custom kickstart script. For example using a `[customizations.user]` block alongside a `[customizations.installer.kickstart]` block is not supported. See [this issue](https://github.com/osbuild/bootc-image-builder/issues/528) for additional detail. Since multi-line strings are difficult to write and read in json, it's easier to use the toml format when adding kickstart contents: From 4723d32adb3a7e3faa74022c6ecfc072f0ea7fb2 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 9 Jul 2026 15:48:39 +0200 Subject: [PATCH 24/25] imgtestlib: add new function for getting list of items in an S3 prefix We will need this for the transition period where we have both old and new image names in the cache. --- test/scripts/imgtestlib/cache.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/test/scripts/imgtestlib/cache.py b/test/scripts/imgtestlib/cache.py index 20c69386fe..c317875fea 100644 --- a/test/scripts/imgtestlib/cache.py +++ b/test/scripts/imgtestlib/cache.py @@ -7,7 +7,7 @@ from .build import (gen_build_name, get_manifest_id, read_build_info, write_build_info) from .gitlab import log_section -from .run import runcmd_nc +from .run import runcmd, runcmd_nc from .testenv import get_host_distro, get_osbuild_commit S3_BUCKET = "s3://" + os.environ.get("AWS_BUCKET", "images-ci-cache") @@ -149,3 +149,32 @@ def upload_results(distro, arch, image_type, config_path): print(f"⬆️ Uploading info.json to {s3url}") runcmd_nc(["aws", "s3", "cp", "--no-progress", "--acl=private", "--recursive", build_dir+"/", s3url]) print("βœ… DONE!!") + + +def s3ls(path): + """ + Calls 'aws s3api list-objects-v2' and returns a list of S3 URLs for each object in the path. + Expects an S3 URL. + """ + if not path.startswith("s3://"): + raise ValueError(f"unexpected path {path}: expected S3 URL (s3://)") + + path = path.removeprefix("s3://") + bucket, prefix = path.split("/", 1) + response_json, _ = runcmd(["aws", "s3api", "list-objects-v2", + "--output", "json", + "--bucket", bucket, + "--prefix", prefix]) + + response = json.loads(response_json) + if "Contents" not in response: + raise RuntimeError(f"unexpected or empty response: {response_json}") + + item_paths = [] + for item in response["Contents"]: + if "Key" in item: # explicitly ignore items with no Key + key = item["Key"] + item_s3url = f"s3://{bucket}/{key}" + item_paths.append(item_s3url) + + return item_paths From 0be37d3c33225af7970b72b8048d99d3510736eb Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 9 Jul 2026 15:49:20 +0200 Subject: [PATCH 25/25] test/generate-ostree-build-config: find container filename in prefix While we transition to using image-builder to build images, which produces images with different filenames, let's rely on listing the contents of the prefix in the S3 bucket and downloading the image based on the filename extension. --- test/scripts/generate-ostree-build-config | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/scripts/generate-ostree-build-config b/test/scripts/generate-ostree-build-config index f2256ea6d8..6e317e6fb3 100755 --- a/test/scripts/generate-ostree-build-config +++ b/test/scripts/generate-ostree-build-config @@ -181,6 +181,24 @@ def default_ref(distro, arch): return f"{name}/{version}/{arch}/{product}" # pylint: disable=possibly-used-before-assignment +def find_container_tarball(path): + """ + Since we changed to building images in CI with image-builder [1], exported images now have more meaningful names + that contain the distribution, architecture, and image type name. It's possible to determine this name from the + information, but for a while we're going to be in a transition where the cache might contain images from before the + PR that are still relevant, cached images. Use this function to get the first tarball in the given cache directory + to use. + + https://github.com/osbuild/image-builder/pull/2462 + """ + items = testlib.cache.s3ls(path) + for item in items: + if item.endswith(".tar"): + return item + + raise RuntimeError(f"failed to find container tarball in {path}") + + @contextmanager def setup_dependencies(manifests, config_list, distro, arch): # pylint: disable=too-many-statements @@ -237,7 +255,7 @@ def setup_dependencies(manifests, config_list, distro, arch): dep_build_name = testlib.build.gen_build_name(ic_distro, ic_arch, dep_image_type, dep_config_name) manifest_id = manifests[dep_build_name + ".json"]["id"] container_s3_prefix = testlib.cache.gen_build_info_s3_dir_path(distro, arch, manifest_id) - container_s3_path = os.path.join(container_s3_prefix, "container", "container.tar") + container_s3_path = find_container_tarball(container_s3_prefix) # start each container once on an incremental port port = container_ports.get(container_s3_path)