diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9df47be5ca..b06589f4d2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -241,7 +241,7 @@ jobs: - name: Testing imgtestlib and test scripts run: | - python3 -m pytest -v -k "not images_integration" + python3 -m pytest -v -k "not images_integration" ./test python-lint: name: "🐍 Lint (test scripts)" @@ -263,12 +263,11 @@ jobs: - name: Install build and test dependencies run: | ./test/scripts/install-dependencies - dnf -y install python3-pylint - name: Analysing the code with pylint run: | - python3 -m pylint --version - python3 -m pylint $(grep -l "/usr/bin/env python3" -r test/scripts) test/scripts/*.py + pylint --version + pylint --recursive=y . yaml-checks: name: "YAML checks" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a5366d732d..1310e1cf35 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1067,6 +1067,28 @@ fail: needs: - "generate-build-config: [rhel-9.9, x86_64]" + +"bootc-image-builder [x86_64]": + stage: gen + extends: .terraform + variables: + RUNNER: aws/fedora-44-x86_64 + script: + - sudo ./test/scripts/install-dependencies + - sudo mkdir -pv /var/tmp/bootc-image-builder-tests + - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests ./bootc-image-builder/test/test_build_disk.py::test_image_is_generated + + +"bootc-image-builder [aarch64]": + stage: gen + extends: .terraform + variables: + RUNNER: aws/fedora-44-aarch64 + script: + - sudo ./test/scripts/install-dependencies + - sudo mkdir -pv /var/tmp/bootc-image-builder-tests + - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests ./bootc-image-builder/test/test_build_disk.py::test_image_is_generated + "generate-ostree-build-config: [centos-9, aarch64]": stage: ostree-gen extends: .terraform diff --git a/Containerfile.bootc-image-builder b/bootc-image-builder/Containerfile similarity index 87% rename from Containerfile.bootc-image-builder rename to bootc-image-builder/Containerfile index dba3cdf4a9..7334a072c0 100644 --- a/Containerfile.bootc-image-builder +++ b/bootc-image-builder/Containerfile @@ -1,4 +1,4 @@ -FROM registry.fedoraproject.org/fedora:43 AS builder +FROM registry.fedoraproject.org/fedora:44 AS builder RUN dnf install -y git-core golang gpgme-devel libassuan-devel libvirt-devel && mkdir -p /build/bib COPY go.mod go.sum /build/bib/ RUN cd /build/bib && go mod download @@ -9,7 +9,7 @@ WORKDIR /build # keep in sync with main Containerfile, this was part of bib:build.sh RUN make build -FROM registry.fedoraproject.org/fedora:43 +FROM registry.fedoraproject.org/fedora:44 # Fast-track osbuild so we don't depend on the "slow" Fedora release process to implement new features in bib RUN dnf install -y dnf-plugins-core \ && dnf copr enable -y @osbuild/osbuild \ @@ -18,6 +18,7 @@ RUN dnf install -y dnf-plugins-core \ # copy as bootc-image-builder COPY --from=builder /build/bin/image-builder /usr/bin/bootc-image-builder +COPY ./data/distrodefs/ /usr/share/bootc-image-builder/defs/ ENTRYPOINT ["/usr/bin/bootc-image-builder"] VOLUME /output @@ -29,5 +30,5 @@ VOLUME /var/lib/containers/storage LABEL description="This tools allows to build and deploy disk-images from bootc container inputs." LABEL io.k8s.description="This tools allows to build and deploy disk-images from bootc container inputs." LABEL io.k8s.display-name="Bootc Image Builder" -LABEL io.openshift.tags="base fedora43" +LABEL io.openshift.tags="base fedora44" LABEL summary="A container to create disk-images from bootc container inputs" diff --git a/bootc-image-builder/README.md b/bootc-image-builder/README.md new file mode 100644 index 0000000000..f61a9af13f --- /dev/null +++ b/bootc-image-builder/README.md @@ -0,0 +1,666 @@ +# bootc-image-builder + +A container to create disk images from [bootc](https://github.com/containers/bootc) container inputs, +especially oriented towards [Fedora/CentOS bootc](https://docs.fedoraproject.org/en-US/bootc/) or +derivatives. + +## 🔨 Installation + +Have [podman](https://podman.io/) installed on your system. Either through your systems package manager if you're on +Linux or through [Podman Desktop](https://podman.io/) if you are on macOS or Windows. If you want to run the resulting +virtual machine(s) or installer media you can use [qemu](https://www.qemu.org/). + +A very nice GUI extension for Podman Desktop is also +[available](https://github.com/containers/podman-desktop-extension-bootc). +The command line examples below can be all handled by +Podman Desktop. + +On macOS, the podman machine must be running in rootful mode: + +```bash +$ podman machine stop # if already running +Waiting for VM to exit... +Machine "podman-machine-default" stopped successfully +$ podman machine set --rootful +$ podman machine start +``` + +## ✍ Prerequisites + +If you are on a system with SELinux enforced: The package `osbuild-selinux` or equivalent osbuild SELinux policies must be installed in the system running +`bootc-image-builder`. + +## 🚀 Examples + +The following example builds a `centos-bootc:stream9` bootable container into a QCOW2 image for the architecture you're running +the command on. However, be sure to see the [upstream documentation](https://docs.fedoraproject.org/en-US/bootc/) +for more general information! Note that outside of initial experimentation, it's recommended to build a *derived* container image +(or reuse a derived image built via someone else) and then use this project to make a disk image from your custom image. + +The generic base images do not include a default user. This example injects a [user configuration file](#-build-config) +by adding a volume-mount for the local file to the bootc-image-builder container. + +The following command will create a QCOW2 disk image. First, create `./config.toml` as described above to configure user access. + +```bash +# Ensure the image is fetched +sudo podman pull quay.io/centos-bootc/centos-bootc:stream9 +mkdir output +sudo podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v ./config.toml:/config.toml:ro \ + -v ./output:/output \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + quay.io/centos-bootc/bootc-image-builder:latest \ + --type qcow2 \ + --use-librepo=True \ + quay.io/centos-bootc/centos-bootc:stream9 +``` + +Note that some images (like fedora) do not have a default root +filesystem type. In this case adds the switch `--rootfs `, +e.g. `--rootfs btrfs`. + +### Rootless + +There is *experimental* support for rootless builds in `bootc-image-builder`. To perform a rootless build KVM is used. The above example can be tried like so: + +```bash +# Ensure the image is fetched +podman pull quay.io/fedora/fedora-bootc:latest +mkdir output +podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v ./config.toml:/config.toml:ro \ + -v ./output:/output \ + -v ~/.local/share/containers/storage:/var/lib/containers/storage \ + quay.io/centos-bootc/bootc-image-builder:latest \ + --in-vm \ + --type qcow2 \ + --use-librepo=True \ + --rootfs ext4 \ + quay.io/fedora/fedora-bootc:latest +``` + +Note the mounting of the users container storage, addition of the `--in-vm` argument and the removal of `sudo` in the commands. + +### Running the resulting QCOW2 file on Linux (x86_64) + +A virtual machine can be launched using `qemu-system-x86_64` or with `virt-install` as shown below; +however there is more information about virtualization and other +choices in the [Fedora/CentOS bootc documentation](https://docs.fedoraproject.org/en-US/bootc/). + +#### qemu-system-x86_64 + +```bash +qemu-system-x86_64 \ + -M accel=kvm \ + -cpu host \ + -smp 2 \ + -m 4096 \ + -bios /usr/share/OVMF/OVMF_CODE.fd \ + -serial stdio \ + -snapshot output/qcow2/disk.qcow2 +``` + +#### virt-install + +```bash +sudo virt-install \ + --name fedora-bootc \ + --cpu host \ + --vcpus 4 \ + --memory 4096 \ + --import --disk ./output/qcow2/disk.qcow2,format=qcow2 \ + --os-variant fedora-eln +``` + +### Running the resulting QCOW2 file on macOS (aarch64) + +This assumes qemu was installed through [homebrew](https://brew.sh/). + +```bash +qemu-system-aarch64 \ + -M accel=hvf \ + -cpu host \ + -smp 2 \ + -m 4096 \ + -bios /opt/homebrew/Cellar/qemu/8.1.3_2/share/qemu/edk2-aarch64-code.fd \ + -serial stdio \ + -machine virt \ + -snapshot output/qcow2/disk.qcow2 +``` + +## 📝 Arguments + +```bash +Usage: + sudo podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v ./output:/output \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + quay.io/centos-bootc/bootc-image-builder:latest \ + + +Flags: + --chown string chown the ouput directory to match the specified UID:GID + --output string artifact output directory (default ".") + --progress string type of progress bar to use (e.g. verbose,term) (default "auto") + --rootfs string Root filesystem type. If not given, the default configured in the source container image is used. + --target-arch string build for the given target architecture (experimental) + --type stringArray image types to build [ami, anaconda-iso, bootc-installer, gce, iso, qcow2, raw, vhd, vmdk] (default [qcow2]) + --version version for bootc-image-builder + +Global Flags: + --log-level string logging level (debug, info, error); default error + -v, --verbose Switch to verbose mode +``` + +### Detailed description of optional flags + +| Argument | Description | Default Value | +|-------------------|-----------------------------------------------------------------------------------------------------------|:-------------:| +| --chown | chown the output directory to match the specified UID:GID | ❌ | +| --output | output the artifact into the given output directory | `.` | +| --progress | Show progress in the given format, supported: verbose,term,debug. If empty it is auto-detected | `auto` | +| **--rootfs** | Root filesystem type. Overrides the default from the source container. Supported values: ext4, xfs, btrfs | ❌ | +| **--type** | [Image type](#-image-types) to build (can be passed multiple times) | `qcow2` | +| --target-arch | [Target arch](#-target-architecture) to build | ❌ | +| --log-level | Change log level (debug, info, error) | `error` | +| -v,--verbose | Switch output/progress to verbose mode (implies --log-level=info) | `false` | +| --use-librepo | Download rpms using librepo (faster and more robust) | `false` | + +The `--type` parameter can be given multiple times and multiple +outputs will be produced. Note that comma or space separating the +`image-types`will not work, but this example will: `--type qcow2 +--type ami`. + + +*💡 Tip: Flags in **bold** are the most important ones.* + +## 💾 Image types + +The following image types are currently available via the `--type` argument: + +| Image type | Target environment | +|-----------------------|-------------------------------------------------------------------------------------------| +| `ami` | [Amazon Machine Image](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) | +| `qcow2` **(default)** | [QEMU](https://www.qemu.org/) | +| `vmdk` | [VMDK](https://en.wikipedia.org/wiki/VMDK) usable in vSphere, among others | +| `bootc-installer` | An installer ISO image based on the specified bootc container image. | +| `anaconda-iso` | An unattended Anaconda installer that installs to the first disk found. Built from RPMs. | +| `raw` | Unformatted [raw disk](https://en.wikipedia.org/wiki/Rawdisk). | +| `vhd` | [vhd](https://en.wikipedia.org/wiki/VHD_(file_format)) usable in Virtual PC, among others | +| `gce` | [GCE](https://cloud.google.com/compute/docs/images#custom_images) | +| `pxe-tar-xz` | A stateless image useful in PXE network boot environments | + + +## 💾 Image Type Requirements + +### pxe-tar-xz + +The container image being built must have the `dracut-live` and `squashfs-tools` packages installed as well as a rebuilding the initramfs with the 'dmsquash-live' module. See [osbuild documentation](https://github.com/osbuild/image-builder/blob/main/data/files/pxetree/README) for more information and a sample Containerfile. + +### bootc-installer + +When building `bootc-installer` the positional container argument is expected to be a container that has Anaconda inside it; an example `Containerfile` for such a container is: + +``` +FROM your-favorite-bootc-container:latest +RUN dnf install -y \ + anaconda \ + anaconda-install-env-deps \ + anaconda-dracut \ + dracut-config-generic \ + dracut-network \ + net-tools \ + squashfs-tools \ + grub2-efi-x64-cdboot \ + python3-mako \ + lorax-templates-* \ + biosdevname \ + prefixdevname \ + && dnf clean all + +# On Fedora 42 this is necessary to get files in the right places +# RUN dnf reinstall -y shim-x64 + +# On Fedora 43 and up this is necessary to get files in the right +# places +RUN mkdir -p /boot/efi && cp -ra /usr/lib/efi/*/*/EFI /boot/efi + +# lorax wants to create a symlink in /mnt which points to /var/mnt +# on bootc but /var/mnt does not exist on some images. +# +# If https://gitlab.com/fedora/bootc/base-images/-/merge_requests/294 +# gets merged this will be no longer needed +RUN mkdir /var/mnt +``` + +You must also pass the `--bootc-installer-payload-ref` argument. This is a container reference to the payload to be installed by Anaconda. It will be embedded inside the installer and Anaconda will be configured to install it. + +## 💾 Target architecture + +Specify the target architecture of the system on which the disk image will be installed on. By default, +`bootc-image-builder` will build for the native host architecture. The target architecture +must match an available architecture of the `bootc-image-builder` image you are using to build the disk image. +Navigate to the [centos-image-builder repository tags page](https://quay.io/repository/centos-bootc/bootc-image-builder?tab=tags) +and hover over the Tux icons to see the supported target architectures. +The architecture of the bootc OCI image and the bootc-image-builder image must match. For example, when building +a non-native architecture bootc OCI image, say, building for x86_64 from an arm-based Mac, it is possible to run +`podman build` with the `--platform linux/amd64` flag. In this case, to then build a disk image from the same arm-based Mac, +you should provide `--target-arch amd64` when running the `bootc-image-builder` command. + +## Progress types + +The following progress types are supported: + +* verbose: No spinners or progress bar, just information and full osbuild output +* term: Terminal based output, spinner, progressbar and most details of osbuild are hidden +* debug: Details how the progress is called, mostly useful for bugreports + +Note that when no value is given the progress is auto-detected based on the environment. When `stdin` is a terminal the "term" progress is used, otherwise "verbose". The output of `verbose` is exactly the same as it was before progress reporting was implemented. + +## ☁️ Cloud uploaders + +### Amazon Machine Images (AMIs) + +#### Prerequisites + +In order to successfully import an AMI into your AWS account, you need to have the [vmimport service role](https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html) configured on your account with the following additional permissions: + +``` +{ + "Effect": "Allow", + "Action": [ + "s3:ListAllMyBuckets", + "s3:GetBucketAcl", + "s3:DeleteObject" + ], + "Resource": [ + "arn:aws:s3:::amzn-s3-demo-import-bucket", + "arn:aws:s3:::amzn-s3-demo-import-bucket/*", + "arn:aws:s3:::amzn-s3-demo-export-bucket", + "arn:aws:s3:::amzn-s3-demo-export-bucket/*" + ] +}, +``` + +Replace `amzn-s3-demo-import-bucket` in the ARN with the bucket name. + +#### Flags + +AMIs can be automatically uploaded to AWS by specifying the following flags: + +| Argument | Description | +|----------------|------------------------------------------------------------------| +| --aws-ami-name | Name for the AMI in AWS | +| --aws-bucket | Target S3 bucket name for intermediate storage when creating AMI | +| --aws-region | Target region for AWS uploads | + +*Notes:* + +- *These flags must all be specified together. If none are specified, the AMI is exported to the output directory.* +- *The bucket must already exist in the selected region, bootc-image-builder will not create it if it is missing.* +- *The output volume is not needed in this case. The image is uploaded to AWS and not exported.* + +#### AWS credentials file + +If you already have a credentials file (usually in `$HOME/.aws/credentials`) you need to forward the +directory to the container + +For example: + +```bash +$ sudo podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v $HOME/.aws:/root/.aws:ro \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + --env AWS_PROFILE=default \ + quay.io/centos-bootc/bootc-image-builder:latest \ + --type ami \ + --aws-ami-name centos-bootc-ami \ + --aws-bucket fedora-bootc-bucket \ + --aws-region us-east-1 \ + quay.io/centos-bootc/centos-bootc:stream9 +``` + +Notes: + +- *you can also inject **ALL** your AWS configuration parameters with `--env AWS_*`* + +see the [AWS CLI documentation](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) for more information about other environment variables + +#### AWS credentials via environment + +AWS credentials can be specified through two environment variables: +| Variable name | Description | +|-----------------------|---------------------------------------------------------------------------------------------------------------------| +| AWS_ACCESS_KEY_ID | AWS access key associated with an IAM account. | +| AWS_SECRET_ACCESS_KEY | Specifies the secret key associated with the access key. This is essentially the "password" for the access key. | + +Those **should not** be specified with `--env` as plain value, but you can silently hand them over with `--env AWS_*` or +save these variables in a file and pass them using the `--env-file` flag for `podman run`. + +For example: + +```bash +$ cat aws.secrets +AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +$ sudo podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + --env-file=aws.secrets \ + quay.io/centos-bootc/bootc-image-builder:latest \ + --type ami \ + --aws-ami-name centos-bootc-ami \ + --aws-bucket centos-bootc-bucket \ + --aws-region us-east-1 \ + quay.io/centos-bootc/centos-bootc:stream9 +``` + +## 💽 Volumes + +The following volumes can be mounted inside the container: + +| Volume | Purpose | Required | +|-----------|--------------------------------------------------------|:--------:| +| `/output` | Used for storing the resulting artifacts | ✅ | +| `/store` | Used for the [osbuild store](https://www.osbuild.org/) | No | +| `/rpmmd` | Used for the DNF cache | No | + +## 📝 Build config + +A build config is a TOML (or JSON) file with customizations for the resulting image. The config file is mapped into the container directory to `/config.toml`. The customizations are specified under a `customizations` object. + +The build config is a [Blueprint file](https://github.com/osbuild/blueprint), documented in the [osbuild.org User Guide](https://osbuild.org/docs/user-guide/blueprint-reference/). Note that not all Blueprint options are supported in bootc-image-builder. Refer to the **bootc** tab for information on whether a specific customization is supported. + +As an example, let's show how you can add a user to the image: + +Firstly create a file `./config.toml` and put the following content into it: + +```toml +[[customizations.user]] +name = "alice" +password = "bob" +key = "ssh-rsa AAA ... user@email.com" +groups = ["wheel"] +``` + +Then, run `bootc-image-builder` with the following arguments: + +```bash +sudo podman run \ + --rm \ + -it \ + --privileged \ + --pull=newer \ + --security-opt label=type:unconfined_t \ + -v ./config.toml:/config.toml:ro \ + -v ./output:/output \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + quay.io/centos-bootc/bootc-image-builder:latest \ + --type qcow2 \ + quay.io/centos-bootc/centos-bootc:stream9 +``` + +The configuration can also be passed in via stdin when `--config -` +is used. Only JSON configuration is supported in this mode. + +Additionally, images can embed a build config file, either as +`config.json` or `config.toml` in the `/usr/lib/bootc-image-builder` +directory. If this exist, and contains filesystem or disk +customizations, then these are used by default if no such +customization are specified in the regular build config. + +### Users (`user`, array) + +Possible fields: + +| Field | Use | Required | +|------------|--------------------------------------------|:--------:| +| `name` | Name of the user | ✅ | +| `password` | Unencrypted password | No | +| `key` | Public SSH key contents | No | +| `groups` | An array of secondary to put the user into | No | + +Example: + +```json +{ + "customizations": { + "user": [ + { + "name": "alice", + "password": "bob", + "key": "ssh-rsa AAA ... user@email.com", + "groups": [ + "wheel", + "admins" + ] + } + ] + } +} +``` + +### Kernel Arguments (`kernel`, mapping) + +```json +{ + "customizations": { + "kernel": { + "append": "mitigations=auto,nosmt" + } + } +} + +``` + +### Filesystems (`filesystem`, array) + +The filesystem section of the customizations can be used to set the minimum size of the base partitions (`/` and `/boot`) as well as to create extra partitions with mountpoints under `/var`. + +```toml +[[customizations.filesystem]] +mountpoint = "/" +minsize = "10 GiB" + +[[customizations.filesystem]] +mountpoint = "/var/data" +minsize = "20 GiB" +``` + +```json +{ + "customizations": { + "filesystem": [ + { + "mountpoint": "/", + "minsize": "10 GiB" + }, + { + "mountpoint": "/var/data", + "minsize": "20 GiB" + } + ] + } +} +``` + +#### Interaction with `rootfs` + +#### Filesystem types + +The `--rootfs` option also sets the filesystem types for all additional mountpoints, where appropriate. See the see [Detailed description of optional flags](#detailed-description-of-optional-flags). + + +#### Allowed mountpoints and sizes + +The following restrictions and rules apply, unless the rootfs is `btrfs`: +- `/` can be specified to set the desired (minimum) size of the root filesystem. The final size of the filesystem, mounted at `/sysroot` on a booted system, is the value specified in this configuration or 2x the size of the base container, whichever is largest. +- `/boot`can be specified to set the desired size of the boot partition. +- Subdirectories of `/var` are supported, but symlinks in `/var` are not. For example, `/var/home` and `/var/run` are symlinks and cannot be filesystems on their own. +- `/var` itself cannot be a mountpoint. + +The `rootfs` option (or source container config, see [Detailed description of optional flags](#detailed-description-of-optional-flags) section) defines the filesystem type for the root filesystem. Currently, creation of btrfs subvolumes at build time is not supported. Therefore, if the `rootfs` is `btrfs`, no custom mountpoints are supported under `/var`. Only `/` and `/boot` can be configured. + + +### Anaconda ISO (installer) options (`installer`, mapping) + +Users can include kickstart file content that will be added to an ISO build to configure the installation process. When using custom kickstart scripts the customization needs to be done via the custom kickstart script. For example using a `[customizations.user]` block alongside a `[customizations.installer.kickstart]` block is not supported. See [this issue](https://github.com/osbuild/bootc-image-builder/issues/528) for additional detail. + +Since multi-line strings are difficult to write and read in json, it's easier to use the toml format when adding kickstart contents: + +```toml +[customizations.installer.kickstart] +contents = """ +text --non-interactive +zerombr +clearpart --all --initlabel --disklabel=gpt +autopart --noswap --type=lvm +network --bootproto=dhcp --device=link --activate --onboot=on +""" +``` + +The equivalent in json would be: +```json +{ + "customizations": { + "installer": { + "kickstart": { + "contents": "text --non-interactive\nzerombr\nclearpart --all --initlabel --disklabel=gpt\nautopart --noswap --type=lvm\nnetwork --bootproto=dhcp --device=link --activate --onboot=on" + } + } + } +} +``` + +Note that bootc-image-builder will automatically add the command that installs the container image (`ostreecontainer ...`), so this line or any line that conflicts with it should not be included. See the relevant [Kickstart documentation](https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#ostreecontainer) for more information. +No other kickstart commands are added by bootc-image-builder in this case, so it is the responsibility of the user to provide all other commands (for example, for partitioning, network, language, etc). + +#### Anaconda ISO (installer) Modules + +The Anaconda installer can be configured by enabling or disabling its dbus modules. + +```toml +[customizations.installer.modules] +enable = [ + "org.fedoraproject.Anaconda.Modules.Localization" +] +disable = [ + "org.fedoraproject.Anaconda.Modules.Users" +] +``` + +```json +{ + "customizations": { + "installer": { + "modules": { + "enable": [ + "org.fedoraproject.Anaconda.Modules.Localization" + ], + "disable": [ + "org.fedoraproject.Anaconda.Modules.Users" + ] + } + } + } +} +``` + +The following module names are known and supported: +- `org.fedoraproject.Anaconda.Modules.Localization` +- `org.fedoraproject.Anaconda.Modules.Network` +- `org.fedoraproject.Anaconda.Modules.Payloads` +- `org.fedoraproject.Anaconda.Modules.Runtime` +- `org.fedoraproject.Anaconda.Modules.Security` +- `org.fedoraproject.Anaconda.Modules.Services` +- `org.fedoraproject.Anaconda.Modules.Storage` +- `org.fedoraproject.Anaconda.Modules.Subscription` +- `org.fedoraproject.Anaconda.Modules.Timezone` +- `org.fedoraproject.Anaconda.Modules.Users` + +*Note: The values are not validated. Any name listed under `enable` will be added to the Anaconda configuration. This way, new or unknown modules can be enabled. However, it also means that mistyped or incorrect values may cause Anaconda to fail to start.* + +By default, the following modules are enabled for all Anaconda ISOs: +- `org.fedoraproject.Anaconda.Modules.Network` +- `org.fedoraproject.Anaconda.Modules.Payloads` +- `org.fedoraproject.Anaconda.Modules.Security` +- `org.fedoraproject.Anaconda.Modules.Services` +- `org.fedoraproject.Anaconda.Modules.Storage` +- `org.fedoraproject.Anaconda.Modules.Users` + +### Anaconda ISO (media) options (`iso`, mapping) + +Users can customize the volume_id (which will be the ISO's label, used also in boot/grub.cfg). + + +```toml +[customizations.iso] +volume_id = "TheISOLabel" +application_id = "MyFancyAPP" +publisher = "ThePublisher" +``` + +##### Enable vs Disable priority + +The `disable` list is processed after the `enable` list and therefore takes priority. In other words, adding the same module in both `enable` and `disable` will result in the module being **disabled**. +Furthermore, adding a module that is enabled by default to `disable` will result in the module being **disabled**. + +## Building + +To build the container locally you can run + +```shell +sudo podman build --tag bootc-image-builder . +``` + +NOTE: running already the `podman build` as root avoids problems later as we need to run the building +of the image as root anyway + +### Accessing the system + +With a virtual machine launched with the above [virt-install](#virt-install) example, access the system with + +```shell +ssh -i /path/to/private/ssh-key alice@ip-address +``` + +Note that if you do not provide a password for the provided user, `sudo` will not work unless passwordless sudo +is configured. The base image `quay.io/centos-bootc/centos-bootc:stream9` does not configure passwordless sudo. +This can be configured in a derived bootc container by including the following in a Containerfile. + +```dockerfile +FROM quay.io/centos-bootc/centos-bootc:stream9 +ADD wheel-passwordless-sudo /etc/sudoers.d/wheel-passwordless-sudo +``` + +The contents of the file `wheel-passwordless-sudo` should be + +```text +%wheel ALL=(ALL) NOPASSWD: ALL +``` diff --git a/test/bib/_conftest.py b/bootc-image-builder/test/conftest.py similarity index 90% rename from test/bib/_conftest.py rename to bootc-image-builder/test/conftest.py index 53285426fb..d473ff5e2c 100644 --- a/test/bib/_conftest.py +++ b/bootc-image-builder/test/conftest.py @@ -2,7 +2,6 @@ # pylint: disable=wrong-import-order from testcases import TestCase -from vmtest.util import get_free_port def pytest_addoption(parser): @@ -30,8 +29,3 @@ def pytest_make_parametrize_id(config, val): # pylint: disable=W0613 if isinstance(val, TestCase): return f"{val}" return None - - -@pytest.fixture(name="free_port") -def free_port_fixture(): - return get_free_port() diff --git a/test/bib/_containerbuild.py b/bootc-image-builder/test/containerbuild.py similarity index 92% rename from test/bib/_containerbuild.py rename to bootc-image-builder/test/containerbuild.py index 02d5d1d209..856fff18a7 100644 --- a/test/bib/_containerbuild.py +++ b/bootc-image-builder/test/containerbuild.py @@ -37,16 +37,19 @@ def make_container(container_path, arch=None): def build_container_fixture(): """Build a container from the Containerfile and returns the name""" if tag_from_env := os.getenv("BIB_TEST_BUILD_CONTAINER_TAG"): - return tag_from_env + yield tag_from_env + return container_tag = "bootc-image-builder-test" subprocess.check_call([ "podman", "build", "--cache-ttl=1h", - "-f", "Containerfile.bib", + "-f", "bootc-image-builder/Containerfile", "-t", container_tag, + ".", ]) - return container_tag + yield container_tag + subprocess.check_call(["podman", "rmi", container_tag]) @pytest.fixture(name="build_fake_container", scope="session") @@ -98,7 +101,8 @@ def build_fake_container_fixture(tmpdir_factory, build_container): "-t", container_tag, tmp_path, ]) - return container_tag + yield container_tag + subprocess.check_call(["podman", "rmi", container_tag]) @pytest.fixture(name="build_erroring_container", scope="session") @@ -145,4 +149,5 @@ def build_erroring_container_fixture(tmpdir_factory, build_container): "-t", container_tag, tmp_path, ]) - return container_tag + yield container_tag + subprocess.check_call(["podman", "rmi", container_tag]) diff --git a/test/bib/_requirements.txt b/bootc-image-builder/test/requirements.txt similarity index 100% rename from test/bib/_requirements.txt rename to bootc-image-builder/test/requirements.txt diff --git a/test/bib/_test_build_cross.py b/bootc-image-builder/test/test_build_cross.py similarity index 78% rename from test/bib/_test_build_cross.py rename to bootc-image-builder/test/test_build_cross.py index 12b89eebd7..8b43e8c6dd 100644 --- a/test/bib/_test_build_cross.py +++ b/bootc-image-builder/test/test_build_cross.py @@ -1,17 +1,10 @@ import platform import pytest - -from testcases import gen_testcases - from test_build_disk import ( # pylint: disable=unused-import - assert_disk_image_boots, - build_container_fixture, - gpg_conf_fixture, - image_type_fixture, - registry_conf_fixture, - shared_tmpdir_fixture, -) + assert_disk_image_boots, build_container_fixture, gpg_conf_fixture, + image_type_fixture, registry_conf_fixture, shared_tmpdir_fixture) +from testcases import gen_testcases # This testcase is not part of "test_build_disk.py:test_image_boots" diff --git a/test/bib/_test_build_disk.py b/bootc-image-builder/test/test_build_disk.py similarity index 96% rename from test/bib/_test_build_disk.py rename to bootc-image-builder/test/test_build_disk.py index 9aaadcd49d..5461d501d5 100644 --- a/test/bib/_test_build_disk.py +++ b/bootc-image-builder/test/test_build_disk.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import json import os import pathlib @@ -9,17 +11,18 @@ import subprocess import tempfile import uuid -from contextlib import contextmanager, ExitStack -from typing import NamedTuple +from contextlib import ExitStack, contextmanager from dataclasses import dataclass +from typing import NamedTuple +import imgtestlib as testlib import pytest + # local test utils import testutil -from containerbuild import build_container_fixture # pylint: disable=unused-import +from containerbuild import \ + build_container_fixture # pylint: disable=unused-import from testcases import CLOUD_BOOT_IMAGE_TYPES, DISK_IMAGE_TYPES, gen_testcases -import vmtest.util -from vmtest.vm import AWS_REGION, AWS, QEMU if not testutil.has_executable("podman"): pytest.skip("no podman, skipping integration tests that required podman", allow_module_level=True) @@ -35,7 +38,7 @@ class ImageBuildResult(NamedTuple): img_type: str - img_path: str + img_path: pathlib.Path img_arch: str container_ref: str build_container_ref: str @@ -114,7 +117,7 @@ def registry_conf_fixture(shared_tmpdir, request): {local_registry}: lookaside: file:///{sigstore_dir} """ - registry_port = vmtest.util.get_free_port() + registry_port = testlib.vm.get_free_port() # We cannot use localhost as we need to access the registry from both # the host system and the bootc-image-builder container. default_ip = testutil.get_ip_from_default_route() @@ -211,8 +214,8 @@ def sign_container_image(gpg_conf: GPGConf, registry_conf: RegistryConf, contain @pytest.fixture(name="image_type", scope="session") -# pylint: disable=too-many-arguments -def image_type_fixture(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf, registry_conf): +# pylint: disable=too-many-arguments,too-many-positional-arguments +def image_type_fixture(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf): """ Build an image inside the passed build_container and return an ImageBuildResult with the resulting image path and user/password @@ -221,13 +224,12 @@ def image_type_fixture(shared_tmpdir, build_container, request, force_aws_upload """ testutil.pull_container(request.param.container_ref, request.param.target_arch) - with build_images(shared_tmpdir, build_container, - request, force_aws_upload, gpg_conf, registry_conf) as build_results: + with build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf) as build_results: return build_results[0] @pytest.fixture(name="images", scope="session") -# pylint: disable=too-many-arguments +# pylint: disable=too-many-arguments,too-many-positional-arguments def images_fixture(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf, registry_conf): """ Build one or more images inside the passed build_container and return an @@ -240,9 +242,9 @@ def images_fixture(shared_tmpdir, build_container, request, force_aws_upload, gp # XXX: refactor -# pylint: disable=too-many-locals,too-many-branches,too-many-statements,too-many-arguments +# pylint: disable=too-many-locals,too-many-branches,too-many-statements,too-many-arguments,too-many-positional-arguments @contextmanager -def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf, registry_conf): +def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_conf, registry_conf=None): """ Build all available image types if necessary and return the results for the image types that were requested via :request:. @@ -411,7 +413,7 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_ upload_args = [ f"--aws-ami-name=bootc-image-builder-test-{str(uuid.uuid4())}", - f"--aws-region={AWS_REGION}", + f"--aws-region={testlib.vm.AWS_REGION}", "--aws-bucket=bootc-image-builder-ci", ] elif force_aws_upload: @@ -493,7 +495,7 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_ metadata["ami_id"] = parse_ami_id_from_log(journal_output) def del_ami(): - testutil.deregister_ami(metadata["ami_id"], AWS_REGION) + testutil.deregister_ami(metadata["ami_id"], testlib.vm.AWS_REGION) request.addfinalizer(del_ami) journal_log_path.write_text(journal_output, encoding="utf8") @@ -553,12 +555,13 @@ def assert_kernel_args(test_vm, image_type): @pytest.mark.skipif(platform.system() != "Linux", reason="boot test only runs on linux right now") @pytest.mark.parametrize("image_type", gen_testcases("qemu-boot"), indirect=["image_type"]) +@pytest.mark.skip(reason="kvm boot tests are currently disabled") def test_image_boots(image_type): assert_disk_image_boots(image_type) def assert_disk_image_boots(image_type): - with QEMU(image_type.img_path, arch=image_type.img_arch) as test_vm: + with testlib.vm.QEMU(image_type.img_path, arch=image_type.img_arch) as test_vm: # user/password login works test_vm.run("true", user=image_type.username, password=image_type.password) # root/ssh login also works @@ -594,7 +597,7 @@ def test_ami_boots_in_aws(image_type, force_aws_upload): # check that upload progress is in the output log. Uploads looks like: # 4.30 GiB / 10.00 GiB [------------>____________] 43.02% 58.04 MiB p/s assert "] 100.00%" in image_type.bib_output - with AWS(image_type.metadata["ami_id"]) as test_vm: + with testlib.vm.AWS(image_type.metadata["ami_id"]) as test_vm: test_vm.run("true", user=image_type.username, password=image_type.password) ret = test_vm.run(["echo", "hello"], user=image_type.username, password=image_type.password) assert "hello" in ret.stdout @@ -647,6 +650,7 @@ def test_image_build_without_se_linux_denials(image_type): @pytest.mark.skipif(platform.system() != "Linux", reason="osinfo detect test only runs on linux right now") @pytest.mark.skipif(not testutil.has_executable("unsquashfs"), reason="need unsquashfs") @pytest.mark.parametrize("image_type", gen_testcases("anaconda-iso"), indirect=["image_type"]) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_iso_install_img_is_squashfs(tmp_path, image_type): installer_iso_path = image_type.img_path with ExitStack() as cm: diff --git a/test/bib/_test_build_iso.py b/bootc-image-builder/test/test_build_iso.py similarity index 90% rename from test/bib/_test_build_iso.py rename to bootc-image-builder/test/test_build_iso.py index c2f3998cd3..d15b8d2938 100644 --- a/test/bib/_test_build_iso.py +++ b/bootc-image-builder/test/test_build_iso.py @@ -1,46 +1,46 @@ -import os -import random +# allow TODO and XXX in the whole file +# pylint: disable=fixme import json +import os import pathlib import platform +import random import string import subprocess import textwrap from contextlib import ExitStack +import imgtestlib as testlib import pytest + # local test utils import testutil -from containerbuild import build_container_fixture, make_container # pylint: disable=unused-import -from testcases import gen_testcases -from test_build_disk import ( - assert_kernel_args, - ImageBuildResult, -) +from containerbuild import ( # pylint: disable=unused-import + build_container_fixture, make_container) +from test_build_disk import ImageBuildResult # pylint: disable=unused-import from test_build_disk import ( # pylint: disable=unused-import - gpg_conf_fixture, - image_type_fixture, - registry_conf_fixture, - shared_tmpdir_fixture, -) -from vmtest.vm import QEMU + assert_kernel_args, gpg_conf_fixture, image_type_fixture, + registry_conf_fixture, shared_tmpdir_fixture) +from testcases import gen_testcases ISO_BOOT_TIMEOUT = 1800 @pytest.mark.skipif(platform.system() != "Linux", reason="boot test only runs on linux right now") @pytest.mark.parametrize("image_type", gen_testcases("anaconda-iso"), indirect=["image_type"]) +@pytest.mark.skip(reason="kvm boot tests are currently disabled") +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_iso_installs(image_type): installer_iso_path = image_type.img_path test_disk_path = installer_iso_path.with_name("test-disk.img") with open(test_disk_path, "w", encoding="utf8") as fp: fp.truncate(10_1000_1000_1000) # install to test disk - with QEMU(test_disk_path, cdrom=installer_iso_path) as vm: + with testlib.vm.QEMU(test_disk_path, cdrom=installer_iso_path) as vm: vm.start(wait_event="qmp:RESET", snapshot=False, use_ovmf=True, timeout_sec=ISO_BOOT_TIMEOUT) vm.force_stop() # boot test disk and do extremly simple check - with QEMU(test_disk_path) as vm: + with testlib.vm.QEMU(test_disk_path) as vm: vm.start(use_ovmf=True) vm.run("true", user=image_type.username, password=image_type.password) assert_kernel_args(vm, image_type) @@ -62,6 +62,7 @@ def osinfo_for(it: ImageBuildResult, arch: str) -> str: @pytest.mark.skipif(platform.system() != "Linux", reason="osinfo detect test only runs on linux right now") @pytest.mark.parametrize("image_type", gen_testcases("anaconda-iso"), indirect=["image_type"]) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_iso_os_detection(image_type): installer_iso_path = image_type.img_path arch = image_type.img_arch @@ -79,6 +80,7 @@ def test_iso_os_detection(image_type): @pytest.mark.skipif(platform.system() != "Linux", reason="osinfo detect test only runs on linux right now") @pytest.mark.skipif(not testutil.has_executable("unsquashfs"), reason="need unsquashfs") @pytest.mark.parametrize("image_type", gen_testcases("anaconda-iso"), indirect=["image_type"]) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_iso_install_img_is_squashfs(tmp_path, image_type): installer_iso_path = image_type.img_path with ExitStack() as cm: @@ -99,6 +101,7 @@ def test_iso_install_img_is_squashfs(tmp_path, image_type): "quay.io/centos-bootc/centos-bootc:stream9", ]) # pylint: disable=too-many-locals +@pytest.mark.skip(reason="kvm boot tests are currently disabled") def test_bootc_installer_iso_installs(tmp_path, build_container, container_ref): # XXX: duplicated from test_build_disk.py username = "test" @@ -197,11 +200,11 @@ def test_bootc_installer_iso_installs(tmp_path, build_container, container_ref): with open(test_disk_path, "w", encoding="utf8") as fp: fp.truncate(10_1000_1000_1000) # install to test disk - with QEMU(test_disk_path, cdrom=installer_iso_path) as vm: + with testlib.vm.QEMU(test_disk_path, cdrom=installer_iso_path) as vm: vm.start(wait_event="qmp:RESET", snapshot=False, use_ovmf=True, timeout_sec=ISO_BOOT_TIMEOUT) vm.force_stop() # boot test disk and do extremly simple check - with QEMU(test_disk_path) as vm: + with testlib.vm.QEMU(test_disk_path) as vm: vm.start(use_ovmf=True) vm.run("true", user=username, password=password) ret = vm.run(["bootc", "status"], user="root", keyfile=ssh_keyfile_private_path) diff --git a/test/bib/_test_build_pxe.py b/bootc-image-builder/test/test_build_pxe.py similarity index 93% rename from test/bib/_test_build_pxe.py rename to bootc-image-builder/test/test_build_pxe.py index 7fba0ae341..cedce04bcf 100644 --- a/test/bib/_test_build_pxe.py +++ b/bootc-image-builder/test/test_build_pxe.py @@ -1,22 +1,25 @@ -import os -import random +# allow TODO and XXX in the whole file +# pylint: disable=fixme import json +import os import pathlib import platform +import random import string import subprocess import textwrap import threading - from contextlib import ExitStack -from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from tempfile import TemporaryDirectory +import imgtestlib as testlib import pytest + # local test utils import testutil -from containerbuild import build_container_fixture, make_container # pylint: disable=unused-import -from vmtest.vm import QEMU +from containerbuild import ( # pylint: disable=unused-import + build_container_fixture, make_container) def get_ostree_path(grub_path): @@ -41,7 +44,7 @@ def finish_request(self, request, client_address): SimpleHTTPRequestHandler(request, client_address, self, directory=self.directory) -# pylint: disable=too-many-arguments,too-many-locals,unused-argument +# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals,unused-argument def boot_qemu_pxe(arch, pxe_tar_path, container_ref, username, password, ssh_key_path, keep=False): with ExitStack() as cm: # unpack the tar and create a combined image @@ -84,7 +87,7 @@ def boot_qemu_pxe(arch, pxe_tar_path, container_ref, username, password, ssh_key "-append", append_arg ] - with QEMU(test_disk_path, memory="4096", arch=arch, extra_args=extra_args) as vm: + with testlib.vm.QEMU(test_disk_path, memory="4096", arch=arch, extra_args=extra_args) as vm: vm.start(use_ovmf=use_ovmf) vm.run("true", user=username, password=password) vm.run("mount", user="root", keyfile=ssh_key_path) diff --git a/test/bib/_test_manifest.py b/bootc-image-builder/test/test_manifest.py similarity index 98% rename from test/bib/_test_manifest.py rename to bootc-image-builder/test/test_manifest.py index fee8acaa06..249dd7250a 100644 --- a/test/bib/_test_manifest.py +++ b/bootc-image-builder/test/test_manifest.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme # pylint: disable=too-many-lines import base64 @@ -53,6 +55,7 @@ def test_manifest_smoke(build_container, tc): @pytest.mark.parametrize("tc", gen_testcases("anaconda-iso")) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_rpm_iso_manifest_smoke(build_container, tc): testutil.pull_container(tc.container_ref, tc.target_arch) @@ -262,6 +265,7 @@ def test_manifest_user_customizations_toml(tmp_path, build_container): } +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_manifest_installer_customizations(tmp_path, build_container): container_ref = "quay.io/centos-bootc/centos-bootc:stream9" testutil.pull_container(container_ref) @@ -393,6 +397,7 @@ def find_image_anaconda_stage(manifest_str): @pytest.mark.parametrize("tc", gen_testcases("anaconda-iso")) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_manifest_anaconda_module_customizations(tmpdir_factory, build_container, tc): testutil.pull_container(tc.container_ref, tc.target_arch) @@ -598,6 +603,7 @@ def find_grub2_iso_stage_from(manifest_str): raise ValueError(f"cannot find grub2.iso stage in manifest:\n{manifest_str}") +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_manifest_fips_customization(tmp_path, build_container): container_ref = "quay.io/centos-bootc/centos-bootc:stream9" testutil.pull_container(container_ref) @@ -817,6 +823,7 @@ def test_manifest_disk_customization_lvm_swap(tmp_path, build_container): @pytest.mark.parametrize("use_librepo", [False, True]) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_iso_manifest_use_librepo(build_container, use_librepo): # no need to parameterize this test, --use-librepo behaves same for all containers container_ref = "quay.io/centos-bootc/centos-bootc:stream9" @@ -1059,6 +1066,7 @@ def test_manifest_image_disk_yaml(tmp_path, build_container): @pytest.mark.parametrize("tc", gen_testcases("anaconda-iso")) +@pytest.mark.skip(reason="anaconda-iso is not supported") def test_ova_manifest_smoke(build_container, tc): testutil.pull_container(tc.container_ref, tc.target_arch) diff --git a/test/bib/_test_opts.py b/bootc-image-builder/test/test_opts.py similarity index 98% rename from test/bib/_test_opts.py rename to bootc-image-builder/test/test_opts.py index c12a8a7631..bd5e1311d2 100644 --- a/test/bib/_test_opts.py +++ b/bootc-image-builder/test/test_opts.py @@ -5,7 +5,8 @@ import pytest import testutil # pylint: disable=unused-import -from containerbuild import build_container_fixture, build_fake_container_fixture +from containerbuild import (build_container_fixture, + build_fake_container_fixture) @pytest.fixture(name="container_storage", scope="session") diff --git a/test/bib/_test_progress.py b/bootc-image-builder/test/test_progress.py similarity index 94% rename from test/bib/_test_progress.py rename to bootc-image-builder/test/test_progress.py index 559f5026e4..64cbd054de 100644 --- a/test/bib/_test_progress.py +++ b/bootc-image-builder/test/test_progress.py @@ -3,13 +3,11 @@ import pytest import testutil +from containerbuild import ( # pylint: disable=unused-import + build_container_fixture, build_erroring_container_fixture, + build_fake_container_fixture) # pylint: disable=unused-import,duplicate-code from test_opts import container_storage_fixture -from containerbuild import ( - build_container_fixture, - build_erroring_container_fixture, - build_fake_container_fixture, -) def test_progress_debug(tmp_path, build_fake_container): diff --git a/test/bib/_testcases.py b/bootc-image-builder/test/testcases.py similarity index 91% rename from test/bib/_testcases.py rename to bootc-image-builder/test/testcases.py index 5ef62fea8d..cc916984dc 100644 --- a/test/bib/_testcases.py +++ b/bootc-image-builder/test/testcases.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import dataclasses import inspect import os @@ -49,7 +51,7 @@ def __str__(self): @dataclasses.dataclass class TestCaseFedora(TestCase): - container_ref: str = "quay.io/fedora/fedora-bootc:42" + container_ref: str = "quay.io/fedora/fedora-bootc:44" rootfs: str = "btrfs" use_librepo: bool = True @@ -78,16 +80,6 @@ class TestCaseC10S(TestCase): use_librepo: bool = True -def test_testcase_nameing(): - """ - Ensure the testcase naming does not change without us knowing as those - are visible when running "pytest --collect-only" - """ - tc = TestCaseFedora() - expected = "container_ref=quay.io/fedora/fedora-bootc:40,rootfs=btrfs" - assert f"{tc}" == expected, f"{tc} != {expected}" - - def gen_testcases(what): # pylint: disable=too-many-return-statements if what == "manifest": return [TestCaseC9S(), TestCaseFedora(), TestCaseC10S()] @@ -133,7 +125,7 @@ def gen_testcases(what): # pylint: disable=too-many-return-statements # single test that specifies all image types image = "+".join(DISK_IMAGE_TYPES) return [ - TestCaseC9S(image=image), + TestCaseC10S(image=image), TestCaseFedora(image=image), ] # Smoke test that all supported --target-arch architecture can diff --git a/test/bib/_testutil.py b/bootc-image-builder/test/testutil.py similarity index 98% rename from test/bib/_testutil.py rename to bootc-image-builder/test/testutil.py index 096d8f6610..9817f36b10 100644 --- a/test/bib/_testutil.py +++ b/bootc-image-builder/test/testutil.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import os import pathlib import platform @@ -176,8 +178,6 @@ def maybe_create_disk_customizations(cfg, tc): "--privileged", "-v", "/var/lib/containers/storage:/var/lib/containers/storage", "--security-opt", "label=type:unconfined_t", - # ensure we run in reasonable memory limits - "--memory=8g", "--memory-swap=8g", ] diff --git a/pyproject.toml b/pyproject.toml index afe1d26434..95e83b3120 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,6 @@ dependencies = [ "scp==0.15.0", ] -[tool.setuptools.packages.find] -where = ["test/scripts"] -include = ["vmtest"] +# make imgtestlib importable when running pytest +[tool.pytest.ini_options] +pythonpath = ["test/scripts"] diff --git a/test/_test_build.py b/test/_test_build.py index ff0f81aa51..e66aa6554a 100644 --- a/test/_test_build.py +++ b/test/_test_build.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import json import os import pathlib @@ -93,6 +95,7 @@ def test_build_generates_rpmlist(tmp_path, build_container, shared_store): ("term", "[|]", "osbuild-stdout-output"), ]) @pytest.mark.skipif(os.getuid() != 0, reason="needs root") +# pylint: disable=too-many-arguments,too-many-positional-arguments def test_build_with_progress(tmp_path, build_fake_container, shared_store, progress, needle, forbidden): output_dir = tmp_path / "output" output_dir.mkdir() diff --git a/test/bib/_test_flake8.py b/test/bib/_test_flake8.py deleted file mode 100644 index bfd79219ee..0000000000 --- a/test/bib/_test_flake8.py +++ /dev/null @@ -1,11 +0,0 @@ -import os -import pathlib -import subprocess - - -def test_flake8(): - p = pathlib.Path(__file__).parent - # TODO: use all static checks from osbuild instead - subprocess.check_call( - ["flake8", "--ignore=E402,F811,F401", "--max-line-length=120", - os.fspath(p)]) diff --git a/test/bib/_test_pylint.py b/test/bib/_test_pylint.py deleted file mode 100644 index eb1c386bba..0000000000 --- a/test/bib/_test_pylint.py +++ /dev/null @@ -1,18 +0,0 @@ -import pathlib -import subprocess - - -def test_pylint(): - p = pathlib.Path(__file__).parent - subprocess.check_call( - ["pylint", - "--disable=fixme", - "--disable=missing-class-docstring", - "--disable=missing-module-docstring", - "--disable=missing-function-docstring", - "--disable=too-many-instance-attributes", - # false positive because of "if yield else yield" in - # the "build_container" fixture, see - # https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/contextmanager-generator-missing-cleanup.html - "--disable=contextmanager-generator-missing-cleanup", - "--max-line-length=120"] + list(p.glob("*.py"))) diff --git a/test/conftest.py b/test/conftest.py index 4b1c84d5ca..f332d21b83 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,68 +1,4 @@ -import random -import string -import subprocess -import textwrap - -import pytest - - def pytest_configure(config): config.addinivalue_line( "markers", "images_integration" ) - - -# XXX: copied from bib -@pytest.fixture(name="build_container", scope="session") -def build_container_fixture(): - """Build a container from the Containerfile and returns the name""" - - container_tag = "ibcli-test-" + "".join(random.choices( - string.ascii_lowercase + string.digits, k=4)) - - subprocess.check_call([ - "podman", "build", - "-f", "Containerfile", - "-t", container_tag, - ]) - yield container_tag - subprocess.check_call(["podman", "rmi", container_tag]) - - -# XXX: copied from bib -@pytest.fixture(name="build_fake_container", scope="session") -def build_fake_container_fixture(tmpdir_factory, build_container): - """Build a container with a fake osbuild and returns the name""" - tmp_path = tmpdir_factory.mktemp("build-fake-container") - - fake_osbuild_path = tmp_path / "fake-osbuild" - fake_osbuild_path.write_text(textwrap.dedent("""\ - #!/bin/bash -e - - # injest generated manifest from the images library, if we do not - # do this images may fail with "broken" pipe errors - cat - >/dev/null - - echo "osbuild-stdout-output" - mkdir -p /output/qcow2 - echo "fake-disk.qcow2" > /output/qcow2/disk.qcow2 - - """), encoding="utf8") - - cntf_path = tmp_path / "Containerfile" - - cntf_path.write_text(textwrap.dedent(f"""\n - FROM {build_container} - COPY fake-osbuild /usr/bin/osbuild - RUN chmod 755 /usr/bin/osbuild - """), encoding="utf8") - - container_tag = "ibcli-test-faked-osbuild-" + "".join(random.choices( - string.ascii_lowercase + string.digits, k=4)) - subprocess.check_call([ - "podman", "build", - "-t", container_tag, - tmp_path, - ]) - yield container_tag - subprocess.check_call(["podman", "rmi", container_tag]) diff --git a/test/scripts/boot-image b/test/scripts/boot-image index c39d32ae4c..d8b987d488 100755 --- a/test/scripts/boot-image +++ b/test/scripts/boot-image @@ -18,7 +18,7 @@ def main(): build_config_path = args.config keep = args.keep_booted - testlib.boot_image(search_path, build_config_path, keep) + testlib.boot.boot_image(search_path, build_config_path, keep) if __name__ == "__main__": diff --git a/test/scripts/build-image b/test/scripts/build-image index cc997ff491..368cef5cb0 100755 --- a/test/scripts/build-image +++ b/test/scripts/build-image @@ -21,7 +21,7 @@ def main(): arch = args.arch config_path = args.config - testlib.build_image(distro, arch, image_type, config_path) + testlib.build.build_image(distro, arch, image_type, config_path) if __name__ == "__main__": diff --git a/test/scripts/check-build-coverage b/test/scripts/check-build-coverage index c2cec170c8..c39d1fcf14 100755 --- a/test/scripts/check-build-coverage +++ b/test/scripts/check-build-coverage @@ -43,7 +43,7 @@ def check_build_coverage(cachedir): built_pr.add((distro, arch, image, config, commit)) all_combos = set() - for config in testlib.list_images(arches=["x86_64", "aarch64"]): + for config in testlib.core.list_images(arches=["x86_64", "aarch64"]): distro = config["distro"] arch = config["arch"] image = config["image-type"] @@ -60,7 +60,7 @@ def main(): parser.add_argument("cachedir", type=str, help="path to download the build test cache") args = parser.parse_args() cachedir = args.cachedir - _, ok = testlib.dl_build_info(cachedir) + _, ok = testlib.cache.dl_build_info(cachedir) # fail the job if the sync failed if not ok: sys.exit(1) diff --git a/test/scripts/dl-image-build-cache b/test/scripts/dl-image-build-cache index 0219a21ed2..52c7c6627b 100755 --- a/test/scripts/dl-image-build-cache +++ b/test/scripts/dl-image-build-cache @@ -117,8 +117,8 @@ def main(): if args.image_type and args.skip_image_type: parser.error("--image-type and --skip-image-type are mutually exclusive") - runner_distro = testlib.get_common_ci_runner_distro() - osbuild_ref = testlib.get_osbuild_commit(runner_distro) + runner_distro = testlib.testenv.get_common_ci_runner_distro() + osbuild_ref = testlib.testenv.get_osbuild_commit(runner_distro) if osbuild_ref is None: raise RuntimeError(f"Failed to determine osbuild commit for {runner_distro} from the Schutzfile") @@ -128,7 +128,7 @@ def main(): with tempfile.TemporaryDirectory() as tmpdir: print("📜 Generating current manifests to determine their IDs") - err = testlib.gen_manifests(tmpdir, arches=args.arch, distros=args.distro, images=args.image_type) + err = testlib.core.gen_manifests(tmpdir, arches=args.arch, distros=args.distro, images=args.image_type) # print stderr in case there were errors or warnings about skipped configurations # but filter out the annoying ones stderr = err.decode().splitlines() @@ -138,7 +138,7 @@ def main(): if "Failed to load consumer certs" in line: continue print(line) - manifest_gen_data = testlib.read_manifests(tmpdir) + manifest_gen_data = testlib.core.read_manifests(tmpdir) build_cache_infos = gen_manifest_data_to_build_cache_info(manifest_gen_data, args.config, args.skip_image_type) @@ -162,9 +162,9 @@ def main(): config = build_cache_info["config"] manifest_id = build_cache_info["manifest-id"] - target_dir = os.path.join(output_dir, testlib.gen_build_name(distro, arch, image_type, config)) + target_dir = os.path.join(output_dir, testlib.build.gen_build_name(distro, arch, image_type, config)) - out, dl_ok = testlib.dl_build_cache( + out, dl_ok = testlib.cache.dl_build_cache( target_dir, distro, arch, osbuild_ref, runner_distro, manifest_id, s3_include_only) if not dl_ok: failed_downloads.append(build_cache_info) diff --git a/test/scripts/dl-one-image-build-cache b/test/scripts/dl-one-image-build-cache index c1f4704655..4094b7d0f5 100755 --- a/test/scripts/dl-one-image-build-cache +++ b/test/scripts/dl-one-image-build-cache @@ -39,7 +39,7 @@ def main(): build_info_dir = os.path.dirname(args.build_info) if args.build_info else build_dir print(f"📜 Reading 'info.json' from {build_info_dir}") - build_info = testlib.read_build_info(build_info_dir) + build_info = testlib.build.read_build_info(build_info_dir) distro = build_info["distro"] arch = build_info["arch"] @@ -48,7 +48,7 @@ def main(): runner_distro = build_info.get("runner-distro") if runner_distro is None: - runner_distro = testlib.get_common_ci_runner_distro() + runner_distro = testlib.testenv.get_common_ci_runner_distro() print("⚠️ Runner distro not found in the build info. " + f"Using the CI runner distro from the current branch: {runner_distro}", file=sys.stderr) @@ -59,14 +59,14 @@ def main(): print(f" osbuild-ref: {osbuild_ref}") print(f" runner-distro: {runner_distro}") - out, dl_ok = testlib.dl_build_cache(build_dir, distro, arch, osbuild_ref, runner_distro, manifest_id) + out, dl_ok = testlib.cache.dl_build_cache(build_dir, distro, arch, osbuild_ref, runner_distro, manifest_id) print(out) if not dl_ok: print("❌ Failed to download the image build cache", file=sys.stderr) sys.exit(1) print("✅ Successfully downloaded the image build cache") - testlib.touch_s3(distro, arch, manifest_id, osbuild_ref, runner_distro) + testlib.cache.touch_s3(distro, arch, manifest_id, osbuild_ref, runner_distro) if __name__ == "__main__": diff --git a/test/scripts/generate-build-config b/test/scripts/generate-build-config index b0e9b9c87d..b52717723e 100755 --- a/test/scripts/generate-build-config +++ b/test/scripts/generate-build-config @@ -25,7 +25,7 @@ build/{distro}/{arch}/{image_type}/{config_name}: """ -@testlib.log_section("Generating manifests") +@testlib.gitlab.log_section("Generating manifests") def generate_manifests(outputdir, distro, arch): """ Generate all manifest using the default config list and return a dictionary mapping each manifest file to the @@ -38,7 +38,7 @@ def generate_manifests(outputdir, distro, arch): distros = [distro] print(f"🗒️ Generating all manifests using the default config list for {target}") - err = testlib.gen_manifests(outputdir, arches=[arch], distros=distros) + err = testlib.core.gen_manifests(outputdir, arches=[arch], distros=distros) # print stderr in case there were errors or warnings about skipped configurations # but filter out the annoying ones @@ -51,7 +51,7 @@ def generate_manifests(outputdir, distro, arch): print(line) print("✅ Manifest generation done!\n") - return testlib.read_manifests(outputdir) + return testlib.core.read_manifests(outputdir) def generate_configs(build_requests, pipeline_file): @@ -64,13 +64,13 @@ def generate_configs(build_requests, pipeline_file): config_name = config["name"] - build_name = testlib.gen_build_name(distro, arch, image_type, config_name) + build_name = testlib.build.gen_build_name(distro, arch, image_type, config_name) image_path = f"./build/{build_name}" - runner = testlib.get_ci_runner_for(distro, arch, image_type) - tag = testlib.get_tag_for(runner) + runner = testlib.testenv.get_ci_runner_for(distro, arch, image_type) + tag = testlib.core.get_tag_for(runner) - config_path = os.path.join(testlib.CONFIGS_PATH, config_name+".json") + config_path = os.path.join(testlib.core.CONFIGS_PATH, config_name+".json") pipeline_file.write(JOB_TEMPLATE.format(distro=distro, arch=arch, image_type=image_type, runner=runner, tag=tag, config_name=config_name, config=config_path, @@ -80,26 +80,26 @@ def generate_configs(build_requests, pipeline_file): def main(): - parser = testlib.clargs() + parser = testlib.core.clargs() args = parser.parse_args() config_path = args.config distro = args.distro arch = args.arch - testlib.check_config_names() + testlib.core.check_config_names() with TemporaryDirectory() as manifest_dir: manifests = generate_manifests(manifest_dir, distro, arch) - build_requests = testlib.filter_builds(manifests, distro=distro, arch=arch) + build_requests = testlib.core.filter_builds(manifests, distro=distro, arch=arch) with open(config_path, "w", encoding="utf-8") as config_file: if len(build_requests) == 0: print("⚫ No manifest changes detected. Generating null config.") - config_file.write(testlib.NULL_CONFIG) + config_file.write(testlib.core.NULL_CONFIG) return - config_file.write(testlib.BASE_CONFIG) + config_file.write(testlib.core.BASE_CONFIG) generate_configs(build_requests, config_file) diff --git a/test/scripts/generate-gitlab-ci b/test/scripts/generate-gitlab-ci index 070852785c..12450778cb 100755 --- a/test/scripts/generate-gitlab-ci +++ b/test/scripts/generate-gitlab-ci @@ -7,7 +7,7 @@ import imgtestlib as testlib ARCHITECTURES = ["x86_64", "aarch64"] MANIFEST_ONLY_ARCHES = ["ppc64le", "s390x"] -RUNNER = testlib.get_common_ci_runner() +RUNNER = testlib.testenv.get_common_ci_runner() OSTREE_DISTROS = ["fedora", "rhel-8", "rhel-9", "centos-9"] @@ -138,6 +138,18 @@ MANIFEST_GEN_TEMPLATE = """ exit "$errors" """ +BOOTC_IMAGE_BUILDER_TEMPLATE = """ +"bootc-image-builder [{arch}]": + stage: gen + extends: .terraform + variables: + RUNNER: {runner}-{arch} + script: + - sudo ./test/scripts/install-dependencies + - sudo mkdir -pv /var/tmp/bootc-image-builder-tests + - sudo -E XDG_RUNTIME_DIR= PYTHONPATH=. pytest -vv --setup-show --basetemp=/var/tmp/bootc-image-builder-tests ./bootc-image-builder/test/test_build_disk.py::test_image_is_generated +""" + def sort_configs(configs): return sorted(configs, key=lambda img: img["distro"]+img["arch"]+img["image-type"]) @@ -145,7 +157,7 @@ def sort_configs(configs): def main(): config_path = sys.argv[1] - images = testlib.list_images(arches=ARCHITECTURES) + images = testlib.core.list_images(arches=ARCHITECTURES) combos = set() gen_stage = [] @@ -184,7 +196,7 @@ def main(): arch=img["arch"], runner=RUNNER)) - man_only_images = testlib.list_images(arches=MANIFEST_ONLY_ARCHES) + man_only_images = testlib.core.list_images(arches=MANIFEST_ONLY_ARCHES) man_gen_stage = [] for img in sort_configs(man_only_images): combo = (img["distro"], img["arch"]) @@ -194,9 +206,16 @@ def main(): combos.add(combo) man_gen_stage.append(MANIFEST_GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], runner=RUNNER)) + # Run bootc-image-builder tests on Fedora 44 only for now (both arches) + for arch in ("x86_64", "aarch64"): + trigger_stage.append(BOOTC_IMAGE_BUILDER_TEMPLATE.format( + runner=RUNNER, + arch=arch, + )) + with open(config_path, "w", encoding="utf-8") as config_file: config_file.write(BASE_CONFIG.format(runner=RUNNER)) - config_file.write(testlib.BASE_CONFIG) + config_file.write(testlib.core.BASE_CONFIG) config_file.write("\n".join(gen_stage)) config_file.write("\n".join(trigger_stage)) config_file.write("\n".join(ostree_gen_stage)) diff --git a/test/scripts/generate-ostree-build-config b/test/scripts/generate-ostree-build-config index e8ec3c512c..6e317e6fb3 100755 --- a/test/scripts/generate-ostree-build-config +++ b/test/scripts/generate-ostree-build-config @@ -29,7 +29,7 @@ build/{distro}/{arch}/{image_type}/{config_name}: def read_config_list(): - with open(testlib.CONFIG_LIST, "r", encoding="utf-8") as config_list_file: + with open(testlib.core.CONFIG_LIST, "r", encoding="utf-8") as config_list_file: return json.load(config_list_file) @@ -43,7 +43,7 @@ def configs_with_deps(configs, distro=None, arch=None): Filter on distro and arch if specified. """ with_deps = [] - config_list_dir = os.path.abspath(os.path.dirname(testlib.CONFIG_LIST)) + config_list_dir = os.path.abspath(os.path.dirname(testlib.core.CONFIG_LIST)) for item in configs: path = item["path"] filters = item["filters"] @@ -105,8 +105,8 @@ def gen_dependency_manifests(config_list, distro, arch, outputdir): with NamedTemporaryFile(mode="w") as tmpfile: json.dump(dep_config_list, tmpfile) tmpfile.flush() - err = testlib.gen_manifests(outputdir, config_list=tmpfile.name, - distros=distros, arches=[arch], images=gen_image_types) + err = testlib.core.gen_manifests(outputdir, config_list=tmpfile.name, + distros=distros, arches=[arch], images=gen_image_types) # print stderr in case there were errors or warnings about skipped configurations # but filter out the annoying ones stderr = err.decode().splitlines() @@ -118,7 +118,7 @@ def gen_dependency_manifests(config_list, distro, arch, outputdir): print(line) print("✅ Manifest generation done!\n") - return testlib.read_manifests(outputdir) + return testlib.core.read_manifests(outputdir) def gen_image_manifests(config_list, configs, distro, arch, outputdir): @@ -147,9 +147,9 @@ def gen_image_manifests(config_list, configs, distro, arch, outputdir): with open(config_list_path, "w", encoding="utf-8") as config_list_file: json.dump(config_list, config_list_file) - err = testlib.gen_manifests(outputdir, config_list=config_list_path, - arches=[arch], distros=distros, commits=True, - flatpaks=True, skip_no_config=True) + err = testlib.core.gen_manifests(outputdir, config_list=config_list_path, + arches=[arch], distros=distros, commits=True, + flatpaks=True, skip_no_config=True) # print stderr in case there were errors or warnings about skipped configurations # but filter out the annoying ones @@ -162,7 +162,7 @@ def gen_image_manifests(config_list, configs, distro, arch, outputdir): print(line) print("✅ Manifest generation done!\n") - return testlib.read_manifests(outputdir) + return testlib.core.read_manifests(outputdir) def default_ref(distro, arch): @@ -181,6 +181,24 @@ def default_ref(distro, arch): return f"{name}/{version}/{arch}/{product}" # pylint: disable=possibly-used-before-assignment +def find_container_tarball(path): + """ + Since we changed to building images in CI with image-builder [1], exported images now have more meaningful names + that contain the distribution, architecture, and image type name. It's possible to determine this name from the + information, but for a while we're going to be in a transition where the cache might contain images from before the + PR that are still relevant, cached images. Use this function to get the first tarball in the given cache directory + to use. + + https://github.com/osbuild/image-builder/pull/2462 + """ + items = testlib.cache.s3ls(path) + for item in items: + if item.endswith(".tar"): + return item + + raise RuntimeError(f"failed to find container tarball in {path}") + + @contextmanager def setup_dependencies(manifests, config_list, distro, arch): # pylint: disable=too-many-statements @@ -217,9 +235,9 @@ def setup_dependencies(manifests, config_list, distro, arch): if distro: distros = [distro] arches = [arch] - image_configs = testlib.list_images(distros=distros, - arches=arches, - images=filters.get("image-types")) + image_configs = testlib.core.list_images(distros=distros, + arches=arches, + images=filters.get("image-types")) distros_to_skip = [] for skip in filters.get("skip-distros", []): @@ -234,10 +252,10 @@ def setup_dependencies(manifests, config_list, distro, arch): continue ic_arch = image_config["arch"] ic_image_type = image_config["image-type"] - dep_build_name = testlib.gen_build_name(ic_distro, ic_arch, dep_image_type, dep_config_name) + dep_build_name = testlib.build.gen_build_name(ic_distro, ic_arch, dep_image_type, dep_config_name) manifest_id = manifests[dep_build_name + ".json"]["id"] - container_s3_prefix = testlib.gen_build_info_s3_dir_path(distro, arch, manifest_id) - container_s3_path = os.path.join(container_s3_prefix, "container", "container.tar") + container_s3_prefix = testlib.cache.gen_build_info_s3_dir_path(distro, arch, manifest_id) + container_s3_path = find_container_tarball(container_s3_prefix) # start each container once on an incremental port port = container_ports.get(container_s3_path) @@ -284,19 +302,19 @@ def setup_dependencies(manifests, config_list, distro, arch): with TemporaryDirectory() as container_dir: print(f"⬇️ Downloading container archive from {container_s3_path}") container_archive = os.path.join(container_dir, "container.tar") - testlib.runcmd_nc(["aws", "s3", "cp", "--no-progress", container_s3_path, container_archive]) + testlib.run.runcmd_nc(["aws", "s3", "cp", "--no-progress", container_s3_path, container_archive]) print(f"📦 Starting container oci-archive:{container_archive} {port}") # Running podman as root is not necessary, but currently it's failing with permission errors. # Run it with sudo for now until we figure out the issue. - cont_id, _ = testlib.runcmd(["sudo", "podman", "run", "-d", "--rm", f"-p{port}:8080", - f"oci-archive:{container_archive}"]) + cont_id, _ = testlib.run.runcmd(["sudo", "podman", "run", "-d", "--rm", f"-p{port}:8080", + f"oci-archive:{container_archive}"]) container_ids.append(cont_id.strip().decode()) yield new_config_list, new_configs, container_configs finally: if container_ids: print("📦 Stopping containers") - out, _ = testlib.runcmd(["sudo", "podman", "stop", *container_ids]) + out, _ = testlib.run.runcmd(["sudo", "podman", "stop", *container_ids]) print(out.decode()) @@ -311,7 +329,7 @@ def generate_configs(build_requests, container_configs, pipeline_file, configs_d config_name = config["name"] - build_name = testlib.gen_build_name(distro, arch, image_type, config_name) + build_name = testlib.build.gen_build_name(distro, arch, image_type, config_name) image_path = f"./build/{build_name}" # generate script line to pull and start container container = container_configs[config_name] @@ -328,7 +346,7 @@ def generate_configs(build_requests, container_configs, pipeline_file, configs_d run_container_cmd = f"sudo podman run -d --rm -p{container_port}:8080 oci-archive:container.tar" pipeline_file.write(JOB_TEMPLATE.format(distro=distro, arch=arch, image_type=image_type, - runner=testlib.get_common_ci_runner(), + runner=testlib.testenv.get_common_ci_runner(), config_name=config_name, config=build_config_path, dl_container=dl_container_cmd, start_container=run_container_cmd, @@ -338,7 +356,7 @@ def generate_configs(build_requests, container_configs, pipeline_file, configs_d def main(): - parser = testlib.clargs() + parser = testlib.core.clargs() parser.add_argument("build_configs", type=str, help="directory to write individual build configs") args = parser.parse_args() @@ -347,7 +365,7 @@ def main(): distro = args.distro arch = args.arch - testlib.check_config_names() + testlib.core.check_config_names() config_list = configs_with_deps(read_config_list(), distro, arch) # filtered config list: only configs with deps @@ -360,15 +378,15 @@ def main(): manifest_dir = os.path.join(cache_root, "manifests") manifests = gen_image_manifests(pull_config_list, pull_configs, distro, arch, manifest_dir) - build_requests = testlib.filter_builds(manifests, distro=distro, arch=arch, skip_ostree_pull=False) + build_requests = testlib.core.filter_builds(manifests, distro=distro, arch=arch, skip_ostree_pull=False) with open(config_path, "w", encoding="utf-8") as config_file: if len(build_requests) == 0: print("⚫ No manifest changes detected. Generating null config.") - config_file.write(testlib.NULL_CONFIG) + config_file.write(testlib.core.NULL_CONFIG) return - config_file.write(testlib.BASE_CONFIG) + config_file.write(testlib.core.BASE_CONFIG) generate_configs(build_requests, containers, config_file, configs_dir) diff --git a/test/scripts/imgtestlib/__init__.py b/test/scripts/imgtestlib/__init__.py index fe1d86bfbb..5ad4b175b5 100644 --- a/test/scripts/imgtestlib/__init__.py +++ b/test/scripts/imgtestlib/__init__.py @@ -1,7 +1 @@ -from .boot import * -from .build import * -from .cache import * -from .core import * -from .gitlab import * -from .run import * -from .testenv import * +from . import boot, build, cache, core, gitlab, run, testenv, vm diff --git a/test/scripts/imgtestlib/boot.py b/test/scripts/imgtestlib/boot.py index 1ddaed46a1..9663f0aa39 100644 --- a/test/scripts/imgtestlib/boot.py +++ b/test/scripts/imgtestlib/boot.py @@ -12,15 +12,13 @@ from tempfile import TemporaryDirectory from typing import Generator -from vmtest.util import get_free_port -from vmtest.vm import QEMU - from .build import read_build_info, write_build_info from .core import (can_boot_test, find_image_file, read_manifest, skopeo_inspect_id) from .gitlab import log_section from .run import runcmd, runcmd_nc from .testenv import get_bib_ref, host_container_arch +from .vm import QEMU, get_free_port BASE_TEST_EXEC = "check-host-config-" # + arch WSL_TEST_SCRIPT = "test/scripts/wsl-entrypoint.bat" diff --git a/test/scripts/imgtestlib/cache.py b/test/scripts/imgtestlib/cache.py index 20c69386fe..c317875fea 100644 --- a/test/scripts/imgtestlib/cache.py +++ b/test/scripts/imgtestlib/cache.py @@ -7,7 +7,7 @@ from .build import (gen_build_name, get_manifest_id, read_build_info, write_build_info) from .gitlab import log_section -from .run import runcmd_nc +from .run import runcmd, runcmd_nc from .testenv import get_host_distro, get_osbuild_commit S3_BUCKET = "s3://" + os.environ.get("AWS_BUCKET", "images-ci-cache") @@ -149,3 +149,32 @@ def upload_results(distro, arch, image_type, config_path): print(f"⬆️ Uploading info.json to {s3url}") runcmd_nc(["aws", "s3", "cp", "--no-progress", "--acl=private", "--recursive", build_dir+"/", s3url]) print("✅ DONE!!") + + +def s3ls(path): + """ + Calls 'aws s3api list-objects-v2' and returns a list of S3 URLs for each object in the path. + Expects an S3 URL. + """ + if not path.startswith("s3://"): + raise ValueError(f"unexpected path {path}: expected S3 URL (s3://)") + + path = path.removeprefix("s3://") + bucket, prefix = path.split("/", 1) + response_json, _ = runcmd(["aws", "s3api", "list-objects-v2", + "--output", "json", + "--bucket", bucket, + "--prefix", prefix]) + + response = json.loads(response_json) + if "Contents" not in response: + raise RuntimeError(f"unexpected or empty response: {response_json}") + + item_paths = [] + for item in response["Contents"]: + if "Key" in item: # explicitly ignore items with no Key + key = item["Key"] + item_s3url = f"s3://{bucket}/{key}" + item_paths.append(item_s3url) + + return item_paths diff --git a/test/scripts/imgtestlib/core.py b/test/scripts/imgtestlib/core.py index e0d23a1ee2..650c51db7f 100644 --- a/test/scripts/imgtestlib/core.py +++ b/test/scripts/imgtestlib/core.py @@ -7,7 +7,7 @@ from typing import Dict from .build import get_manifest_id -from .cache import dl_build_info, gen_build_info_dir_path_prefix, touch_s3 +from .cache import dl_build_info, gen_build_info_dir_path_prefix from .gitlab import log_section from .run import runcmd from .testenv import (get_bib_ref, get_ci_runner_for, host_container_arch, diff --git a/test/scripts/vmtest/vm.py b/test/scripts/imgtestlib/vm.py similarity index 90% rename from test/scripts/vmtest/vm.py rename to test/scripts/imgtestlib/vm.py index 5c3a2eb88a..b768e764da 100644 --- a/test/scripts/vmtest/vm.py +++ b/test/scripts/imgtestlib/vm.py @@ -1,10 +1,13 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import abc import os import pathlib import platform import shlex import shutil -import subprocess +import socket +import subprocess as sp import sys import tempfile import time @@ -12,7 +15,12 @@ from io import StringIO try: - # The vmtest module is imported by imgtestlib. + # Allow importing the module without boto3 since most times it's not needed. + from qemu import qmp +except ImportError: + qmp = None + +try: # Allow importing the module without boto3 since most times it's not needed. import boto3 from botocore.exceptions import ClientError @@ -20,8 +28,6 @@ boto3 = None ClientError = None -from vmtest.util import get_free_port, wait_ssh_ready - AWS_REGION = "us-east-1" @@ -80,7 +86,7 @@ def _ensure_ssh(self, user, password="", keyfile=None): try: self._run("true", user=user, password=password, keyfile=keyfile) return - except Exception as e: + except Exception as e: # pylint: disable=broad-exception-caught print(f"ssh not ready {i+1}/{ssh_ready_n_retries}: {e}", file=sys.stderr) time.sleep(ssh_ready_wait_sec) raise RuntimeError(f"no ssh after {ssh_ready_n_retries} retries of {ssh_ready_wait_sec}s") @@ -104,20 +110,21 @@ def _run(self, args, user, password="", keyfile=None): ssh_cmd.append(f"{user}@{self._address}") ssh_cmd.append(run_cmd) output = StringIO() - with subprocess.Popen( + with sp.Popen( ssh_cmd, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + stdout=sp.PIPE, stderr=sp.STDOUT, text=True, bufsize=1, ) as p: for out in p.stdout: self._log(out) output.write(out) - ret = subprocess.CompletedProcess(run_cmd, p.returncode) + ret = sp.CompletedProcess(run_cmd, p.returncode) ret.stdout = output.getvalue() # this will raise an CalledProcessError on error ret.check_returncode() return ret + # pylint: disable=too-many-arguments,too-many-positional-arguments def scp(self, src, dst, user, password="", keyfile=None): self._ensure_ssh(user, password, keyfile) scp_cmd = self._sshpass(password) + [ @@ -127,7 +134,7 @@ def scp(self, src, dst, user, password="", keyfile=None): scp_cmd.extend(["-i", keyfile]) scp_cmd.append(src) scp_cmd.append(f"{user}@{self._address}:{dst}") - subprocess.check_call(scp_cmd) + sp.check_call(scp_cmd) @property def ssh_port(self): @@ -157,7 +164,9 @@ def find_ovmf(): raise ValueError("cannot find a OVMF bios") +# pylint: disable=too-many-instance-attributes class QEMU(VM): + # pylint: disable=too-many-arguments,too-many-positional-arguments def __init__(self, img, arch="", snapshot=True, cdrom=None, extra_args=None, memory="2048"): super().__init__() self._img = pathlib.Path(img) @@ -183,6 +192,7 @@ def _num_cores(self): """ return os.cpu_count() or 1 + # pylint: disable=too-many-branches def _gen_qemu_cmdline(self, snapshot, use_ovmf): virtio_scsi_hd = [ "-device", "virtio-scsi-pci,id=scsi", @@ -270,7 +280,7 @@ def start(self, wait_event="ssh", snapshot=True, use_ovmf=False, timeout_sec=180 # XXX: use systemd-run to ensure cleanup? # pylint: disable=consider-using-with - self._qemu_p = subprocess.Popen( + self._qemu_p = sp.Popen( self._gen_qemu_cmdline(snapshot, use_ovmf), stdout=sys.stdout, stderr=sys.stderr, @@ -295,10 +305,8 @@ def _wait_qmp_socket(self, timeout_sec): raise TimeoutError(f"no {self._qmp_socket} after {timeout_sec} seconds") def wait_qmp_event(self, qmp_event, timeout_sec=120): - # import lazy to avoid requiring it for all operations - import qmp # pylint: disable=import-outside-toplevel self._wait_qmp_socket(30) - mon = qmp.QEMUMonitorProtocol(os.fspath(self._qmp_socket)) + mon = qmp.legacy.QEMUMonitorProtocol(os.fspath(self._qmp_socket)) mon.connect() start = time.monotonic() while True: @@ -424,3 +432,25 @@ def force_stop(self): def running(self): return self._ec2_instance is not None + + +def get_free_port() -> int: + # this is racy but there is no race-free way to do better with the qemu CLI + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("localhost", 0)) + return s.getsockname()[1] + + +def wait_ssh_ready(address, port, sleep, max_wait_sec): + for _ in range(int(max_wait_sec / sleep)): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(sleep) + try: + s.connect((address, port)) + data = s.recv(256) + if b"OpenSSH" in data: + return + except (ConnectionRefusedError, ConnectionResetError, TimeoutError): + pass + time.sleep(sleep) + raise ConnectionRefusedError(f"cannot connect to port {port} after {max_wait_sec}s") diff --git a/test/scripts/install-dependencies b/test/scripts/install-dependencies index 178f53e7b3..93d90e54f7 100755 --- a/test/scripts/install-dependencies +++ b/test/scripts/install-dependencies @@ -10,11 +10,12 @@ install_deps() { btrfs-progs-devel \ cloud-utils \ device-mapper-devel \ - libvirt-devel \ gcc \ go \ gpgme-devel \ + iproute \ krb5-devel \ + libvirt-devel \ lorax \ osbuild \ osbuild-depsolve-dnf \ @@ -23,25 +24,27 @@ install_deps() { osbuild-ostree \ osbuild-selinux \ podman \ + pylint \ python3 \ + python3-boto3 \ python3-pip \ python3-pytest \ - python3-boto3 \ python3-qemu-qmp \ python3-scp \ - yamllint \ - xz + sshpass \ + xz \ + yamllint } install_deps_el() { dnf -y install \ awscli2 \ device-mapper-devel \ - libvirt-devel \ gcc \ go \ gpgme-devel \ krb5-devel \ + libvirt-devel \ lorax \ osbuild \ osbuild-depsolve-dnf \ @@ -65,11 +68,3 @@ case "${ID}" in install_deps ;; esac - -# We need this for the qemu boot tests and for any python tests but -# could skip it for other testing. This can be removed once we figure -# out why it actually does not work: -# -# https://github.com/osbuild/images/issues/2288 -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -pip install "${SCRIPT_DIR}/../.." diff --git a/test/scripts/setup-osbuild-repo b/test/scripts/setup-osbuild-repo index 859970fe16..a1a219ba28 100755 --- a/test/scripts/setup-osbuild-repo +++ b/test/scripts/setup-osbuild-repo @@ -49,10 +49,10 @@ def write_repo(commit, distro_version): repofile.write(REPO_TEMPLATE.format(commit=commit, baseurl=repo_baseurl)) -@testlib.log_section("Setting up osbuild repo") +@testlib.gitlab.log_section("Setting up osbuild repo") def main(): - distro_version = testlib.get_host_distro() - commit_id = testlib.get_osbuild_commit(distro_version) + distro_version = testlib.testenv.get_host_distro() + commit_id = testlib.testenv.get_osbuild_commit(distro_version) if not commit_id: print(f"Error: {distro_version} does not have the osbuild commit ID defined in the Schutzfile") sys.exit(1) diff --git a/test/scripts/test_imgtestlib.py b/test/scripts/test_imgtestlib.py index 3115a6ff13..465ce6cfcf 100644 --- a/test/scripts/test_imgtestlib.py +++ b/test/scripts/test_imgtestlib.py @@ -1,8 +1,11 @@ +import contextlib import json import os +import shutil import subprocess as sp import tempfile -from unittest.mock import patch +import textwrap +from unittest.mock import call, patch import pytest @@ -20,14 +23,14 @@ def can_sudo_nopw() -> bool: def test_runcmd(): - stdout, stderr = testlib.runcmd(["/bin/echo", "hello"]) + stdout, stderr = testlib.run.runcmd(["/bin/echo", "hello"]) assert stdout == b"hello\n" assert stderr == b"" def test_runcmd_env(): os.environ["RUNCMD_GLOBAL_TEST_VAR"] = "global test value" - stdout, stderr = testlib.runcmd(["env"], extra_env={"RUNCMD_TEST_VAR": "the test value"}) + stdout, stderr = testlib.run.runcmd(["env"], extra_env={"RUNCMD_TEST_VAR": "the test value"}) assert b"RUNCMD_TEST_VAR=the test value\n" in stdout, "extra env var not set" assert b"RUNCMD_GLOBAL_TEST_VAR=global test value\n" in stdout, "global env vars not preserved" assert stderr == b"" @@ -35,7 +38,7 @@ def test_runcmd_env(): def test_read_seed(): # check that it's read without error - no need to test the value itself - seed_env = testlib.rng_seed_env() + seed_env = testlib.testenv.rng_seed_env() assert "OSBUILD_TESTING_RNG_SEED" in seed_env @@ -118,7 +121,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): # we need to patch the functions that were imported into the cache namespace, not the originals in .testenv with patch("imgtestlib.cache.get_host_distro", return_value="fedora-999"), \ patch("imgtestlib.cache.get_osbuild_commit", return_value="abcdef123456"): - assert testlib.gen_build_info_dir_path_prefix(**kwargs) == expected + assert testlib.cache.gen_build_info_dir_path_prefix(**kwargs) == expected @pytest.mark.parametrize("kwargs,expected", ( @@ -130,7 +133,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "arch": "aarch64", "manifest_id": "abc123" }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/aarch64/manifest-id-abc123/", ), ( @@ -140,7 +143,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "distro": "fedora-41", "arch": "aarch64", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/aarch64/", ), ( @@ -149,7 +152,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "runner_distro": "fedora-41", "distro": "fedora-41", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/", ), ( @@ -157,7 +160,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "osbuild_ref": "abcdef123456", "runner_distro": "fedora-41", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/", ), # Optional arg 'distro' not specified, thus following optional args 'arch' and 'manifest_id' are ignored @@ -168,7 +171,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "arch": "aarch64", "manifest_id": "abc123" }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/", ), # Optional arg 'arch' not specified, thus following optional arg 'manifest_id' is ignored @@ -179,7 +182,7 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "distro": "fedora-41", "manifest_id": "abc123" }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/", ), # default osbuild_ref @@ -187,26 +190,26 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): { "runner_distro": "fedora-41", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/" + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-41/" ), # default runner_distro ( { "osbuild_ref": "abc123", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + "/osbuild-ref-abc123/runner-fedora-999/" + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abc123/runner-fedora-999/" ), # default osbuild_ref and runner_distro ( {}, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-999/" + testlib.cache.S3_BUCKET + "/" + testlib.cache.S3_PREFIX + "/osbuild-ref-abcdef123456/runner-fedora-999/" ), )) def test_gen_build_info_s3_dir_path(kwargs, expected): # we need to patch the functions that were imported into the cache namespace, not the originals in .testenv with patch("imgtestlib.cache.get_host_distro", return_value="fedora-999"), \ patch("imgtestlib.cache.get_osbuild_commit", return_value="abcdef123456"): - assert testlib.gen_build_info_s3_dir_path(**kwargs) == expected + assert testlib.cache.gen_build_info_s3_dir_path(**kwargs) == expected test_container = "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/manifest-list-test" @@ -236,8 +239,8 @@ def test_gen_build_info_s3_dir_path(kwargs, expected): def test_skopeo_inspect_id_manifest_list(arch): transport = "docker://" image_id = image_ids[arch] - assert testlib.skopeo_inspect_id(f"{transport}{test_container}:latest", arch) == image_id - assert testlib.skopeo_inspect_id(f"{transport}{test_container}@{manifest_list_digest}", arch) == image_id + assert testlib.core.skopeo_inspect_id(f"{transport}{test_container}:latest", arch) == image_id + assert testlib.core.skopeo_inspect_id(f"{transport}{test_container}@{manifest_list_digest}", arch) == image_id @pytest.mark.parametrize("arch", TEST_ARCHES) @@ -246,7 +249,7 @@ def test_skopeo_inspect_image_manifest(arch): manifest_id = manifest_ids[arch] image_id = image_ids[arch] # arch arg to skopeo_inspect_id doesn't matter here - assert testlib.skopeo_inspect_id(f"{transport}{test_container}@{manifest_id}", arch) == image_id + assert testlib.core.skopeo_inspect_id(f"{transport}{test_container}@{manifest_id}", arch) == image_id @pytest.mark.skipif(not can_sudo_nopw(), reason="requires passwordless sudo") @@ -256,10 +259,11 @@ def test_skopeo_inspect_localstore(arch): transport = "containers-storage:" image = "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/manifest-list-test:latest" with tempfile.TemporaryDirectory() as tmpdir: - testlib.runcmd(["sudo", "podman", "pull", f"--arch={arch}", "--storage-driver=vfs", f"--root={tmpdir}", image]) + testlib.run.runcmd(["sudo", "podman", "pull", + f"--arch={arch}", "--storage-driver=vfs", f"--root={tmpdir}", image]) # arch arg to skopeo_inspect_id doesn't matter here - assert testlib.skopeo_inspect_id(f"{transport}[vfs@{tmpdir}]{image}", arch) == image_ids[arch] + assert testlib.core.skopeo_inspect_id(f"{transport}[vfs@{tmpdir}]{image}", arch) == image_ids[arch] def test_find_image_file_single_export(): @@ -284,7 +288,7 @@ def test_find_image_file_single_export(): f.write("fake image") # Test that it finds the correct file - result = testlib.find_image_file(tmpdir) + result = testlib.core.find_image_file(tmpdir) assert result == image_file @@ -312,7 +316,7 @@ def test_find_image_file_multiple_pipelines_one_export(): f.write("fake archive") # Test that it finds the correct file from the exported pipeline - result = testlib.find_image_file(tmpdir) + result = testlib.core.find_image_file(tmpdir) assert result == image_file @@ -332,7 +336,7 @@ def test_find_image_file_no_export_directory(): # Don't create any export directories with pytest.raises(RuntimeError, match="Expected exactly one exported pipeline directory"): - testlib.find_image_file(tmpdir) + testlib.core.find_image_file(tmpdir) def test_find_image_file_multiple_export_directories(): @@ -359,7 +363,7 @@ def test_find_image_file_multiple_export_directories(): # Should raise error about multiple export directories with pytest.raises(RuntimeError, match="Expected exactly one exported pipeline directory"): - testlib.find_image_file(tmpdir) + testlib.core.find_image_file(tmpdir) def test_find_image_file_no_files_in_export(): @@ -382,7 +386,7 @@ def test_find_image_file_no_files_in_export(): # Should raise error about no files with pytest.raises(RuntimeError, match="Expected exactly one file in export directory"): - testlib.find_image_file(tmpdir) + testlib.core.find_image_file(tmpdir) def test_find_image_file_multiple_files_in_export(): @@ -408,4 +412,132 @@ def test_find_image_file_multiple_files_in_export(): # Should raise error about multiple files with pytest.raises(RuntimeError, match="Expected exactly one file in export directory"): - testlib.find_image_file(tmpdir) + testlib.core.find_image_file(tmpdir) + + +def test_get_free_port(): + port_nr = testlib.vm.get_free_port() + assert 1024 < port_nr < 65535 + + +@patch("time.sleep") +def test_wait_ssh_ready_sleeps_no_connection(mocked_sleep): + free_port = testlib.vm.get_free_port() + with pytest.raises(ConnectionRefusedError): + testlib.vm.wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.35) + assert mocked_sleep.call_args_list == [call(0.1), call(0.1), call(0.1)] + + +@pytest.mark.skipif(not shutil.which("nc"), reason="needs nc") +def test_wait_ssh_ready_sleeps_wrong_reply(): + free_port = testlib.vm.get_free_port() + with contextlib.ExitStack() as cm: + with sp.Popen( + f"echo not-ssh | nc -vv -l {free_port}", + shell=True, + stdout=sp.PIPE, + stderr=sp.STDOUT, + encoding="utf-8", + ) as p: + cm.callback(p.kill) + # wait for nc to be ready + while True: + # netcat tranditional uses "listening", others "Listening" + # so just omit the first char + if "istening " in p.stdout.readline(): + break + # now connect + with patch("time.sleep") as mocked_sleep: + with pytest.raises(ConnectionRefusedError): + testlib.vm.wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.55) + assert mocked_sleep.call_args_list == [ + call(0.1), call(0.1), call(0.1), call(0.1), call(0.1)] + + +class MockVM(testlib.vm.VM): + _address = None + _ssh_port = None + + def start(self): + pass + + def force_stop(self): + pass + + def running(self): + return True + + def set_ssh(self, address, port): + self._address = address + self._ssh_port = port + + +def make_fake_ssh(fake_bin_path, extra_script=""): + mock_ssh = fake_bin_path / "ssh" + mock_ssh.write_text(textwrap.dedent(f"""\ + #!/bin/bash -e + echo "calling $0 with: $@" + {extra_script} + """)) + mock_ssh.chmod(0o755) + return mock_ssh + + +def test_ssh_calls_cmd_happy(tmp_path, monkeypatch): + monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) + make_fake_ssh(tmp_path) + vm = MockVM() + res = vm.run(["cmd1", "arg1", "arg2"], user="user1", keyfile="keyfile1") + assert res.returncode == 0 + assert res.stdout.endswith("cmd1 arg1 arg2\n") + + +def test_ssh_calls_cmd_happy_single_cmd(tmp_path, monkeypatch): + monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) + make_fake_ssh(tmp_path) + vm = MockVM() + res = vm.run("true", user="user1", keyfile="keyfile1") + assert res.returncode == 0 + assert res.stdout.endswith("true\n") + + +def test_ssh_calls_cmd_happy_quoting_works(tmp_path, monkeypatch): + monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) + make_fake_ssh(tmp_path) + vm = MockVM() + res = vm.run("this needs quoting", user="user1", keyfile="keyfile1") + assert res.returncode == 0 + assert res.stdout.endswith("'this needs quoting'\n") + + +def test_ssh_calls_cmd_sad(tmp_path, monkeypatch): + monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) + make_fake_ssh(tmp_path, """ echo bad-output ; if [ "${@: -1}" = "bad-cmd" ]; then exit 42; fi """) + vm = MockVM() + with pytest.raises(sp.CalledProcessError) as e: + vm.run("bad-cmd", user="user1", keyfile="keyfile1") + assert e.value.returncode == 42 + assert e.value.stdout.endswith("bad-output\n") + + +@patch("time.sleep") +def test_ssh_calls_retries(mocked_sleep, tmp_path, monkeypatch, capsys): + monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) + make_fake_ssh(tmp_path, "echo ssh-very-sad; exit 21") + vm = MockVM() + with pytest.raises(RuntimeError) as e: + vm.run(["cmd1", "arg1", "arg2"], user="user1", keyfile="keyfile1") + assert str(e.value) == "no ssh after 30 retries of 10s" + assert mocked_sleep.call_args_list == 30 * [call(10)] + assert capsys.readouterr().err == "\n".join([ + f"ssh not ready {i+1}/30: Command 'true' returned non-zero exit status 21." + for i in range(30) + ]) + "\n" + + +def test_wait_ssh_ready_timeout(): + vm = MockVM() + vm.set_ssh("localhost", testlib.vm.get_free_port()) + with pytest.raises(ConnectionRefusedError) as e: + vm.wait_ssh_ready(timeout_sec=3) + assert "after 3s" in str(e.value) diff --git a/test/scripts/update-schutzfile-bib b/test/scripts/update-schutzfile-bib index f24791ad1d..713e5c06f7 100755 --- a/test/scripts/update-schutzfile-bib +++ b/test/scripts/update-schutzfile-bib @@ -8,9 +8,9 @@ import imgtestlib as testlib def main(): ref = "quay.io/centos-bootc/bootc-image-builder:latest" cmd = ["skopeo", "inspect", "--raw", f"docker://{ref}"] - out, _ = testlib.runcmd(cmd) + out, _ = testlib.run.runcmd(cmd) data = json.loads(out) - if not testlib.is_manifest_list(data): + if not testlib.core.is_manifest_list(data): raise RuntimeError(f"{ref} is not a manifest list") checksum = hashlib.sha256(out).hexdigest() @@ -18,11 +18,11 @@ def main(): ref_no_tag = ":".join(ref.split(":")[:-1]) new_ref = f"{ref_no_tag}@{digest}" - with open(testlib.SCHUTZFILE, encoding="utf-8") as schutzfile: + with open(testlib.testenv.SCHUTZFILE, encoding="utf-8") as schutzfile: data = json.load(schutzfile) data["common"]["dependencies"]["bootc-image-builder"]["ref"] = new_ref - with open(testlib.SCHUTZFILE, "w", encoding="utf-8") as schutzfile: + with open(testlib.testenv.SCHUTZFILE, "w", encoding="utf-8") as schutzfile: json.dump(data, schutzfile, indent=2) diff --git a/test/scripts/update-schutzfile-osbuild b/test/scripts/update-schutzfile-osbuild index 19a59ab193..053a595b8b 100755 --- a/test/scripts/update-schutzfile-osbuild +++ b/test/scripts/update-schutzfile-osbuild @@ -36,7 +36,7 @@ def osbuild_main_commit_id(): def update_osbuild_commit_ids(new): - with open(testlib.SCHUTZFILE, encoding="utf-8") as schutzfile: + with open(testlib.testenv.SCHUTZFILE, encoding="utf-8") as schutzfile: data = json.load(schutzfile) unique_changes = [] @@ -50,7 +50,7 @@ def update_osbuild_commit_ids(new): unique_changes.append(change) data[distro].setdefault("dependencies", {}).setdefault("osbuild", {})["commit"] = new - with open(testlib.SCHUTZFILE, encoding="utf-8", mode="w") as schutzfile: + with open(testlib.testenv.SCHUTZFILE, encoding="utf-8", mode="w") as schutzfile: json.dump(data, schutzfile, indent=" ") with open("github_pr_body.txt", encoding="utf-8", mode="w") as pr_body: diff --git a/test/scripts/upload-results b/test/scripts/upload-results index 4eae11d43e..55803e286a 100755 --- a/test/scripts/upload-results +++ b/test/scripts/upload-results @@ -18,7 +18,7 @@ def main(): config_path = args.config arch = os.uname().machine - testlib.upload_results(distro, arch, image_type, config_path) + testlib.cache.upload_results(distro, arch, image_type, config_path) if __name__ == "__main__": diff --git a/test/scripts/validate-config-list b/test/scripts/validate-config-list index ffacecc4d0..ed982fdaec 100755 --- a/test/scripts/validate-config-list +++ b/test/scripts/validate-config-list @@ -56,7 +56,7 @@ def validate_build_config(config_list): arches = filters.get("arches", ["*"]) image_types = filters.get("image-types", ["*"]) - matches = testlib.list_images(distros=distros, arches=arches, images=image_types) + matches = testlib.core.list_images(distros=distros, arches=arches, images=image_types) if not matches: no_matches.append((config, filters)) diff --git a/test/scripts/vmtest/__init__.py b/test/scripts/vmtest/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/scripts/vmtest/test/test_ssh.py b/test/scripts/vmtest/test/test_ssh.py deleted file mode 100644 index 739787d9d7..0000000000 --- a/test/scripts/vmtest/test/test_ssh.py +++ /dev/null @@ -1,95 +0,0 @@ -import os -import subprocess -import textwrap -import time -from unittest.mock import call, patch - -import pytest - -import vmtest.vm -from vmtest.vm import VM -from vmtest.util import get_free_port - - -class MockVM(VM): - _address = None - _ssh_port = None - def start(self): - pass - def force_stop(self): - pass - def running(self): - return True - def set_ssh(self, address, port): - self._address = address - self._ssh_port = port - - - -def make_fake_ssh(fake_bin_path, extra_script=""): - mock_ssh = fake_bin_path / "ssh" - mock_ssh.write_text(textwrap.dedent(f"""\ - #!/bin/bash -e - echo "calling $0 with: $@" - {extra_script} - """)) - mock_ssh.chmod(0o755) - return mock_ssh - - -def test_ssh_calls_cmd_happy(tmp_path, monkeypatch): - monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) - make_fake_ssh(tmp_path) - vm = MockVM() - res = vm.run(["cmd1", "arg1", "arg2"], user="user1", keyfile="keyfile1") - assert res.returncode == 0 - assert res.stdout.endswith("cmd1 arg1 arg2\n") - - -def test_ssh_calls_cmd_happy_single_cmd(tmp_path, monkeypatch): - monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) - make_fake_ssh(tmp_path) - vm = MockVM() - res = vm.run("true", user="user1", keyfile="keyfile1") - assert res.returncode == 0 - assert res.stdout.endswith("true\n") - - -def test_ssh_calls_cmd_happy_quoting_works(tmp_path, monkeypatch): - monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) - make_fake_ssh(tmp_path) - vm = MockVM() - res = vm.run("this needs quoting", user="user1", keyfile="keyfile1") - assert res.returncode == 0 - assert res.stdout.endswith("'this needs quoting'\n") - -def test_ssh_calls_cmd_sad(tmp_path, monkeypatch): - monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) - make_fake_ssh(tmp_path, """ echo bad-output ; if [ "${@: -1}" = "bad-cmd" ]; then exit 42; fi """) - vm = MockVM() - with pytest.raises(subprocess.CalledProcessError) as e: - vm.run("bad-cmd", user="user1", keyfile="keyfile1") - assert e.value.returncode == 42 - assert e.value.stdout.endswith("bad-output\n") - - -@patch("time.sleep") -def test_ssh_calls_retries(mocked_sleep, tmp_path, monkeypatch, capsys): - monkeypatch.setenv("PATH", os.fspath(tmp_path), prepend=os.pathsep) - make_fake_ssh(tmp_path, "echo ssh-very-sad; exit 21") - vm = MockVM() - with pytest.raises(RuntimeError) as e: - res = vm.run(["cmd1", "arg1", "arg2"], user="user1", keyfile="keyfile1") - assert str(e.value) == "no ssh after 30 retries of 10s" - assert mocked_sleep.call_args_list == 30 * [ call(10) ] - assert capsys.readouterr().err == "\n".join([ - f"ssh not ready {i+1}/30: Command 'true' returned non-zero exit status 21." - for i in range(30) - ]) + "\n" - -def test_wait_ssh_ready_timeout(): - vm = MockVM() - vm.set_ssh("localhost", get_free_port()) - with pytest.raises(ConnectionRefusedError) as e: - vm.wait_ssh_ready(timeout_sec=3) - assert "after 3s" in str(e.value) diff --git a/test/scripts/vmtest/util.py b/test/scripts/vmtest/util.py deleted file mode 100644 index 195f52134a..0000000000 --- a/test/scripts/vmtest/util.py +++ /dev/null @@ -1,24 +0,0 @@ -import socket -import time - - -def get_free_port() -> int: - # this is racy but there is no race-free way to do better with the qemu CLI - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("localhost", 0)) - return s.getsockname()[1] - - -def wait_ssh_ready(address, port, sleep, max_wait_sec): - for _ in range(int(max_wait_sec / sleep)): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(sleep) - try: - s.connect((address, port)) - data = s.recv(256) - if b"OpenSSH" in data: - return - except (ConnectionRefusedError, ConnectionResetError, TimeoutError): - pass - time.sleep(sleep) - raise ConnectionRefusedError(f"cannot connect to port {port} after {max_wait_sec}s") diff --git a/test/scripts/vmtest/util_test.py b/test/scripts/vmtest/util_test.py deleted file mode 100644 index 6d91720e13..0000000000 --- a/test/scripts/vmtest/util_test.py +++ /dev/null @@ -1,47 +0,0 @@ -import contextlib -import shutil -import subprocess -from unittest.mock import call, patch - -import pytest - -from vmtest.util import get_free_port, wait_ssh_ready - - -def test_get_free_port(): - port_nr = get_free_port() - assert 1024 < port_nr < 65535 - - -@patch("time.sleep") -def test_wait_ssh_ready_sleeps_no_connection(mocked_sleep): - free_port = get_free_port() - with pytest.raises(ConnectionRefusedError): - wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.35) - assert mocked_sleep.call_args_list == [call(0.1), call(0.1), call(0.1)] - - -@pytest.mark.skipif(not shutil.which("nc"), reason="needs nc") -def test_wait_ssh_ready_sleeps_wrong_reply(): - free_port = get_free_port() - with contextlib.ExitStack() as cm: - with subprocess.Popen( - f"echo not-ssh | nc -vv -l -p {free_port}", - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - encoding="utf-8", - ) as p: - cm.callback(p.kill) - # wait for nc to be ready - while True: - # netcat tranditional uses "listening", others "Listening" - # so just omit the first char - if "istening " in p.stdout.readline(): - break - # now connect - with patch("time.sleep") as mocked_sleep: - with pytest.raises(ConnectionRefusedError): - wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.55) - assert mocked_sleep.call_args_list == [ - call(0.1), call(0.1), call(0.1), call(0.1), call(0.1)] diff --git a/test/test_build_integration.py b/test/test_build_integration.py index a1c1fd77b8..11f0514a7b 100644 --- a/test/test_build_integration.py +++ b/test/test_build_integration.py @@ -1,3 +1,5 @@ +# allow TODO and XXX in the whole file +# pylint: disable=fixme import os import platform import subprocess @@ -22,7 +24,8 @@ def _test_cases(): boot_tests = set() for tcase in all_test_cases: _, arch, image_type, _ = tcase.split(",") - if not (image_type in testlib.CAN_BOOT_TEST["*"] or image_type in testlib.CAN_BOOT_TEST.get(arch, [])): + if not (image_type in testlib.core.CAN_BOOT_TEST["*"] or + image_type in testlib.core.CAN_BOOT_TEST.get(arch, [])): continue # XXX: we need to filter further here, i.e. not all qemu tests can be run currently, e.g. # if sshd is missing (see boot_image.ensure_can_run_qemu_test - however this needs @@ -44,7 +47,7 @@ def test_build_only(distro, arch, image_type, config_name): config_path = f"test/configs/{config_name}.json" subprocess.check_call( ["./test/scripts/build-image", distro, image_type, config_path]) - build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name)) + build_dir = os.path.join("build", testlib.build.gen_build_name(distro, arch, image_type, config_name)) subprocess.check_call( ["./test/scripts/boot-image", build_dir, config_path]) @@ -56,6 +59,6 @@ def test_build_and_boot(distro, arch, image_type, config_name): config_path = f"test/configs/{config_name}.json" subprocess.check_call( ["./test/scripts/build-image", distro, image_type, config_path]) - build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name)) + build_dir = os.path.join("build", testlib.build.gen_build_name(distro, arch, image_type, config_name)) subprocess.check_call( ["./test/scripts/boot-image", build_dir, config_path]) diff --git a/test/test_cross_arch_integration.py b/test/test_cross_arch_integration.py index 04e917ea19..57260fbe6b 100644 --- a/test/test_cross_arch_integration.py +++ b/test/test_cross_arch_integration.py @@ -29,6 +29,6 @@ def test_build_boot_cross_arch_smoke(arch): config_path = f"test/configs/{config_name}.json" subprocess.check_call( ["./test/scripts/build-image", f"--arch={arch}", distro, image_type, config_path]) - build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name)) + build_dir = os.path.join("build", testlib.build.gen_build_name(distro, arch, image_type, config_name)) subprocess.check_call( ["./test/scripts/boot-image", build_dir, config_path])