Drop gitlab dynamic pipelines [HMS-9712]

Schutzfile

@@ -1,6 +1,6 @@
 {
   "common": {
-    "rngseed": 2026052100,
+    "rngseed": 2026060901,
     "dependencies": {
       "bootc-image-builder": {
         "ref": "quay.io/centos-bootc/bootc-image-builder@sha256:9893e7209e5f449b86ababfd2ee02a58cca2e5990f77b06c3539227531fc8120"

test/scripts/generate-gitlab-ci

@@ -7,26 +7,35 @@ import imgtestlib as testlib
 
 ARCHITECTURES = ["x86_64", "aarch64"]
 MANIFEST_ONLY_ARCHES = ["ppc64le", "s390x"]
-RUNNER = testlib.get_common_ci_runner()
 OSTREE_DISTROS = ["fedora", "rhel-8", "rhel-9", "centos-9"]
 
 
 BASE_CONFIG = """---
 stages:
   - init
-  - gen
   - build
-  - ostree-gen
   - ostree-build
-  - verify
   - finish
 
 variables:
-  # Set the parent pipeline ID as a global variable so that the child pipeline
-  # jobs can use it to retrieve artifacts.
-  PARENT_PIPELINE_ID: $CI_PIPELINE_ID
   GOPROXY: "https://proxy.golang.org,direct"
 
+.base:
+  before_script:
+    - cat schutzbot/team_ssh_keys.txt |
+        tee -a ~/.ssh/authorized_keys > /dev/null
+  interruptible: true
+  retry: 1
+  tags:
+    - terraform
+  variables:
+    PYTHONUNBUFFERED: 1
+
+.terraform:
+  extends: .base
+  tags:
+    - terraform
+
 init:
   stage: init
   interruptible: true
@@ -53,75 +62,53 @@ fail:
   when: on_failure
 """
 
-GEN_TEMPLATE = """
-"generate-build-config: [{distro}, {arch}]":
-  stage: gen
+TEST_IMAGES_TEMPLATE = """
+"test-images: [{distro}, {arch}]":
+  stage: build
   extends: .terraform
+  tags:
+    - {tag}
   variables:
     RUNNER: {runner}-{arch}
     INTERNAL_NETWORK: "true"
   script:
     - sudo -E ./test/scripts/setup-osbuild-repo
-    - sudo ./test/scripts/install-dependencies
-    - ./test/scripts/generate-build-config --distro {distro} --arch {arch} build-config.yml
+    - sudo -E ./test/scripts/install-dependencies
+    - ./test/scripts/test-new-manifests --distro {distro} --arch {arch}
   artifacts:
     paths:
-      - build-config.yml
+      - {logdir}
 """
 
-TRIGGER_TEMPLATE = """
-"image-build-trigger: [{distro}, {arch}]":
-  stage: build
-  trigger:
-    include:
-      - artifact: build-config.yml
-        job: "generate-build-config: [{distro}, {arch}]"
-    strategy: depend
-  needs:
-    - "generate-build-config: [{distro}, {arch}]"
-"""
-
-OSTREE_GEN_TEMPLATE = """
-"generate-ostree-build-config: [{distro}, {arch}]":
-  stage: ostree-gen
+TEST_OSTREE_IMAGES_TEMPLATE = """
+"test-ostree-images: [{distro}, {arch}]":
+  stage: ostree-build
   extends: .terraform
+  tags:
+    - {tag}
   variables:
     RUNNER: {runner}-{arch}
     INTERNAL_NETWORK: "true"
   script:
     - sudo -E ./test/scripts/setup-osbuild-repo
-    - sudo ./test/scripts/install-dependencies
-    - ./test/scripts/generate-ostree-build-config --distro {distro} --arch {arch} build-config.yml build-configs
+    - sudo -E ./test/scripts/install-dependencies
+    - ./test/scripts/test-new-ostree-manifests --distro {distro} --arch {arch}
+  needs:
+    - "test-images: [{distro}, {arch}]"
   artifacts:
     paths:
-      - build-config.yml
-      - build-configs
-  needs:
-    - "image-build-trigger: [{distro}, {arch}]"
+      - {logdir}
 """
 
-OSTREE_TRIGGER_TEMPLATE = """
-"image-build-ostree-trigger: [{distro}, {arch}]":
-  stage: ostree-build
-  trigger:
-    include:
-      - artifact: build-config.yml
-        job: "generate-ostree-build-config: [{distro}, {arch}]"
-    strategy: depend
-  needs:
-    - "generate-ostree-build-config: [{distro}, {arch}]"
-"""
-
-
 MANIFEST_GEN_TEMPLATE = """
 "generate-manifests: [{distro}, {arch}]":
-  stage: gen
+  stage: build
   extends: .terraform
   variables:
     RUNNER: {runner}-x86_64
     INTERNAL_NETWORK: "true"
   script: |
-    sudo -E ./test/scripts/setup-osbuild-repo
+    sudo ./test/scripts/setup-osbuild-repo
     sudo ./test/scripts/install-dependencies
     echo "GOPROXY=$GOPROXY"
     go run ./cmd/gen-manifests --arches {arch} --distros {distro} --workers 10 --metadata=false --output ./manifests
@@ -148,41 +135,36 @@ def main():
     images = testlib.list_images(arches=ARCHITECTURES)
     combos = set()
 
-    gen_stage = []
-    trigger_stage = []
-    ostree_gen_stage = []
-    ostree_trigger_stage = []
+    build_stage = []
+    ostree_build_stage = []
     for img in sort_configs(images):
         combo = (img["distro"], img["arch"])
         if combo in combos:
             continue
 
-        combos.add(combo)
-        gen_stage.append(GEN_TEMPLATE.format(
-            distro=img["distro"],
-            arch=img["arch"],
-            runner=RUNNER))
+        runner = testlib.get_ci_runner_for(img["distro"], img["arch"], "qcow2")
+        tag = testlib.get_tag_for(runner)
 
-        trigger_stage.append(TRIGGER_TEMPLATE.format(
+        combos.add(combo)
+        build_stage.append(TEST_IMAGES_TEMPLATE.format(
             distro=img["distro"],
             arch=img["arch"],
-            runner=RUNNER))
+            runner=runner,
+            tag=tag,
+            logdir=testlib.BUILD_LOG_PATH))
 
         for ostree_distro_prefix in OSTREE_DISTROS:
             if img["distro"].startswith(ostree_distro_prefix):
                 break
         else:
             continue
 
-        ostree_gen_stage.append(OSTREE_GEN_TEMPLATE.format(
+        ostree_build_stage.append(TEST_OSTREE_IMAGES_TEMPLATE.format(
             distro=img["distro"],
             arch=img["arch"],
-            runner=RUNNER))
-
-        ostree_trigger_stage.append(OSTREE_TRIGGER_TEMPLATE.format(
-            distro=img["distro"],
-            arch=img["arch"],
-            runner=RUNNER))
+            runner=runner,
+            tag=tag,
+            logdir=testlib.BUILD_LOG_PATH))
 
     man_only_images = testlib.list_images(arches=MANIFEST_ONLY_ARCHES)
     man_gen_stage = []
@@ -192,15 +174,18 @@ def main():
             continue
 
         combos.add(combo)
-        man_gen_stage.append(MANIFEST_GEN_TEMPLATE.format(distro=img["distro"], arch=img["arch"], runner=RUNNER))
+        runner = testlib.get_ci_runner_for(img["distro"], img["arch"], "qcow2")
+        tag = testlib.get_tag_for(runner)
+        man_gen_stage.append(MANIFEST_GEN_TEMPLATE.format(
+            distro=img["distro"],
+            arch=img["arch"],
+            runner=runner,
+            tag=tag))
 
     with open(config_path, "w", encoding="utf-8") as config_file:
-        config_file.write(BASE_CONFIG.format(runner=RUNNER))
-        config_file.write(testlib.BASE_CONFIG)
-        config_file.write("\n".join(gen_stage))
-        config_file.write("\n".join(trigger_stage))
-        config_file.write("\n".join(ostree_gen_stage))
-        config_file.write("\n".join(ostree_trigger_stage))
+        config_file.write(BASE_CONFIG)
+        config_file.write("\n".join(build_stage))
+        config_file.write("\n".join(ostree_build_stage))
         config_file.write("\n".join(man_gen_stage))
 
 

test/scripts/imgtestlib/build.py

@@ -3,9 +3,11 @@
 from typing import Dict
 
 from .gitlab import log_section
-from .run import runcmd, runcmd_nc
+from .run import runcmd
 from .testenv import get_host_distro, get_osbuild_commit, rng_seed_env
 
+BUILD_LOG_PATH = "./logs"
+
 
 @log_section("Building image")
 def build_image(distro, arch, image_type, config_path):
@@ -14,18 +16,17 @@ def build_image(distro, arch, image_type, config_path):
 
     config_name = config["name"]
 
-    print(f"👷 Building image {distro}/{image_type} using config {config_path}")
-
     # print the config for logging
     print(json.dumps(config, indent=2))
 
     runcmd(["go", "build", "-o", "./bin/build", "./cmd/build"])
 
     cmd = ["sudo", "-E", "./bin/build", "--output", "./build", "--checkpoints", "build",
            "--distro", distro, "--arch", arch, "--type", image_type, "--config", config_path]
-    runcmd_nc(cmd, extra_env=rng_seed_env())
+    stdout, stderr = runcmd(cmd, extra_env=rng_seed_env())
 
-    print("✅ Build finished!!")
+    build_name = gen_build_name(distro, arch, image_type, config_name)
+    save_logs(build_name, stdout, stderr)
 
     # Build artifacts are owned by root. Make them world accessible.
     runcmd(["sudo", "chmod", "a+rwX", "-R", "./build"])
@@ -89,3 +90,14 @@ def gen_build_name(distro, arch, image_type, config_name):
 
 def _u(s):
     return s.replace("-", "_")
+
+
+def save_logs(build_name, out, err):
+    """
+    Save stdout and stderr output for a job to the BUILD_LOG_PATH.
+    """
+    os.makedirs(BUILD_LOG_PATH, exist_ok=True)
+    with open(os.path.join(BUILD_LOG_PATH, f"{build_name}.out"), mode="w", encoding="utf-8") as log:
+        log.write(out.decode())
+    with open(os.path.join(BUILD_LOG_PATH, f"{build_name}.err"), mode="w", encoding="utf-8") as log:
+        log.write(err.decode())