Updating cloud-init config files and base image

base-image/Containerfile

@@ -5,8 +5,7 @@ RUN dnf -y install cloud-init && \
     rm -rf /var/{cache,log} /var/lib/{dnf,rhsm}
 COPY usr/ /usr/
 
-COPY base_config.sh /usr/bin/
-COPY base_config.service /etc/systemd/system
+COPY base.service /etc/systemd/system
 
-RUN systemctl unmask base_config.service
-RUN systemctl enable base_config.service
+RUN systemctl unmask base.service
+RUN systemctl enable base.service

base-image/README.md

@@ -15,7 +15,7 @@ There are three distros of bootc image available to use. Fedora, CentOS and RHEL
 
     **Note:**
     - You need Red Hat account to get the credentials to pull the image. 
-    - Need to build the image on RHEL machine where subsciption activated.
+    - Need to build the image on RHEL machine where subscription activated.
 
 
 
@@ -28,11 +28,10 @@ COPY usr/ /usr/
 Install cloud-init to configure AI image and PIM partition's network and user
 
 ```Dockerfile
-COPY base_config.sh /usr/bin/
-COPY base_config.service /etc/systemd/system
+COPY base.service /etc/systemd/system
 
-RUN systemctl unmask base_config.service
-RUN systemctl enable base_config.service
+RUN systemctl unmask base.service
+RUN systemctl enable base.service
 ```
 systemd service to setup pimconfig like copying cloud init config and pim config files to respective directory
 

base-image/base_config.service → base-image/base.service

@@ -5,7 +5,7 @@ Wants=network-online.target cloud-config.target
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/env /bin/bash /usr/bin/base_config.sh
+ExecStart=/bin/true
 RemainAfterExit=yes
 TimeoutSec=0 
 

cli/cloud-init-iso/templates/network-config

@@ -0,0 +1,10 @@
+version: 2
+ethernets:
+  env{{ config["partition"]["network"]["slot_num"] }}:
+    dhcp4: false
+    addresses:
+      - {{ config["partition"]["network"]["ip"]["address"] }}/{{ config["partition"]["network"]["ip"]["prefix-length"] }}
+    gateway4: {{ config["partition"]["network"]["ip"]["gateway"] }}
+    nameservers:
+      addresses:
+        - {{ config["partition"]["network"]["ip"]["nameserver"] }}

cli/cloud-init-iso/templates/user-data

@@ -0,0 +1,29 @@
+#cloud-config
+users:
+  - name: {{ config["ssh"]["user-name"] }}
+    sudo: ['ALL=(ALL) NOPASSWD:ALL']
+    lock_passwd: true
+    groups: sudo
+    shell: /bin/bash
+    ssh_authorized_keys:
+      - {{ config["ssh"]["pub-key"] }}
+
+write_files:
+  - path: /etc/pim/pim_config.json
+    content: |
+      {{ config["ai"]["config-json"] }}
+    owner: root:root
+    permissions: '0644'
+    append: false
+  - path: /etc/pim/auth.json
+    content: |
+      {{ config["ai"]["auth-json"] }}
+    owner: root:root
+    permissions: '0644'
+    append: false
+  - path: /etc/pim/env.conf
+    content: |
+      REGISTRY_AUTH_FILE=/etc/pim/auth.json
+    owner: root:root
+    permissions: '0644'
+    append: false

cli/cmd/update_config.py

@@ -1,15 +1,14 @@
 import shutil
+from scp import SCPClient
+import os
 
 import cli.network.virtual_network as virtual_network
-import cli.partition.activation as activation
 import cli.partition.partition as partition
 import cli.utils.monitor_util as monitor_util
-import cli.storage.vopt_storage as vopt
 import cli.utils.command_util as command_util
 import cli.utils.common as common
 import cli.utils.iso_util as iso_util
 import cli.utils.string_util as util
-import cli.vios.vios as vios
 
 
 logger = common.get_logger("pim-update-config")
@@ -20,18 +19,18 @@ def update_config(config_file_path):
         logger.info("Updating PIM partition's config")
         config = common.initialize_config(config_file_path)
         # Invoking initialize_command to perform common actions like validation, authentication etc.
-        is_config_valid, cookies, sys_uuid, vios_uuid_list = command_util.initialize_command(
+        is_config_valid, cookies, sys_uuid, _ = command_util.initialize_command(
             config)
         if is_config_valid:
-            _update_config(config, cookies, sys_uuid, vios_uuid_list)
+            _update_config(config, cookies, sys_uuid)
             logger.info("PIM partition's config successfully updated")
     except Exception as e:
         logger.error(f"encountered an error: {e}")
     finally:
         if cookies:
             command_util.cleanup(config, cookies)
 
-def _update_config(config, cookies, sys_uuid, vios_uuid_list):
+def _update_config(config, cookies, sys_uuid):
     try:
         logger.debug("Checking partition exists")
         exists, _, partition_uuid = partition.check_partition_exists(config, cookies, sys_uuid)
@@ -60,34 +59,45 @@ def _update_config(config, cookies, sys_uuid, vios_uuid_list):
             shutil.rmtree(common.cloud_init_update_config_dir)
             return
         logger.info("Detected config change, updating")
-        iso_util.generate_cloud_init_iso_file(common.update_iso_dir, config, common.cloud_init_update_config_dir)
 
-        logger.info("Shutting down the partition")
-        activation.shutdown_partition(config, cookies, partition_uuid)
-        logger.info("Partition shut down to attach the new config")
+        # Create pim_config.json file
+        pim_config = util.get_pim_config_json(config)
+        with open(f"{common.cloud_init_update_config_dir}/pim_config.json", "w") as config_file:
+            config_file.write(pim_config)
 
-        cloud_init_iso = util.get_cloud_init_iso(config)
-        logger.info("Uploading the new cloud init with the config changes")
-        vios_cloudinit_media_uuid = iso_util.upload_iso_to_media_repository(config, cookies, common.update_iso_dir, cloud_init_iso, sys_uuid, vios_uuid_list)
-        logger.debug("Cloud init uploaded")
-
-        logger.info("Attaching the cloud init to the partition")
-        vios_payload = vios.get_vios_details(config, cookies, sys_uuid, vios_cloudinit_media_uuid)
-        vopt.attach_vopt(vios_payload, config, cookies, partition_uuid, sys_uuid, vios_cloudinit_media_uuid, cloud_init_iso)
-        logger.info("New cloud init config attached to the partition.")
+        ssh_client = common.ssh_to_partition(config)
 
-        logger.info("Activating the partition")
-        activation.activate_partition(config, cookies, partition_uuid)
-        logger.info("Partition activated")
+        with SCPClient(ssh_client.get_transport()) as scp:
+            scp.put(f'{common.cloud_init_update_config_dir}/pim_config.json', '/tmp')
+
+        move_cmd = "sudo mv /tmp/pim_config.json /etc/pim/"
+        _, stdout, stderr = ssh_client.exec_command(move_cmd)
+        exit_status = stdout.channel.recv_exit_status()
+        if exit_status == 0:
+            logger.info("Successfully updated the config of the partition.")
+        else:
+            errorMsg = stderr.read().decode('utf-8')
+            logger.error(f"failed to update config of the partition. error: {errorMsg}")
+            raise Exception(errorMsg)
 
-        logger.info("Monitoring boot process, this will take a while")
-        monitor_util.monitor_pim(config)
+
+        # Restart base.service
+        restart_command = "sudo systemctl restart base.service"
+        _, stdout, stderr = ssh_client.exec_command(restart_command)
+        exit_status = stdout.channel.recv_exit_status()
 
-        # Move used cloud init iso to iso dir
-        shutil.move(f"{common.update_iso_dir}/{cloud_init_iso}", f"{common.iso_dir}/{cloud_init_iso}")
+        if exit_status == 0:
+            logger.info("Successfully restarted base.service")
+        else:
+            errorMsg = stderr.read().decode('utf-8')
+            logger.error(f"failed to restart base.service. error: {errorMsg}")
+            raise Exception(errorMsg)
 
+        os.remove(f"{common.cloud_init_update_config_dir}/pim_config.json")
         # Cleanup existing config and move updated config
         shutil.rmtree(common.cloud_init_config_dir)
         shutil.move(common.cloud_init_update_config_dir, common.cloud_init_config_dir)
+        logger.info("Monitoring AI application, this will take a while")
+        monitor_util.monitor_pim(config)
     except Exception as e:
         raise e

cli/utils/iso_util.py

@@ -28,27 +28,37 @@ def generate_cloud_init_iso_config(config, slot_num, config_dir):
     file_loader = FileSystemLoader(f'{common.getclidir()}/cloud-init-iso/templates')
     env = Environment(loader=file_loader)
 
-    network_config_template = env.get_template('99_custom_network.cfg')
+    network_config_template = env.get_template('network-config')
     network_config_output = network_config_template.render(config=config)
 
-    common.create_dir(config_dir)
-
     pim_config_json = config["ai"]["config-json"] if config["ai"]["config-json"] != "" else "{}"
     pim_config_json = json.loads(pim_config_json)
 
     # 'workloadImage' is being used inside the bootstrap iso to write the bootc image into disk, in case of modification of this field name, needs same modification in bootstrap.iso too.
     pim_config_json["workloadImage"] = get_workload_image(config)
+    config["ai"]["config-json"] = json.dumps(pim_config_json, separators=(',', ':'))
+
+    auth_json = config["ai"]["auth-json"]
+    if auth_json == "":
+        auth_json = "{}"
+    else:
+        auth_data = json.loads(auth_json)
+        auth_json = json.dumps(auth_data, separators=(',', ':'))
+    config["ai"]["auth-json"] = auth_json
 
-    pim_config_file = open(config_dir + "/pim_config.json", "w")
-    pim_config_file.write(json.dumps(pim_config_json))
+    user_data_template = env.get_template('user-data')
+    user_data_output = user_data_template.render(config=config)
+
+    common.create_dir(config_dir)
 
-    network_config_file = open(
-        config_dir + "/99_custom_network.cfg", "w")
+    network_config_file = open(config_dir + "/network-config", "w")
     network_config_file.write(network_config_output)
+
+    user_data_file = open(config_dir + "/user-data", "w")
+    user_data_file.write(user_data_output)
 
-    auth_json = "{}" if config["ai"]["auth-json"] == "" else config["ai"]["auth-json"]
-    auth_config_file = open(config_dir + "/auth.json", "w")
-    auth_config_file.write(auth_json)
+    open(config_dir+"/meta-data", "w")
+
     logger.debug("Generated config files for the cloud-init ISO")
 
 
@@ -57,7 +67,7 @@ def generate_cloud_init_iso_file(iso_dir, config, config_dir):
     common.create_dir(iso_dir)
 
     cloud_init_image_name = get_cloud_init_iso(config)
-    generate_cmd = f"mkisofs -l -o {iso_dir}/{cloud_init_image_name} {config_dir}"
+    generate_cmd = f"mkisofs -l -volid cidata -joliet -o {iso_dir}/{cloud_init_image_name} -rock {config_dir}"
 
     try:
         subprocess.run(generate_cmd.split(), check=True, capture_output=True)

cli/utils/monitor_util.py

@@ -9,48 +9,7 @@
 logger = common.get_logger("monitor")
 
 
-def monitor_pim_boot(config):
-    # Re-run scenario: If lpar is in 2nd boot stage, check base_config service logs
-    logger.debug("PIM boot: Checking base_config.service")
-    try:
-        ssh_client = common.ssh_to_partition(config)
-
-        base_config_svc_exists = "ls /etc/systemd/system/base_config.service"
-        _, stdout, _ = ssh_client.exec_command(
-            base_config_svc_exists, get_pty=True)
-        if stdout.channel.recv_exit_status() == 0:
-            logger.debug("base_config.service exists")
-
-            logger.debug("Checking base_config.service logs")
-            base_cfg_svc_cmd = "sudo journalctl -u base_config.service -f 2>&1 | awk '{print} /base_config.sh run successfully/ {print \"Match found: \" $0; exit 0}'"
-            _, stdout, _ = ssh_client.exec_command(
-                base_cfg_svc_cmd, get_pty=True)
-            while True:
-                out = stdout.readline()
-                logger.debug(out)
-                if stdout.channel.exit_status_ready():
-                    if stdout.channel.recv_exit_status() == 0:
-                        logger.debug(
-                            "Found 'base_config.sh run successfully' message")
-                        ssh_client.close()
-                        return
-                if "base_config.service: Failed with result 'exit-code'" in out:
-                    ssh_client.close()
-                    logger.error(f"failed to start AI application. error: {out}")
-                    raise Exception(f"failed to start AI application. error: {out}")
-        else:
-            ssh_client.close()
-            logger.error(
-                "failed to find '/etc/systemd/system/base_config.service', please check console for more possible errors")
-            raise Exception(
-                "failed to  find '/etc/systemd/system/base_config.service', please check console for more possible errors")
-    except Exception as e:
-        logger.error(f"failed to monitor PIM boot, error: {e}")
-        raise Exception(f"failed to monitor PIM boot, error: {e}")
-
-
 def monitor_pim(config):
-    monitor_pim_boot(config)
     logger.info("Partition booted with PIM image")
 
     # No need to validate the AI application deployed via PIM flow if 'ai.validation.request' set to no, can complete the workflow
@@ -113,7 +72,7 @@ def monitor_bootstrap_boot(config):
                     raise Exception(f"failed to detect bootc based PIM AI image install completion signature. error: {out}")
         else:
             logger.debug(
-                "Could not find 'getcontainer.service', will look for 'base_config.service' in PIM boot since it could be a re-run and bootstrap might have already finished")
+                "Could not find 'getcontainer.service' in PIM boot since it could be a re-run and bootstrap might have already finished")
             ssh_client.close()
     except Exception as e:
         logger.error(f"failed to monitor bootstrap boot, error: {e}")

examples/email-ner/email-ner-config.service

@@ -1,7 +1,7 @@
 [Unit]
 Description=Mount and setup email-ner config
-Requires=base_config.service
-After=base_config.service
+Requires=vllm.service
+After=vllm.service
 
 [Service]
 Type=oneshot

examples/email-ner/email-ner.container

@@ -1,7 +1,7 @@
 [Unit]
 Description=Run email named entity recognition app
-Requires=vllm.service
-After=vllm.service
+Requires=email-ner-config.service
+After=email-ner-config.service
 
 [Service]
 Restart=on-failure

examples/hmc-agent/hmc_agent.container

@@ -1,7 +1,7 @@
 [Unit]
 Description=Run simple HMC AI agent application and MCP server
-Requires=vllm.service hmc_config.service hmc_server.service
-After=vllm.service hmc_config.service hmc_server.service
+Requires=hmc_server.service
+After=hmc_server.service
 
 [Service]
 Restart=on-failure

examples/hmc-agent/hmc_config.service

@@ -1,7 +1,7 @@
 [Unit]
 Description=Mount and setup HMC Agent config
-Requires=base_config.service
-After=base_config.service
+Requires=llm_readiness.service
+After=llm_readiness.service
 
 [Service]
 Type=oneshot