diff --git a/osac-aap/collections/ansible_collections/osac/templates/README.md b/osac-aap/collections/ansible_collections/osac/templates/README.md index 5022f84f15..2759d3f2b7 100644 --- a/osac-aap/collections/ansible_collections/osac/templates/README.md +++ b/osac-aap/collections/ansible_collections/osac/templates/README.md @@ -191,6 +191,7 @@ template role with the filtered tier subset. supports_nfs: true provisioning_targets: - vmaas + # Add caas only when this provider implements guest-cluster provisioning. ``` 3. Implement the three required action task files. Each receives `_provider_tiers` @@ -214,16 +215,16 @@ template role with the filtered tier subset. osac-operator from the Tier API and keyed by `backend_id`) — there is no env-var or K8s Secret fallback for admin credentials. -6. **Provisioning targets:** Each provider handles both VMaaS and CaaS provisioning - targets via the `_provisioning_target` parameter. Currently supported: `vmaas`. - CaaS targets (`hcp_control_plane`, `hcp_worker_root`, `hcp_data_plane`) are - defined in the enum but not yet implemented. +6. **Provisioning targets:** Each provider declares only the targets it implements + via `meta/osac.yaml`. `caas` is currently supported by the `lvms_storage` + provider (installs LVMS on the guest cluster, creates per-tenant StorageClasses). + HCP-level targets (`hcp_control_plane`, `hcp_worker_root`, `hcp_data_plane`) + are defined but not yet implemented by any provider. -**CaaS provisioning targets:** `hcp_control_plane`, `hcp_worker_root`, and -`hcp_data_plane` are defined in the provisioning target enum but not yet implemented. -When implemented, `hcp_data_plane` will support provisioning multiple StorageClasses -into the guest HCP cluster (e.g., separate tiers for databases and general workloads). -Currently, all CaaS targets return an explicit "not yet implemented" error. +**CaaS provisioning targets:** `caas` is supported by the `lvms_storage` provider +(OSAC-3234). `hcp_control_plane`, `hcp_worker_root`, and `hcp_data_plane` remain +unimplemented — they are defined in the provisioning target enum for future +network-attached storage providers that serve HCP worker nodes directly. **Configuration:** Storage tiers arrive via the `osac_job_vars.storage_tier_definitions` extra_var, populated by osac-operator from the Tier API: diff --git a/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/defaults/main.yaml b/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/defaults/main.yaml index 363111dd84..ecf158b867 100644 --- a/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/defaults/main.yaml +++ b/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/defaults/main.yaml @@ -16,3 +16,24 @@ lvms_storage_provisioner: "topolvm.io" # LVMCluster device class name created by osac-installer's configure-lvms.sh hook. lvms_storage_device_class: "vg1" + +# OLM subscription channel for LVMS operator — used when installing LVMS on CaaS +# guest clusters. When empty (default), the role queries the lvms-operator +# PackageManifest on the guest cluster and uses its defaultChannel, which OLM +# sets automatically for the running OCP version. Set explicitly via extra_vars +# or the storage-operations-ig ConfigMap to pin a specific channel (e.g. +# "stable-4.18") and override auto-detection. +lvms_storage_operator_channel: "" + +# Namespace where the LVMS operator is installed via OLM. +lvms_storage_operator_namespace: "openshift-storage" + +# OLM CatalogSource for LVMS operator. Override for disconnected clusters that +# mirror operators to a custom catalog. +lvms_storage_operator_catalog_source: "redhat-operators" +lvms_storage_operator_catalog_namespace: "openshift-marketplace" + +# Block device path to bind to the LVMCluster deviceSelector on CaaS guest clusters +# (e.g. /dev/vdb for the second qcow2 disk created by setup-caas-agents.sh). +# Empty (default): LVMS auto-discovers all eligible block devices. +lvms_storage_data_device: "" diff --git a/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/meta/osac.yaml b/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/meta/osac.yaml index 4937f6f407..0b06b8ecf1 100644 --- a/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/meta/osac.yaml +++ b/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/meta/osac.yaml @@ -1,12 +1,15 @@ --- title: LVMS Storage Provider description: > - Provisions per-tenant StorageClasses on the hub cluster using the LVMS (LVM Storage) - topolvm provisioner. No external backend or credentials required — LVMS is - installed on the hub by osac-installer when lvms.enabled=true. Hub cluster only; - CaaS guest cluster storage is out of scope. + Provisions per-tenant StorageClasses using the LVMS (LVM Storage) topolvm provisioner. + No external backend or credentials required. Hub path: LVMS is installed on the hub by + osac-installer when lvms.enabled=true. CaaS guest-cluster path: LVMS operator is + installed via OLM on the guest cluster when admin_kubeconfig is provided in the event + (requires a raw block device on the worker node — see setup-caas-agents.sh + AGENT_VM_DATA_DISK_SIZE). template_type: storage_provider implementation_strategy: lvms capabilities: provisioning_targets: - vmaas + - caas diff --git a/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/tasks/ensure_storage_class.yaml b/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/tasks/ensure_storage_class.yaml index b3ad4e3215..044bb76884 100644 --- a/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/tasks/ensure_storage_class.yaml +++ b/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/tasks/ensure_storage_class.yaml @@ -1,7 +1,168 @@ --- -# LVMS ensure_storage_class: create per-tenant labeled StorageClass on the hub cluster. -# Uses the topolvm provisioner backed by the lvms-vg1 VolumeGroup created by -# osac-installer's configure-lvms.sh hook. One StorageClass per tier. Idempotent. +# LVMS ensure_storage_class: create per-tenant labeled StorageClass on the target cluster. +# +# Hub path (_remote_kubeconfig undefined): targets the hub cluster. Assumes LVMS is +# already installed (by osac-installer's configure-lvms.sh hook). +# +# CaaS path (_remote_kubeconfig defined): targets a HCP guest cluster. Installs the +# LVMS operator via OLM, creates an LVMCluster that auto-discovers the second data +# disk (added to the agent VM by setup-caas-agents.sh via AGENT_VM_DATA_DISK_SIZE), +# then creates per-tenant StorageClasses on the guest cluster. +# +# Both paths are idempotent. + +- name: Install LVMS on guest cluster + when: _remote_kubeconfig is defined + block: + - name: Create openshift-storage namespace on guest cluster + kubernetes.core.k8s: + kubeconfig: "{{ _remote_kubeconfig | default(omit) }}" + validate_certs: "{{ not (_remote_kubeconfig_insecure | default(false)) }}" + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ lvms_storage_operator_namespace }}" + + - name: Create LVMS OperatorGroup on guest cluster + kubernetes.core.k8s: + kubeconfig: "{{ _remote_kubeconfig | default(omit) }}" + validate_certs: "{{ not (_remote_kubeconfig_insecure | default(false)) }}" + state: present + definition: + apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + name: lvms-operator + namespace: "{{ lvms_storage_operator_namespace }}" + spec: + targetNamespaces: + - "{{ lvms_storage_operator_namespace }}" + + - name: Resolve effective LVMS operator channel + when: lvms_storage_operator_channel | length == 0 + block: + - name: Query LVMS packagemanifest default channel + kubernetes.core.k8s_info: + kubeconfig: "{{ _remote_kubeconfig | default(omit) }}" + validate_certs: "{{ not (_remote_kubeconfig_insecure | default(false)) }}" + api_version: packages.operators.coreos.com/v1 + kind: PackageManifest + name: lvms-operator + namespace: "{{ lvms_storage_operator_catalog_namespace }}" + register: _lvms_pm + + - name: Set channel from packagemanifest default + ansible.builtin.set_fact: + _lvms_effective_channel: "{{ _lvms_pm.resources[0].status.defaultChannel }}" + when: _lvms_pm.resources | length > 0 + + - name: Fail if packagemanifest not found and no channel override set + ansible.builtin.fail: + msg: >- + lvms-operator PackageManifest not found in openshift-marketplace and + lvms_storage_operator_channel is not set. Set lvms_storage_operator_channel + explicitly (e.g. stable-4.18) to proceed. + when: _lvms_pm.resources | length == 0 + + - name: Use explicit channel override + ansible.builtin.set_fact: + _lvms_effective_channel: "{{ lvms_storage_operator_channel }}" + when: lvms_storage_operator_channel | length > 0 + + - name: Create LVMS Subscription on guest cluster + kubernetes.core.k8s: + kubeconfig: "{{ _remote_kubeconfig | default(omit) }}" + validate_certs: "{{ not (_remote_kubeconfig_insecure | default(false)) }}" + state: present + definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + name: lvms-operator + namespace: "{{ lvms_storage_operator_namespace }}" + spec: + channel: "{{ _lvms_effective_channel }}" + installPlanApproval: Automatic + name: lvms-operator + source: "{{ lvms_storage_operator_catalog_source }}" + sourceNamespace: "{{ lvms_storage_operator_catalog_namespace }}" + + - name: Wait for LVMS CSV to reach Succeeded phase + kubernetes.core.k8s_info: + kubeconfig: "{{ _remote_kubeconfig | default(omit) }}" + validate_certs: "{{ not (_remote_kubeconfig_insecure | default(false)) }}" + api_version: operators.coreos.com/v1alpha1 + kind: ClusterServiceVersion + namespace: "{{ lvms_storage_operator_namespace }}" + label_selectors: + - "operators.coreos.com/lvms-operator.{{ lvms_storage_operator_namespace }}" + register: _lvms_csv + until: + - _lvms_csv.resources | length > 0 + - _lvms_csv.resources[0].status.phase | default('') in ['Succeeded', 'Failed', 'CopierFailed'] + failed_when: + - _lvms_csv.resources | length > 0 + - _lvms_csv.resources[0].status.phase | default('') != 'Succeeded' + retries: 30 + delay: 20 + + - name: Create LVMCluster on guest cluster + vars: + _lvms_base_device_class: + name: "{{ lvms_storage_device_class }}" + thinPoolConfig: + name: thin-pool-1 + sizePercent: 90 + overprovisionRatio: 10 + _lvms_device_selector: >- + {{ {'deviceSelector': {'paths': [lvms_storage_data_device]}} + if lvms_storage_data_device else {} }} + kubernetes.core.k8s: + kubeconfig: "{{ _remote_kubeconfig | default(omit) }}" + validate_certs: "{{ not (_remote_kubeconfig_insecure | default(false)) }}" + state: present + definition: + apiVersion: lvm.topolvm.io/v1alpha1 + kind: LVMCluster + metadata: + name: lvms-cluster + namespace: "{{ lvms_storage_operator_namespace }}" + spec: + storage: + deviceClasses: + - "{{ _lvms_base_device_class | combine(_lvms_device_selector) }}" + + - name: Wait for LVMCluster to be Ready on guest cluster + kubernetes.core.k8s_info: + kubeconfig: "{{ _remote_kubeconfig | default(omit) }}" + validate_certs: "{{ not (_remote_kubeconfig_insecure | default(false)) }}" + api_version: lvm.topolvm.io/v1alpha1 + kind: LVMCluster + name: lvms-cluster + namespace: "{{ lvms_storage_operator_namespace }}" + register: _lvms_cluster_status + until: + - _lvms_cluster_status.resources | length > 0 + - _lvms_cluster_status.resources[0].status.state | default('') in ['Ready', 'Failed'] + failed_when: + - _lvms_cluster_status.resources | length > 0 + - _lvms_cluster_status.resources[0].status.state | default('') == 'Failed' + retries: 30 + delay: 20 + + - name: Wait for LVMS StorageClass to appear on guest cluster + kubernetes.core.k8s_info: + kubeconfig: "{{ _remote_kubeconfig | default(omit) }}" + validate_certs: "{{ not (_remote_kubeconfig_insecure | default(false)) }}" + api_version: storage.k8s.io/v1 + kind: StorageClass + name: "lvms-{{ lvms_storage_device_class }}" + register: _lvms_guest_sc + until: _lvms_guest_sc.resources | length > 0 + retries: 30 + delay: 20 - name: Assert _provider_tiers is defined and non-empty ansible.builtin.assert: @@ -33,6 +194,8 @@ - name: Check existing tenant StorageClasses kubernetes.core.k8s_info: + kubeconfig: "{{ _remote_kubeconfig | default(omit) }}" + validate_certs: "{{ not (_remote_kubeconfig_insecure | default(false)) }}" api_version: storage.k8s.io/v1 kind: StorageClass label_selectors: @@ -49,6 +212,8 @@ - name: Create missing per-tenant StorageClasses kubernetes.core.k8s: + kubeconfig: "{{ _remote_kubeconfig | default(omit) }}" + validate_certs: "{{ not (_remote_kubeconfig_insecure | default(false)) }}" state: present definition: apiVersion: storage.k8s.io/v1 diff --git a/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/tasks/teardown_cluster_storage.yaml b/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/tasks/teardown_cluster_storage.yaml index fdd34579f4..e7c5dee2b0 100644 --- a/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/tasks/teardown_cluster_storage.yaml +++ b/osac-aap/collections/ansible_collections/osac/templates/roles/lvms_storage/tasks/teardown_cluster_storage.yaml @@ -1,5 +1,5 @@ --- -# LVMS teardown_cluster_storage: remove per-tenant StorageClasses from the hub cluster. +# LVMS teardown_cluster_storage: remove per-tenant StorageClasses from the target cluster. # Finds SCs by label selector. The target cluster may be unreachable (being destroyed) # so failures are tolerated and reported rather than fatal. diff --git a/osac-installer/scripts/setup-caas-agents.sh b/osac-installer/scripts/setup-caas-agents.sh index 3c68332d0d..0d738f1c79 100755 --- a/osac-installer/scripts/setup-caas-agents.sh +++ b/osac-installer/scripts/setup-caas-agents.sh @@ -23,6 +23,7 @@ AGENT_VM_VCPUS=${AGENT_VM_VCPUS:-"4"} [[ "${AGENT_VM_MEMORY}" =~ ^[1-9][0-9]*$ ]] || { echo "ERROR: AGENT_VM_MEMORY must be a positive integer: ${AGENT_VM_MEMORY}" >&2; exit 1; } [[ "${AGENT_VM_VCPUS}" =~ ^[1-9][0-9]*$ ]] || { echo "ERROR: AGENT_VM_VCPUS must be a positive integer: ${AGENT_VM_VCPUS}" >&2; exit 1; } AGENT_VM_DISK_SIZE=${AGENT_VM_DISK_SIZE:-"120G"} +AGENT_VM_DATA_DISK_SIZE=${AGENT_VM_DATA_DISK_SIZE:-""} AGENT_VM_STORAGE_DIR=${AGENT_VM_STORAGE_DIR:-"/data/osac-storage"} LIBVIRT_NETWORK=${LIBVIRT_NETWORK:?"LIBVIRT_NETWORK must be set"} SSH_CONFIG=${SSH_CONFIG:-""} @@ -39,6 +40,11 @@ validate_safe() { validate_safe "AGENT_VM_STORAGE_DIR" "${AGENT_VM_STORAGE_DIR}" validate_safe "LIBVIRT_NETWORK" "${LIBVIRT_NETWORK}" validate_safe "AGENT_VM_DISK_SIZE" "${AGENT_VM_DISK_SIZE}" +[[ -n "${AGENT_VM_DATA_DISK_SIZE}" ]] && { + [[ "${AGENT_VM_DATA_DISK_SIZE}" =~ ^[1-9][0-9]*[kMGT]?$ ]] || { + echo "ERROR: AGENT_VM_DATA_DISK_SIZE must be a positive integer with optional size suffix (k/M/G/T): ${AGENT_VM_DATA_DISK_SIZE}" >&2; exit 1 + } +} echo "=== Setting up CaaS agent infrastructure ===" echo "Agent namespace: ${AGENT_NAMESPACE}" @@ -184,20 +190,34 @@ curl -k -L --fail-with-body -o '${ISO_FILE}' '${ISO_URL}' virsh --connect qemu:///system destroy '${AGENT_VM_NAME}' 2>/dev/null || true virsh --connect qemu:///system undefine '${AGENT_VM_NAME}' 2>/dev/null || true rm -f '${AGENT_VM_STORAGE_DIR}/${AGENT_VM_NAME}.qcow2' +if [[ -n '${AGENT_VM_DATA_DISK_SIZE}' ]]; then + rm -f '${AGENT_VM_STORAGE_DIR}/${AGENT_VM_NAME}-data.qcow2' +fi qemu-img create -f qcow2 '${AGENT_VM_STORAGE_DIR}/${AGENT_VM_NAME}.qcow2' '${AGENT_VM_DISK_SIZE}' +if [[ -n '${AGENT_VM_DATA_DISK_SIZE}' ]]; then + qemu-img create -f qcow2 '${AGENT_VM_STORAGE_DIR}/${AGENT_VM_NAME}-data.qcow2' '${AGENT_VM_DATA_DISK_SIZE}' +fi -virt-install --connect qemu:///system \ - --name '${AGENT_VM_NAME}' \ - --memory '${AGENT_VM_MEMORY}' \ - --vcpus '${AGENT_VM_VCPUS}' \ - --disk '${AGENT_VM_STORAGE_DIR}/${AGENT_VM_NAME}.qcow2' \ - --disk '${ISO_FILE},device=cdrom,readonly=on' \ - --network network='${LIBVIRT_NETWORK}' \ - --os-variant rhel9.0 \ - --boot hd,cdrom \ - --events on_poweroff=restart \ +_virt_install_args=( + --connect qemu:///system + --name '${AGENT_VM_NAME}' + --memory '${AGENT_VM_MEMORY}' + --vcpus '${AGENT_VM_VCPUS}' + --disk '${AGENT_VM_STORAGE_DIR}/${AGENT_VM_NAME}.qcow2' +) +if [[ -n '${AGENT_VM_DATA_DISK_SIZE}' ]]; then + _virt_install_args+=(--disk '${AGENT_VM_STORAGE_DIR}/${AGENT_VM_NAME}-data.qcow2') +fi +_virt_install_args+=( + --disk '${ISO_FILE},device=cdrom,readonly=on' + --network network='${LIBVIRT_NETWORK}' + --os-variant rhel9.0 + --boot hd,cdrom + --events on_poweroff=restart --noautoconsole +) +virt-install "\${_virt_install_args[@]}" echo "Agent VM created and booting" HVEOF diff --git a/osac-operator/charts/operator/templates/clusterrole.yaml b/osac-operator/charts/operator/templates/clusterrole.yaml index a1f3c98993..b960a3b541 100644 --- a/osac-operator/charts/operator/templates/clusterrole.yaml +++ b/osac-operator/charts/operator/templates/clusterrole.yaml @@ -18,6 +18,14 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch - apiGroups: - "" resources: @@ -40,6 +48,14 @@ rules: - get - list - watch +# hostedcontrolplanes: get only — storage controller reads the kubeconfig secret name, +# no watch/list needed (looked up by name from the ClusterOrder). +- apiGroups: + - hypershift.openshift.io + resources: + - hostedcontrolplanes + verbs: + - get - apiGroups: - k8s.ovn.org resources: