From eebc3bf43e6a9cda35ca14b21550a7ca2ee85b53 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Thu, 18 Jun 2026 16:28:26 +0200 Subject: [PATCH 01/25] Draft initial implementation --- class/defaults.yml | 50 +++- component/main.jsonnet | 267 +++++++++++++++++- .../capi_cluster.yaml | 86 ++++++ .../worker_group_worker.yaml | 54 ++++ 4 files changed, 453 insertions(+), 4 deletions(-) create mode 100644 tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml create mode 100644 tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml diff --git a/class/defaults.yml b/class/defaults.yml index 844aa49..a10808d 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -1,5 +1,53 @@ parameters: talos_capi_cluster_cloudscale: =_metadata: + multi_instance: true multi_tenant: true - namespace: syn-talos-capi-cluster-cloudscale + namespace: syn-cluster-api + + clusterName: ${cluster:name} + + # TODO(sg): how will updates work with CAPI? + talosVersion: '1.13.4' + # NOTE(sg): this is the well-known default schematic UUID + talosSchematicUUID: 376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba + kubernetesVersion: '1.36.1' + + cloudscale: + # TODO(sg): document this + customImageSlug: talos-v${talos_capi_cluster_cloudscale:talosVersion}-37656798 + # TODO(sg): decide on default sizing + privateNetwork: + name: privnet_${talos_capi_cluster_cloudscale:clusterName} + uuid: TO_BE_REPLACED + region: ${facts:region} + + controlPlane: + count: 1 + flavor: plus-16-4 + rootVolumeSize: 50 + + workerGroups: + worker: + count: 1 + flavor: plus-16-4 + rootVolumeSize: 50 + + cluster: + spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.128.0.0/14 + services: + cidrBlocks: + - 172.30.0.0/16 + serviceDomain: cluster.local + + cloudscaleCluster: + spec: + region: ${talos_capi_cluster_cloudscale:cloudscale:region} + credentialsRef: + name: cloudscale-credentials + + talosControlPlane: {} diff --git a/component/main.jsonnet b/component/main.jsonnet index 00e30bb..65547cc 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -1,10 +1,271 @@ -// main template for talos-capi-cluster-cloudscale +local com = import 'lib/commodore.libjsonnet'; local kap = import 'lib/kapitan.libjsonnet'; local kube = import 'lib/kube.libjsonnet'; + local inv = kap.inventory(); -// The hiera parameters for the component local params = inv.parameters.talos_capi_cluster_cloudscale; -// Define outputs below +local cloudscaleImageSlug = 'custom:%s' % params.cloudscale.customImageSlug; + +local ccmResourceSetLabelKey = 'talos-capi-cluster-cloudscale.syn.tools/cloudscale-ccm'; + +local capiCluster = params.cluster { + apiVersion: 'cluster.x-k8s.io/v1beta2', + kind: 'Cluster', + metadata+: { + name: params.clusterName, + namespace: params.namespace, + labels+: { + [ccmResourceSetLabelKey]: 'cloudscale', + }, + }, + spec+: { + infrastructureRef: { + apiGroup: 'infrastructure.cluster.x-k8s.io', + kind: 'CloudscaleCluster', + name: params.clusterName, + }, + controlPlaneRef: { + apiGroup: 'controlplane.cluster.x-k8s.io', + kind: 'TalosControlPlane', + name: params.clusterName, + }, + }, +}; + +local capiCloudscaleCluster = params.cloudscaleCluster { + apiVersion: 'infrastructure.cluster.x-k8s.io/v1beta2', + kind: 'CloudscaleCluster', + metadata+: { + name: params.clusterName, + namespace: params.namespace, + }, + spec+: { + networks: [ + { + name: params.cloudscale.privateNetwork.name, + uuid: params.cloudscale.privateNetwork.uuid, + }, + ], + }, +}; + +local capiCloudscaleMachineTemplateControlPlane = { + apiVersion: 'infrastructure.cluster.x-k8s.io/v1beta2', + kind: 'CloudscaleMachineTemplate', + metadata+: { + name: '%s-control-plane' % params.clusterName, + namespace: params.namespace, + }, + spec: { + template: { + spec: { + flavor: params.controlPlane.flavor, + image: cloudscaleImageSlug, + rootVolumeSize: params.controlPlane.rootVolumeSize, + serverGroup: { + name: $.metadata.name, + }, + interfaces: [ + { + network: params.cloudscale.privateNetwork.name, + }, + ], + }, + }, + }, +}; + +local talosStrategicPatch = { + machine: { + network: { + interfaces: [ + { + deviceSelector: { + physical: true, + }, + dhcp: true, + }, + ], + }, + install: { + disk: '/dev/sda', + }, + }, + cluster: { + // TODO(sg): document how to inject CCM manifests during bootstrap + externalCloudProvider: { + enabled: true, + }, + }, +}; + +local capiTalosControlPlane = params.talosControlPlane { + apiVersion: 'controlplane.cluster.x-k8s.io/v1alpha3', + kind: 'TalosControlPlane', + metadata+: { + name: params.clusterName, + namespace: params.namespace, + }, + spec+: { + infrastructureTemplate: { + apiVersion: 'infrastructure.cluster.x-k8s.io/v1beta2', + kind: 'CloudscaleMachineTemplate', + name: capiCloudscaleMachineTemplateControlPlane.metadata.name, + }, + controlPlaneConfig: { + controlplane: { + generateType: 'controlplane', + talosVersion: params.talosVersion, + hostname: { + // we want to use the VM name defined by the cloudscale CAPI + // provider. + source: 'InfrastructureName', + }, + strategicPatches: [ + std.manifestJsonMinified(talosStrategicPatch { + install+: { + // TODO(sg): do installers for custom schematic ids even exist? + image: 'factory.talos.dev/openstack-installer/%(schematic_uuid)s:v%(version)s' % { + schematic_uuid: params.talosSchematicUUID, + version: params.talosVersion, + }, + }, + }), + ], + }, + }, + }, +}; + +// NOTE(sg): figure out if this is even needed after initial bootstrap +local capiClusterResourceSetCloudscaleCCM = { + apiVersion: 'addons.cluster.x-k8s.io/v1beta2', + kind: 'ClusterResourceSet', + metadata: { + name: 'cloudscale-ccm-%s' % params.clusterName, + namespace: params.namespace, + }, + spec: { + strategy: 'ApplyOnce', + clusterSelector: { + matchLabels: { + [ccmResourceSetLabelKey]: 'cloudscale', + }, + }, + resources: [ + // NOTE(sg): the configmap is externally generated for bootstrap + { + name: '%s-ccm' % params.clusterName, + kind: 'ConfigMap', + }, + ], + }, +}; + +local capiWorkerGroup(name) = + local machineDeployment = { + apiVersion: 'cluster.x-k8s.io/v1beta2', + kind: 'MachineDeployment', + metadata: { + name: name, + namespace: params.namespace, + }, + spec: { + clusterName: params.clusterName, + replicas: params.workerGroups[name].count, + selector: { + matchLabels: null, + }, + template: { + spec: { + clusterName: params.clusterName, + version: params.talosVersion, + bootstrap: { + configRef: { + name: name, + apiGroup: 'bootstrap.cluster.x-k8s.io', + kind: 'TalosConfigTemplate', + }, + }, + infrastructureRef: { + name: name, + apiGroup: 'infrastructure.cluster.x-k8s.io', + kind: 'CloudscaleMachineTemplate', + }, + }, + }, + }, + }; + local cloudscaleMachineTemplate = { + apiVersion: 'infrastructure.cluster.x-k8s.io/v1beta2', + kind: 'CloudscaleMachineTemplate', + metadata: { + name: name, + namespace: params.namespace, + }, + spec: { + template: { + spec: { + flavor: params.workerGroups[name].flavor, + image: cloudscaleImageSlug, + rootVolumeSize: params.workerGroups[name].rootVolumeSize, + serverGroup: { + name: name, + }, + interfaces: [ + { + network: params.cloudscale.privateNetwork.name, + }, + ], + }, + }, + }, + }; + local talosConfigTemplate = { + apiVersion: 'bootstrap.cluster.x-k8s.io/v1alpha3', + kind: 'TalosConfigTemplate', + metadata: { + name: name, + namespace: params.namespace, + }, + spec: { + template: { + spec: { + generateType: 'join', + talosVersion: params.talosVersion, + hostname: { + source: 'InfrastructureName', + }, + strategicPatches: [ + std.manifestJsonMinified(talosStrategicPatch), + ], + }, + }, + }, + }; + + // NOTE(sg): we're slightly abusing com.generateResources() below. The + // function doesn't really support rendering multiple objects instead of + // rendering a single object and merging the parameter dict values into it. + { + name: name, + resources: [ + machineDeployment, + cloudscaleMachineTemplate, + talosConfigTemplate, + ], + }; + { + capi_cluster: [ + capiCluster, + capiCloudscaleCluster, + capiCloudscaleMachineTemplateControlPlane, + capiTalosControlPlane, + capiClusterResourceSetCloudscaleCCM, + ], +} + { + ['worker_group_%s' % wg.name]: wg.resources + for wg in com.generateResources(params.workerGroups, capiWorkerGroup) } diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml new file mode 100644 index 0000000..a6494b7 --- /dev/null +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -0,0 +1,86 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + labels: + talos-capi-cluster-cloudscale.syn.tools/cloudscale-ccm: cloudscale + name: c-green-test-1234 + namespace: syn-cluster-api +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.128.0.0/14 + serviceDomain: cluster.local + services: + cidrBlocks: + - 172.30.0.0/16 + controlPlaneRef: + apiGroup: controlplane.cluster.x-k8s.io + kind: TalosControlPlane + name: c-green-test-1234 + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleCluster + name: c-green-test-1234 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleCluster +metadata: + name: c-green-test-1234 + namespace: syn-cluster-api +spec: + credentialsRef: + name: cloudscale-credentials + networks: + - name: privnet_c-green-test-1234 + uuid: TO_BE_REPLACED + region: rma1 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: c-green-test-1234-control-plane + namespace: syn-cluster-api +spec: + template: + spec: + flavor: plus-16-4 + image: custom:talos-v1.13.4-37656798 + interfaces: + - network: privnet_c-green-test-1234 + rootVolumeSize: 50 + serverGroup: + name: c-green-test-1234-control-plane +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: TalosControlPlane +metadata: + name: c-green-test-1234 + namespace: syn-cluster-api +spec: + controlPlaneConfig: + controlplane: + generateType: controlplane + hostname: + source: InfrastructureName + strategicPatches: + - '{"cluster":{"externalCloudProvider":{"enabled":true}},"install":{"image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.4"},"machine":{"install":{"disk":"/dev/sda"},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' + talosVersion: 1.13.4 + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: CloudscaleMachineTemplate + name: c-green-test-1234-control-plane +--- +apiVersion: addons.cluster.x-k8s.io/v1beta2 +kind: ClusterResourceSet +metadata: + name: cloudscale-ccm-c-green-test-1234 + namespace: syn-cluster-api +spec: + clusterSelector: + matchLabels: + talos-capi-cluster-cloudscale.syn.tools/cloudscale-ccm: cloudscale + resources: + - kind: ConfigMap + name: c-green-test-1234-ccm + strategy: ApplyOnce diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml new file mode 100644 index 0000000..076ef7a --- /dev/null +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml @@ -0,0 +1,54 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: MachineDeployment +metadata: + name: worker + namespace: syn-cluster-api +spec: + clusterName: c-green-test-1234 + replicas: 1 + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiGroup: bootstrap.cluster.x-k8s.io + kind: TalosConfigTemplate + name: worker + clusterName: c-green-test-1234 + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleMachineTemplate + name: worker + version: 1.13.4 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: worker + namespace: syn-cluster-api +spec: + template: + spec: + flavor: plus-16-4 + image: custom:talos-v1.13.4-37656798 + interfaces: + - network: privnet_c-green-test-1234 + rootVolumeSize: 50 + serverGroup: + name: worker +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: TalosConfigTemplate +metadata: + name: worker + namespace: syn-cluster-api +spec: + template: + spec: + generateType: join + hostname: + source: InfrastructureName + strategicPatches: + - '{"cluster":{"externalCloudProvider":{"enabled":true}},"machine":{"install":{"disk":"/dev/sda"},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' + talosVersion: 1.13.4 From c07089052f06285cfef54d23296f9b381176e091 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Thu, 18 Jun 2026 17:19:29 +0200 Subject: [PATCH 02/25] Make privnet name a valid K8s name --- class/defaults.yml | 3 ++- .../talos-capi-cluster-cloudscale/capi_cluster.yaml | 4 ++-- .../talos-capi-cluster-cloudscale/worker_group_worker.yaml | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/class/defaults.yml b/class/defaults.yml index a10808d..86a8c8c 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -18,7 +18,8 @@ parameters: customImageSlug: talos-v${talos_capi_cluster_cloudscale:talosVersion}-37656798 # TODO(sg): decide on default sizing privateNetwork: - name: privnet_${talos_capi_cluster_cloudscale:clusterName} + # TODO(sg): does this need to match the privnet name on cloudscale? + name: privnet-${talos_capi_cluster_cloudscale:clusterName} uuid: TO_BE_REPLACED region: ${facts:region} diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml index a6494b7..1b9d0d9 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -32,7 +32,7 @@ spec: credentialsRef: name: cloudscale-credentials networks: - - name: privnet_c-green-test-1234 + - name: privnet-c-green-test-1234 uuid: TO_BE_REPLACED region: rma1 --- @@ -47,7 +47,7 @@ spec: flavor: plus-16-4 image: custom:talos-v1.13.4-37656798 interfaces: - - network: privnet_c-green-test-1234 + - network: privnet-c-green-test-1234 rootVolumeSize: 50 serverGroup: name: c-green-test-1234-control-plane diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml index 076ef7a..80a4515 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml @@ -33,7 +33,7 @@ spec: flavor: plus-16-4 image: custom:talos-v1.13.4-37656798 interfaces: - - network: privnet_c-green-test-1234 + - network: privnet-c-green-test-1234 rootVolumeSize: 50 serverGroup: name: worker From caad8f949f44433c24f69af0e2625f0be02eb868 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Thu, 18 Jun 2026 17:25:19 +0200 Subject: [PATCH 03/25] Set `spec.version=params.kubernetesVersion` in TalosControlPlane custom resource --- component/main.jsonnet | 1 + .../talos-capi-cluster-cloudscale/capi_cluster.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/component/main.jsonnet b/component/main.jsonnet index 65547cc..b94f968 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -108,6 +108,7 @@ local capiTalosControlPlane = params.talosControlPlane { namespace: params.namespace, }, spec+: { + version: params.kubernetesVersion, infrastructureTemplate: { apiVersion: 'infrastructure.cluster.x-k8s.io/v1beta2', kind: 'CloudscaleMachineTemplate', diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml index 1b9d0d9..aa6f9f0 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -70,6 +70,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate name: c-green-test-1234-control-plane + version: 1.36.1 --- apiVersion: addons.cluster.x-k8s.io/v1beta2 kind: ClusterResourceSet From 14c610412d92d3813b16517ccefdc2a6430e5aed Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Thu, 18 Jun 2026 17:32:05 +0200 Subject: [PATCH 04/25] add comment re cloudscale creds secret name --- class/defaults.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/class/defaults.yml b/class/defaults.yml index 86a8c8c..5f2c45c 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -49,6 +49,7 @@ parameters: spec: region: ${talos_capi_cluster_cloudscale:cloudscale:region} credentialsRef: + # TODO(sg): figure out the real default name here. name: cloudscale-credentials talosControlPlane: {} From d793f408fdf5ce9bd975c7ceb9e52785710bbc58 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Thu, 18 Jun 2026 17:45:42 +0200 Subject: [PATCH 05/25] Fix controlplane patch merge --- component/main.jsonnet | 12 +++++++----- .../talos-capi-cluster-cloudscale/capi_cluster.yaml | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/component/main.jsonnet b/component/main.jsonnet index b94f968..149d8f5 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -125,11 +125,13 @@ local capiTalosControlPlane = params.talosControlPlane { }, strategicPatches: [ std.manifestJsonMinified(talosStrategicPatch { - install+: { - // TODO(sg): do installers for custom schematic ids even exist? - image: 'factory.talos.dev/openstack-installer/%(schematic_uuid)s:v%(version)s' % { - schematic_uuid: params.talosSchematicUUID, - version: params.talosVersion, + machine+: { + install+: { + // TODO(sg): do installers for custom schematic ids even exist? + image: 'factory.talos.dev/openstack-installer/%(schematic_uuid)s:v%(version)s' % { + schematic_uuid: params.talosSchematicUUID, + version: params.talosVersion, + }, }, }, }), diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml index aa6f9f0..bc8dfeb 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -64,7 +64,7 @@ spec: hostname: source: InfrastructureName strategicPatches: - - '{"cluster":{"externalCloudProvider":{"enabled":true}},"install":{"image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.4"},"machine":{"install":{"disk":"/dev/sda"},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' + - '{"cluster":{"externalCloudProvider":{"enabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.4"},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' talosVersion: 1.13.4 infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 From 5a15dbfabcfbbd9bacf3b1ffa6400faeab5f5665 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Thu, 18 Jun 2026 17:48:54 +0200 Subject: [PATCH 06/25] Set MachineDeployment `spec.version=kubernetesVersion` --- component/main.jsonnet | 2 +- .../talos-capi-cluster-cloudscale/worker_group_worker.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/component/main.jsonnet b/component/main.jsonnet index 149d8f5..0797401 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -183,7 +183,7 @@ local capiWorkerGroup(name) = template: { spec: { clusterName: params.clusterName, - version: params.talosVersion, + version: params.kubernetesVersion, bootstrap: { configRef: { name: name, diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml index 80a4515..b67fd6c 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml @@ -20,7 +20,7 @@ spec: apiGroup: infrastructure.cluster.x-k8s.io kind: CloudscaleMachineTemplate name: worker - version: 1.13.4 + version: 1.36.1 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate From 8d69bc6fa5b29aac621f6513c26cd2b094d31b4a Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 23 Jun 2026 19:54:00 +0200 Subject: [PATCH 07/25] Deploy cilium via ClusterResourceSet --- component/main.jsonnet | 18 +++++++++++------- .../capi_cluster.yaml | 8 +++++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/component/main.jsonnet b/component/main.jsonnet index 0797401..b16b47c 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -7,7 +7,7 @@ local params = inv.parameters.talos_capi_cluster_cloudscale; local cloudscaleImageSlug = 'custom:%s' % params.cloudscale.customImageSlug; -local ccmResourceSetLabelKey = 'talos-capi-cluster-cloudscale.syn.tools/cloudscale-ccm'; +local resourceSetLabelKey = 'talos-capi-cluster-cloudscale.syn.tools/bootstrap'; local capiCluster = params.cluster { apiVersion: 'cluster.x-k8s.io/v1beta2', @@ -16,7 +16,7 @@ local capiCluster = params.cluster { name: params.clusterName, namespace: params.namespace, labels+: { - [ccmResourceSetLabelKey]: 'cloudscale', + [resourceSetLabelKey]: 'cloudscale', }, }, spec+: { @@ -142,26 +142,30 @@ local capiTalosControlPlane = params.talosControlPlane { }; // NOTE(sg): figure out if this is even needed after initial bootstrap -local capiClusterResourceSetCloudscaleCCM = { +local capiClusterResourceSetBootstrap = { apiVersion: 'addons.cluster.x-k8s.io/v1beta2', kind: 'ClusterResourceSet', metadata: { - name: 'cloudscale-ccm-%s' % params.clusterName, + name: 'cloudscale-bootstrap-%s' % params.clusterName, namespace: params.namespace, }, spec: { strategy: 'ApplyOnce', clusterSelector: { matchLabels: { - [ccmResourceSetLabelKey]: 'cloudscale', + [resourceSetLabelKey]: 'cloudscale', }, }, resources: [ - // NOTE(sg): the configmap is externally generated for bootstrap + // NOTE(sg): the configmaps are externally generated for bootstrap { name: '%s-ccm' % params.clusterName, kind: 'ConfigMap', }, + { + name: '%s-cilium' % params.clusterName, + kind: 'ConfigMap', + }, ], }, }; @@ -266,7 +270,7 @@ local capiWorkerGroup(name) = capiCloudscaleCluster, capiCloudscaleMachineTemplateControlPlane, capiTalosControlPlane, - capiClusterResourceSetCloudscaleCCM, + capiClusterResourceSetBootstrap, ], } + { ['worker_group_%s' % wg.name]: wg.resources diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml index bc8dfeb..3607eca 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -2,7 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta2 kind: Cluster metadata: labels: - talos-capi-cluster-cloudscale.syn.tools/cloudscale-ccm: cloudscale + talos-capi-cluster-cloudscale.syn.tools/bootstrap: cloudscale name: c-green-test-1234 namespace: syn-cluster-api spec: @@ -75,13 +75,15 @@ spec: apiVersion: addons.cluster.x-k8s.io/v1beta2 kind: ClusterResourceSet metadata: - name: cloudscale-ccm-c-green-test-1234 + name: cloudscale-bootstrap-c-green-test-1234 namespace: syn-cluster-api spec: clusterSelector: matchLabels: - talos-capi-cluster-cloudscale.syn.tools/cloudscale-ccm: cloudscale + talos-capi-cluster-cloudscale.syn.tools/bootstrap: cloudscale resources: - kind: ConfigMap name: c-green-test-1234-ccm + - kind: ConfigMap + name: c-green-test-1234-cilium strategy: ApplyOnce From 430dda5e11fb8671a8062ad90a698dd161af4f98 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 23 Jun 2026 20:27:20 +0200 Subject: [PATCH 08/25] Refactor component to generate unique names for manifests with immutable fields in `spec` --- class/defaults.yml | 4 +- component/main.jsonnet | 141 ++++++++++-------- .../capi_cluster.yaml | 4 +- .../worker_group_worker.yaml | 11 +- 4 files changed, 89 insertions(+), 71 deletions(-) diff --git a/class/defaults.yml b/class/defaults.yml index 5f2c45c..3d778ad 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -52,4 +52,6 @@ parameters: # TODO(sg): figure out the real default name here. name: cloudscale-credentials - talosControlPlane: {} + talosControlPlane: + metadata: {} + spec: {} diff --git a/component/main.jsonnet b/component/main.jsonnet index b16b47c..da76d15 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -9,38 +9,21 @@ local cloudscaleImageSlug = 'custom:%s' % params.cloudscale.customImageSlug; local resourceSetLabelKey = 'talos-capi-cluster-cloudscale.syn.tools/bootstrap'; -local capiCluster = params.cluster { - apiVersion: 'cluster.x-k8s.io/v1beta2', - kind: 'Cluster', - metadata+: { - name: params.clusterName, - namespace: params.namespace, - labels+: { - [resourceSetLabelKey]: 'cloudscale', - }, - }, - spec+: { - infrastructureRef: { - apiGroup: 'infrastructure.cluster.x-k8s.io', - kind: 'CloudscaleCluster', - name: params.clusterName, - }, - controlPlaneRef: { - apiGroup: 'controlplane.cluster.x-k8s.io', - kind: 'TalosControlPlane', - name: params.clusterName, - }, - }, -}; +// TODO(sg): figure out which resources need to have `nameWithHash()` +local nameWithHash(name, spec, length=16) = + '%s-%s' % [ + name, + std.sha256(std.manifestJsonMinified(spec))[:length], + ]; -local capiCloudscaleCluster = params.cloudscaleCluster { +local capiCloudscaleCluster = { apiVersion: 'infrastructure.cluster.x-k8s.io/v1beta2', kind: 'CloudscaleCluster', - metadata+: { + metadata+: std.get(params.cloudscaleCluster, 'metadata', {}) { name: params.clusterName, namespace: params.namespace, }, - spec+: { + spec+: params.cloudscaleCluster.spec { networks: [ { name: params.cloudscale.privateNetwork.name, @@ -54,7 +37,7 @@ local capiCloudscaleMachineTemplateControlPlane = { apiVersion: 'infrastructure.cluster.x-k8s.io/v1beta2', kind: 'CloudscaleMachineTemplate', metadata+: { - name: '%s-control-plane' % params.clusterName, + name: nameWithHash('%s-control-plane' % params.clusterName, $.spec), namespace: params.namespace, }, spec: { @@ -64,7 +47,7 @@ local capiCloudscaleMachineTemplateControlPlane = { image: cloudscaleImageSlug, rootVolumeSize: params.controlPlane.rootVolumeSize, serverGroup: { - name: $.metadata.name, + name: '%s-control-plane' % params.clusterName, }, interfaces: [ { @@ -103,11 +86,11 @@ local talosStrategicPatch = { local capiTalosControlPlane = params.talosControlPlane { apiVersion: 'controlplane.cluster.x-k8s.io/v1alpha3', kind: 'TalosControlPlane', - metadata+: { + metadata+: std.get(params.talosControlPlane, 'metadata', {}) { name: params.clusterName, namespace: params.namespace, }, - spec+: { + spec+: params.talosControlPlane.spec { version: params.kubernetesVersion, infrastructureTemplate: { apiVersion: 'infrastructure.cluster.x-k8s.io/v1beta2', @@ -128,6 +111,7 @@ local capiTalosControlPlane = params.talosControlPlane { machine+: { install+: { // TODO(sg): do installers for custom schematic ids even exist? + // TODO(sg): figure out the new way to do this image: 'factory.talos.dev/openstack-installer/%(schematic_uuid)s:v%(version)s' % { schematic_uuid: params.talosSchematicUUID, version: params.talosVersion, @@ -171,35 +155,24 @@ local capiClusterResourceSetBootstrap = { }; local capiWorkerGroup(name) = - local machineDeployment = { - apiVersion: 'cluster.x-k8s.io/v1beta2', - kind: 'MachineDeployment', + local talosConfigTemplate = { + apiVersion: 'bootstrap.cluster.x-k8s.io/v1alpha3', + kind: 'TalosConfigTemplate', metadata: { - name: name, + name: nameWithHash(name, $.spec), namespace: params.namespace, }, spec: { - clusterName: params.clusterName, - replicas: params.workerGroups[name].count, - selector: { - matchLabels: null, - }, template: { spec: { - clusterName: params.clusterName, - version: params.kubernetesVersion, - bootstrap: { - configRef: { - name: name, - apiGroup: 'bootstrap.cluster.x-k8s.io', - kind: 'TalosConfigTemplate', - }, - }, - infrastructureRef: { - name: name, - apiGroup: 'infrastructure.cluster.x-k8s.io', - kind: 'CloudscaleMachineTemplate', + generateType: 'join', + talosVersion: params.talosVersion, + hostname: { + source: 'InfrastructureName', }, + strategicPatches: [ + std.manifestJsonMinified(talosStrategicPatch), + ], }, }, }, @@ -208,7 +181,7 @@ local capiWorkerGroup(name) = apiVersion: 'infrastructure.cluster.x-k8s.io/v1beta2', kind: 'CloudscaleMachineTemplate', metadata: { - name: name, + name: nameWithHash(name, $.spec), namespace: params.namespace, }, spec: { @@ -229,24 +202,40 @@ local capiWorkerGroup(name) = }, }, }; - local talosConfigTemplate = { - apiVersion: 'bootstrap.cluster.x-k8s.io/v1alpha3', - kind: 'TalosConfigTemplate', + local machineDeployment = { + apiVersion: 'cluster.x-k8s.io/v1beta2', + kind: 'MachineDeployment', metadata: { name: name, namespace: params.namespace, }, spec: { - template: { + clusterName: params.clusterName, + replicas: params.workerGroups[name].count, + selector: { + matchLabels: null, + }, + template: std.get(params.workerGroups[name], 'template', {}) { + metadata: { + labels+: { + 'node-role.kubernetes.io/worker': '', + }, + }, spec: { - generateType: 'join', - talosVersion: params.talosVersion, - hostname: { - source: 'InfrastructureName', + clusterName: params.clusterName, + version: params.kubernetesVersion, + bootstrap: { + configRef: { + name: talosConfigTemplate.metadata.name, + apiGroup: 'bootstrap.cluster.x-k8s.io', + kind: 'TalosConfigTemplate', + }, + }, + infrastructureRef: { + name: cloudscaleMachineTemplate.metadata.name, + apiGroup: 'infrastructure.cluster.x-k8s.io', + kind: 'CloudscaleMachineTemplate', }, - strategicPatches: [ - std.manifestJsonMinified(talosStrategicPatch), - ], }, }, }, @@ -264,6 +253,30 @@ local capiWorkerGroup(name) = ], }; +local capiCluster = params.cluster { + apiVersion: 'cluster.x-k8s.io/v1beta2', + kind: 'Cluster', + metadata+: { + name: params.clusterName, + namespace: params.namespace, + labels+: { + [resourceSetLabelKey]: 'cloudscale', + }, + }, + spec+: { + infrastructureRef: { + apiGroup: 'infrastructure.cluster.x-k8s.io', + kind: 'CloudscaleCluster', + name: capiCloudscaleCluster.metadata.name, + }, + controlPlaneRef: { + apiGroup: 'controlplane.cluster.x-k8s.io', + kind: 'TalosControlPlane', + name: capiTalosControlPlane.metadata.name, + }, + }, +}; + { capi_cluster: [ capiCluster, diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml index 3607eca..0d83925 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -39,7 +39,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate metadata: - name: c-green-test-1234-control-plane + name: c-green-test-1234-control-plane-e84c4ba30597bd6a namespace: syn-cluster-api spec: template: @@ -69,7 +69,7 @@ spec: infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate - name: c-green-test-1234-control-plane + name: c-green-test-1234-control-plane-e84c4ba30597bd6a version: 1.36.1 --- apiVersion: addons.cluster.x-k8s.io/v1beta2 diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml index b67fd6c..7d12a75 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml @@ -9,23 +9,26 @@ spec: selector: matchLabels: null template: + metadata: + labels: + node-role.kubernetes.io/worker: '' spec: bootstrap: configRef: apiGroup: bootstrap.cluster.x-k8s.io kind: TalosConfigTemplate - name: worker + name: worker-0dd528d045332e9c clusterName: c-green-test-1234 infrastructureRef: apiGroup: infrastructure.cluster.x-k8s.io kind: CloudscaleMachineTemplate - name: worker + name: worker-cb7ea8521a7ab803 version: 1.36.1 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate metadata: - name: worker + name: worker-cb7ea8521a7ab803 namespace: syn-cluster-api spec: template: @@ -41,7 +44,7 @@ spec: apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 kind: TalosConfigTemplate metadata: - name: worker + name: worker-0dd528d045332e9c namespace: syn-cluster-api spec: template: From 80b3cded132f4cd7c70d24661a24b7a6ea2fa183 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 23 Jun 2026 20:28:25 +0200 Subject: [PATCH 09/25] Adjust Talos machineconfig to deploy Cilium via `ClusterResourceSet` during bootstrap --- class/defaults.yml | 2 + component/main.jsonnet | 43 +++++++++++++------ tests/defaults.yml | 8 +++- .../capi_cluster.yaml | 2 +- .../worker_group_worker.yaml | 6 +-- 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/class/defaults.yml b/class/defaults.yml index 3d778ad..627d945 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -13,6 +13,8 @@ parameters: talosSchematicUUID: 376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba kubernetesVersion: '1.36.1' + cni: cilium + cloudscale: # TODO(sg): document this customImageSlug: talos-v${talos_capi_cluster_cloudscale:talosVersion}-37656798 diff --git a/component/main.jsonnet b/component/main.jsonnet index da76d15..5586085 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -80,6 +80,22 @@ local talosStrategicPatch = { externalCloudProvider: { enabled: true, }, + network: { + cni: { + // valid values: `flannel`, `custom`, `none`. + // `custom` uses custom manifests provided via `cni.urls` + // `none` indicates externally provisioned & managed CNI + // we currently assume that we'll always deploy custom CNIs via CAPI + // resourcesets. + [if params.cni != 'flannel' then 'name']: 'none', + }, + }, + proxy: { + disabled: if params.cni == 'cilium' then + inv.parameters.cilium.cilium_helm_values.kubeProxyReplacement == 'true' + else + std.trace('Not disabling kube-proxy for CNI %s' % params.cni, false), + }, }, }; @@ -277,15 +293,18 @@ local capiCluster = params.cluster { }, }; -{ - capi_cluster: [ - capiCluster, - capiCloudscaleCluster, - capiCloudscaleMachineTemplateControlPlane, - capiTalosControlPlane, - capiClusterResourceSetBootstrap, - ], -} + { - ['worker_group_%s' % wg.name]: wg.resources - for wg in com.generateResources(params.workerGroups, capiWorkerGroup) -} +if params.cni == 'cilium' && !std.member(inv.applications, 'cilium') then + error 'Component talos-capi-cluster-cloudscale expects that component-cilium is present when parameter cni=cilium' +else + { + capi_cluster: [ + capiCluster, + capiCloudscaleCluster, + capiCloudscaleMachineTemplateControlPlane, + capiTalosControlPlane, + capiClusterResourceSetBootstrap, + ], + } + { + ['worker_group_%s' % wg.name]: wg.resources + for wg in com.generateResources(params.workerGroups, capiWorkerGroup) + } diff --git a/tests/defaults.yml b/tests/defaults.yml index a4da5b7..f5d828d 100644 --- a/tests/defaults.yml +++ b/tests/defaults.yml @@ -1,3 +1,7 @@ -# Overwrite parameters here +applications: + - cilium -# parameters: {...} +parameters: + cilium: + cilium_helm_values: + kubeProxyReplacement: "true" diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml index 0d83925..9f53a63 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -64,7 +64,7 @@ spec: hostname: source: InfrastructureName strategicPatches: - - '{"cluster":{"externalCloudProvider":{"enabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.4"},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' + - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.4"},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' talosVersion: 1.13.4 infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml index 7d12a75..006b9cb 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml @@ -17,7 +17,7 @@ spec: configRef: apiGroup: bootstrap.cluster.x-k8s.io kind: TalosConfigTemplate - name: worker-0dd528d045332e9c + name: worker-95f76d53508bb41f clusterName: c-green-test-1234 infrastructureRef: apiGroup: infrastructure.cluster.x-k8s.io @@ -44,7 +44,7 @@ spec: apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 kind: TalosConfigTemplate metadata: - name: worker-0dd528d045332e9c + name: worker-95f76d53508bb41f namespace: syn-cluster-api spec: template: @@ -53,5 +53,5 @@ spec: hostname: source: InfrastructureName strategicPatches: - - '{"cluster":{"externalCloudProvider":{"enabled":true}},"machine":{"install":{"disk":"/dev/sda"},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' + - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda"},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' talosVersion: 1.13.4 From 5c2dc5e2947a528a90c4a9b9339fc8b1001d60b2 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 23 Jun 2026 20:29:22 +0200 Subject: [PATCH 10/25] Update TalosControlPlane api version to v1beta1 Also adjust machinetemplate reference for v1beta1 Needs https://github.com/siderolabs/cluster-api-control-plane-provider-talos/pull/251 or equivalent. --- component/main.jsonnet | 16 ++++++++++------ .../capi_cluster.yaml | 12 +++++++----- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/component/main.jsonnet b/component/main.jsonnet index 5586085..41f977a 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -99,8 +99,8 @@ local talosStrategicPatch = { }, }; -local capiTalosControlPlane = params.talosControlPlane { - apiVersion: 'controlplane.cluster.x-k8s.io/v1alpha3', +local capiTalosControlPlane = { + apiVersion: 'controlplane.cluster.x-k8s.io/v1beta1', kind: 'TalosControlPlane', metadata+: std.get(params.talosControlPlane, 'metadata', {}) { name: params.clusterName, @@ -108,10 +108,14 @@ local capiTalosControlPlane = params.talosControlPlane { }, spec+: params.talosControlPlane.spec { version: params.kubernetesVersion, - infrastructureTemplate: { - apiVersion: 'infrastructure.cluster.x-k8s.io/v1beta2', - kind: 'CloudscaleMachineTemplate', - name: capiCloudscaleMachineTemplateControlPlane.metadata.name, + machineTemplate: { + spec: { + infrastructureRef: { + apiGroup: 'infrastructure.cluster.x-k8s.io', + kind: 'CloudscaleMachineTemplate', + name: capiCloudscaleMachineTemplateControlPlane.metadata.name, + }, + }, }, controlPlaneConfig: { controlplane: { diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml index 9f53a63..a6feeef 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -52,7 +52,7 @@ spec: serverGroup: name: c-green-test-1234-control-plane --- -apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: TalosControlPlane metadata: name: c-green-test-1234 @@ -66,10 +66,12 @@ spec: strategicPatches: - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.4"},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' talosVersion: 1.13.4 - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: CloudscaleMachineTemplate - name: c-green-test-1234-control-plane-e84c4ba30597bd6a + machineTemplate: + spec: + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleMachineTemplate + name: c-green-test-1234-control-plane-e84c4ba30597bd6a version: 1.36.1 --- apiVersion: addons.cluster.x-k8s.io/v1beta2 From ac3dd81d0b653c2c7b3e26e409d65e6e25eeab8b Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Thu, 25 Jun 2026 11:07:46 +0200 Subject: [PATCH 11/25] Propagate `controlPlane.count` to `TalosControlPlane` --- component/main.jsonnet | 1 + .../talos-capi-cluster-cloudscale/capi_cluster.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/component/main.jsonnet b/component/main.jsonnet index 41f977a..da943fd 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -107,6 +107,7 @@ local capiTalosControlPlane = { namespace: params.namespace, }, spec+: params.talosControlPlane.spec { + replicas: params.controlPlane.count, version: params.kubernetesVersion, machineTemplate: { spec: { diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml index a6feeef..f994b18 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -72,6 +72,7 @@ spec: apiGroup: infrastructure.cluster.x-k8s.io kind: CloudscaleMachineTemplate name: c-green-test-1234-control-plane-e84c4ba30597bd6a + replicas: 1 version: 1.36.1 --- apiVersion: addons.cluster.x-k8s.io/v1beta2 From 7a8d86c48d4edad7736c9e92740e626c9fcec6a2 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Fri, 26 Jun 2026 08:59:39 +0200 Subject: [PATCH 12/25] Add parameter to set API URL as extra Talos API server cert SAN --- class/defaults.yml | 1 + component/main.jsonnet | 3 +++ 2 files changed, 4 insertions(+) diff --git a/class/defaults.yml b/class/defaults.yml index 627d945..79e6e06 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -12,6 +12,7 @@ parameters: # NOTE(sg): this is the well-known default schematic UUID talosSchematicUUID: 376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba kubernetesVersion: '1.36.1' + apiURL: "" cni: cilium diff --git a/component/main.jsonnet b/component/main.jsonnet index da943fd..e996b26 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -76,6 +76,9 @@ local talosStrategicPatch = { }, }, cluster: { + [if params.apiURL != '' then 'apiServer']: { + certSANs: [ params.apiURL ], + }, // TODO(sg): document how to inject CCM manifests during bootstrap externalCloudProvider: { enabled: true, From 45ae76c9c367f016ab51daa6e81e9bf995670af4 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Fri, 26 Jun 2026 09:00:05 +0200 Subject: [PATCH 13/25] Add talosconfig patch todo --- component/main.jsonnet | 1 + 1 file changed, 1 insertion(+) diff --git a/component/main.jsonnet b/component/main.jsonnet index e996b26..97de763 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -75,6 +75,7 @@ local talosStrategicPatch = { disk: '/dev/sda', }, }, + // TODO(sg): figure out if this section is really needed for worker groups. cluster: { [if params.apiURL != '' then 'apiServer']: { certSANs: [ params.apiURL ], From 45d2c3c27a69e63f6e861939e7f4acbe5656206c Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Thu, 2 Jul 2026 13:10:22 +0200 Subject: [PATCH 14/25] CAPI provider canonical Kubernetes version format is prefixed with `v` Update component defaults and rewrite non-prefixed versions. --- class/defaults.yml | 2 +- component/main.jsonnet | 16 ++++++++++++++-- .../capi_cluster.yaml | 2 +- .../worker_group_worker.yaml | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/class/defaults.yml b/class/defaults.yml index 79e6e06..82e378e 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -11,7 +11,7 @@ parameters: talosVersion: '1.13.4' # NOTE(sg): this is the well-known default schematic UUID talosSchematicUUID: 376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba - kubernetesVersion: '1.36.1' + kubernetesVersion: 'v1.36.1' apiURL: "" cni: cilium diff --git a/component/main.jsonnet b/component/main.jsonnet index 97de763..8232057 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -9,6 +9,18 @@ local cloudscaleImageSlug = 'custom:%s' % params.cloudscale.customImageSlug; local resourceSetLabelKey = 'talos-capi-cluster-cloudscale.syn.tools/bootstrap'; +local kubernetesVersion = + local formatter = + if std.startsWith(params.kubernetesVersion, 'v') then + '%s' + else + std.trace( + "CAPI expects kubernetesVersion to be prefixed with 'v', adjusting %s" + % params.kubernetesVersion, + 'v%s' + ); + formatter % params.kubernetesVersion; + // TODO(sg): figure out which resources need to have `nameWithHash()` local nameWithHash(name, spec, length=16) = '%s-%s' % [ @@ -112,7 +124,7 @@ local capiTalosControlPlane = { }, spec+: params.talosControlPlane.spec { replicas: params.controlPlane.count, - version: params.kubernetesVersion, + version: kubernetesVersion, machineTemplate: { spec: { infrastructureRef: { @@ -248,7 +260,7 @@ local capiWorkerGroup(name) = }, spec: { clusterName: params.clusterName, - version: params.kubernetesVersion, + version: kubernetesVersion, bootstrap: { configRef: { name: talosConfigTemplate.metadata.name, diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml index f994b18..51366f6 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -73,7 +73,7 @@ spec: kind: CloudscaleMachineTemplate name: c-green-test-1234-control-plane-e84c4ba30597bd6a replicas: 1 - version: 1.36.1 + version: v1.36.1 --- apiVersion: addons.cluster.x-k8s.io/v1beta2 kind: ClusterResourceSet diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml index 006b9cb..12db2f0 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml @@ -23,7 +23,7 @@ spec: apiGroup: infrastructure.cluster.x-k8s.io kind: CloudscaleMachineTemplate name: worker-cb7ea8521a7ab803 - version: 1.36.1 + version: v1.36.1 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate From fb1596b3b6bdec1c749678475686c55279dfd560 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Thu, 2 Jul 2026 16:56:38 +0200 Subject: [PATCH 15/25] Set Talos `MachineConfig` `install.wipe=true` by default TODO: figture out if/how we want to allow users to adjust the patches. --- component/main.jsonnet | 1 + .../talos-capi-cluster-cloudscale/capi_cluster.yaml | 2 +- .../talos-capi-cluster-cloudscale/worker_group_worker.yaml | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/component/main.jsonnet b/component/main.jsonnet index 8232057..fb1d3e8 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -85,6 +85,7 @@ local talosStrategicPatch = { }, install: { disk: '/dev/sda', + wipe: true, }, }, // TODO(sg): figure out if this section is really needed for worker groups. diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml index 51366f6..a94bcd6 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -64,7 +64,7 @@ spec: hostname: source: InfrastructureName strategicPatches: - - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.4"},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' + - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.4","wipe":true},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' talosVersion: 1.13.4 machineTemplate: spec: diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml index 12db2f0..982a448 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml @@ -17,7 +17,7 @@ spec: configRef: apiGroup: bootstrap.cluster.x-k8s.io kind: TalosConfigTemplate - name: worker-95f76d53508bb41f + name: worker-d7863e308f443d77 clusterName: c-green-test-1234 infrastructureRef: apiGroup: infrastructure.cluster.x-k8s.io @@ -44,7 +44,7 @@ spec: apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 kind: TalosConfigTemplate metadata: - name: worker-95f76d53508bb41f + name: worker-d7863e308f443d77 namespace: syn-cluster-api spec: template: @@ -53,5 +53,5 @@ spec: hostname: source: InfrastructureName strategicPatches: - - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda"},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' + - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","wipe":true},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' talosVersion: 1.13.4 From 6b39fe85d009ff61b0ce394b4e9a1fae10d90c96 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Fri, 3 Jul 2026 08:31:43 +0200 Subject: [PATCH 16/25] Give users more control over TalosControlPlane config --- class/defaults.yml | 1 + component/main.jsonnet | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/class/defaults.yml b/class/defaults.yml index 82e378e..d8736c1 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -58,3 +58,4 @@ parameters: talosControlPlane: metadata: {} spec: {} + strategicPatches: {} diff --git a/component/main.jsonnet b/component/main.jsonnet index fb1d3e8..dcf2ad0 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -119,7 +119,7 @@ local talosStrategicPatch = { local capiTalosControlPlane = { apiVersion: 'controlplane.cluster.x-k8s.io/v1beta1', kind: 'TalosControlPlane', - metadata+: std.get(params.talosControlPlane, 'metadata', {}) { + metadata: std.get(params.talosControlPlane, 'metadata', {}) { name: params.clusterName, namespace: params.namespace, }, @@ -135,8 +135,8 @@ local capiTalosControlPlane = { }, }, }, - controlPlaneConfig: { - controlplane: { + controlPlaneConfig+: { + controlplane+: { generateType: 'controlplane', talosVersion: params.talosVersion, hostname: { @@ -150,6 +150,7 @@ local capiTalosControlPlane = { install+: { // TODO(sg): do installers for custom schematic ids even exist? // TODO(sg): figure out the new way to do this + // TODO(sg): do we even need this at all? image: 'factory.talos.dev/openstack-installer/%(schematic_uuid)s:v%(version)s' % { schematic_uuid: params.talosSchematicUUID, version: params.talosVersion, @@ -157,6 +158,9 @@ local capiTalosControlPlane = { }, }, }), + ] + [ + std.manifestJsonMinified(patch) + for patch in std.objectValues(params.talosControlPlane.strategicPatches) ], }, }, From 3d63edf38ef0ed2b12af18a51e07e928ab0a1473 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Fri, 3 Jul 2026 08:45:11 +0200 Subject: [PATCH 17/25] Add parameter `strategicPatches` to apply custom patches to all nodes --- class/defaults.yml | 2 ++ component/main.jsonnet | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/class/defaults.yml b/class/defaults.yml index d8736c1..76fc9b3 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -59,3 +59,5 @@ parameters: metadata: {} spec: {} strategicPatches: {} + + talosStrategicPatches: {} diff --git a/component/main.jsonnet b/component/main.jsonnet index dcf2ad0..283308d 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -116,6 +116,11 @@ local talosStrategicPatch = { }, }; +local strategicPatches = [ + std.manifestJsonMinified(patch) + for patch in std.objectValues(params.talosStrategicPatches) +]; + local capiTalosControlPlane = { apiVersion: 'controlplane.cluster.x-k8s.io/v1beta1', kind: 'TalosControlPlane', @@ -144,7 +149,7 @@ local capiTalosControlPlane = { // provider. source: 'InfrastructureName', }, - strategicPatches: [ + strategicPatches: strategicPatches + [ std.manifestJsonMinified(talosStrategicPatch { machine+: { install+: { @@ -212,7 +217,7 @@ local capiWorkerGroup(name) = hostname: { source: 'InfrastructureName', }, - strategicPatches: [ + strategicPatches: strategicPatches + [ std.manifestJsonMinified(talosStrategicPatch), ], }, From 9fb9e29c5923879db9818c0d2391d446b33e35ba Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Fri, 3 Jul 2026 09:24:01 +0200 Subject: [PATCH 18/25] Set `machine.install.image` on all nodes This field is required for Tuppr to be able to perform upgrades. --- component/main.jsonnet | 26 +++++++------------ .../worker_group_worker.yaml | 6 ++--- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/component/main.jsonnet b/component/main.jsonnet index 283308d..29a31ea 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -86,6 +86,11 @@ local talosStrategicPatch = { install: { disk: '/dev/sda', wipe: true, + // NOTE(sg): image is required by Tuppr in order to compute the update + image: 'factory.talos.dev/openstack-installer/%(schematic_uuid)s:v%(version)s' % { + schematic_uuid: params.talosSchematicUUID, + version: params.talosVersion, + }, }, }, // TODO(sg): figure out if this section is really needed for worker groups. @@ -116,9 +121,12 @@ local talosStrategicPatch = { }, }; +// TODO(sg): does order matter here? local strategicPatches = [ std.manifestJsonMinified(patch) for patch in std.objectValues(params.talosStrategicPatches) +] + [ + std.manifestJsonMinified(talosStrategicPatch), ]; local capiTalosControlPlane = { @@ -150,20 +158,6 @@ local capiTalosControlPlane = { source: 'InfrastructureName', }, strategicPatches: strategicPatches + [ - std.manifestJsonMinified(talosStrategicPatch { - machine+: { - install+: { - // TODO(sg): do installers for custom schematic ids even exist? - // TODO(sg): figure out the new way to do this - // TODO(sg): do we even need this at all? - image: 'factory.talos.dev/openstack-installer/%(schematic_uuid)s:v%(version)s' % { - schematic_uuid: params.talosSchematicUUID, - version: params.talosVersion, - }, - }, - }, - }), - ] + [ std.manifestJsonMinified(patch) for patch in std.objectValues(params.talosControlPlane.strategicPatches) ], @@ -217,9 +211,7 @@ local capiWorkerGroup(name) = hostname: { source: 'InfrastructureName', }, - strategicPatches: strategicPatches + [ - std.manifestJsonMinified(talosStrategicPatch), - ], + strategicPatches: strategicPatches, }, }, }, diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml index 982a448..e5d06a0 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml @@ -17,7 +17,7 @@ spec: configRef: apiGroup: bootstrap.cluster.x-k8s.io kind: TalosConfigTemplate - name: worker-d7863e308f443d77 + name: worker-e7cba96fd5f9ccbc clusterName: c-green-test-1234 infrastructureRef: apiGroup: infrastructure.cluster.x-k8s.io @@ -44,7 +44,7 @@ spec: apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 kind: TalosConfigTemplate metadata: - name: worker-d7863e308f443d77 + name: worker-e7cba96fd5f9ccbc namespace: syn-cluster-api spec: template: @@ -53,5 +53,5 @@ spec: hostname: source: InfrastructureName strategicPatches: - - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","wipe":true},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' + - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.4","wipe":true},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' talosVersion: 1.13.4 From 6f03adfc90f4ca9f53ca2df308efe157edacaffa Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Fri, 3 Jul 2026 09:40:13 +0200 Subject: [PATCH 19/25] Configure ArgoCD to ignore changes to `spec.replicas` of MachineDeployment --- component/app.jsonnet | 17 ++++++++++++++++- .../apps/talos-capi-cluster-cloudscale.yaml | 9 +++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/component/app.jsonnet b/component/app.jsonnet index 934fccb..18049b0 100644 --- a/component/app.jsonnet +++ b/component/app.jsonnet @@ -3,7 +3,22 @@ local inv = kap.inventory(); local params = inv.parameters.talos_capi_cluster_cloudscale; local argocd = import 'lib/argocd.libjsonnet'; -local app = argocd.App('talos-capi-cluster-cloudscale', params.namespace); +local app = argocd.App('talos-capi-cluster-cloudscale', params.namespace) { + spec+: { + ignoreDifferences+: [ + { + group: 'cluster.x-k8s.io', + kind: 'MachineDeployment', + jsonPointers: [ '/spec/replicas' ], + }, + ], + syncPolicy+: { + syncOptions+: [ + 'RespectIgnoreDifferences=true', + ], + }, + }, +}; local appPath = local project = std.get(std.get(app, 'spec', {}), 'project', 'syn'); diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/apps/talos-capi-cluster-cloudscale.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/apps/talos-capi-cluster-cloudscale.yaml index e69de29..5861bd2 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/apps/talos-capi-cluster-cloudscale.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/apps/talos-capi-cluster-cloudscale.yaml @@ -0,0 +1,9 @@ +spec: + ignoreDifferences: + - group: cluster.x-k8s.io + jsonPointers: + - /spec/replicas + kind: MachineDeployment + syncPolicy: + syncOptions: + - RespectIgnoreDifferences=true From 58f491ba8283f30e42a39ec14ca93f5b1f6fc79c Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Fri, 3 Jul 2026 10:38:14 +0200 Subject: [PATCH 20/25] Make MachineDeployment deletion order configurable And set default to `Oldest` --- component/main.jsonnet | 12 ++++++++++++ .../worker_group_worker.yaml | 2 ++ 2 files changed, 14 insertions(+) diff --git a/component/main.jsonnet b/component/main.jsonnet index 29a31ea..4bb137e 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -241,6 +241,14 @@ local capiWorkerGroup(name) = }, }, }; + local mdDeletionOrder = + local valOrDefault = std.get(params.workerGroups[name], 'deletionOrder', 'Oldest'); + local validDeletionOrders = [ 'Newest', 'Oldest', 'Random' ]; + assert + std.member(validDeletionOrders, valOrDefault) + : "Invalid value '%s' for deletion order for machinedeployment '%s': " % [ valOrDefault, name ] + + 'valid options are %s' % validDeletionOrders; + valOrDefault; local machineDeployment = { apiVersion: 'cluster.x-k8s.io/v1beta2', kind: 'MachineDeployment', @@ -251,6 +259,10 @@ local capiWorkerGroup(name) = spec: { clusterName: params.clusterName, replicas: params.workerGroups[name].count, + deletion: { + // TODO(sg): decide how we want to expose useful config options. + order: std.get(params.workerGroups[name], 'deletionOrder', 'Oldest'), + }, selector: { matchLabels: null, }, diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml index e5d06a0..5ecf8f2 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml @@ -5,6 +5,8 @@ metadata: namespace: syn-cluster-api spec: clusterName: c-green-test-1234 + deletion: + order: Oldest replicas: 1 selector: matchLabels: null From 588e0f02e5c5adab67f379af0057fa9037627cb9 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Mon, 6 Jul 2026 09:33:32 +0200 Subject: [PATCH 21/25] Update component to make parameter `talosVersion` less misleading We default to patch version `0` for `install.image` in the MachineConfig. This should be unproblematic since the field only needs to be present in order for Tuppr to be able to determine the base image for upgrades and the whole `install` section has no effect on machines created from the OpenStack raw base image. --- class/defaults.yml | 6 +++++- component/main.jsonnet | 20 ++++++++++++++++--- .../capi_cluster.yaml | 10 +++++----- .../worker_group_worker.yaml | 14 ++++++------- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/class/defaults.yml b/class/defaults.yml index 76fc9b3..745bf69 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -8,7 +8,11 @@ parameters: clusterName: ${cluster:name} # TODO(sg): how will updates work with CAPI? - talosVersion: '1.13.4' + # IMPORTANT: this is only used for ensuring CAPI renders a suitable + # MachineConfig and isn't used to define the cluster's actual Talos + # version (at least when using the OpenStack raw image from the image + # factory). + talosVersion: '1.13' # NOTE(sg): this is the well-known default schematic UUID talosSchematicUUID: 376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba kubernetesVersion: 'v1.36.1' diff --git a/component/main.jsonnet b/component/main.jsonnet index 4bb137e..24534e7 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -5,6 +5,19 @@ local kube = import 'lib/kube.libjsonnet'; local inv = kap.inventory(); local params = inv.parameters.talos_capi_cluster_cloudscale; +local validateTalosVersion(tver) = + local parts = std.split(tver, '.'); + assert std.length(parts) == 2 : 'Expected Talos version to contain exacty 1 dot'; + local major = std.parseJson(parts[0]); + local minor = std.parseJson(parts[1]); + if !std.isInteger(major) || !std.isInteger(minor) then + error "Expected Talos version to be '.', got '%s'" % tver + else + { + major: major, + minor: minor, + }; + local cloudscaleImageSlug = 'custom:%s' % params.cloudscale.customImageSlug; local resourceSetLabelKey = 'talos-capi-cluster-cloudscale.syn.tools/bootstrap'; @@ -89,7 +102,8 @@ local talosStrategicPatch = { // NOTE(sg): image is required by Tuppr in order to compute the update image: 'factory.talos.dev/openstack-installer/%(schematic_uuid)s:v%(version)s' % { schematic_uuid: params.talosSchematicUUID, - version: params.talosVersion, + version: + '%(major)s.%(minor)s.0' % validateTalosVersion(params.talosVersion), }, }, }, @@ -151,7 +165,7 @@ local capiTalosControlPlane = { controlPlaneConfig+: { controlplane+: { generateType: 'controlplane', - talosVersion: params.talosVersion, + talosVersion: '%(major)s.%(minor)s' % validateTalosVersion(params.talosVersion), hostname: { // we want to use the VM name defined by the cloudscale CAPI // provider. @@ -207,7 +221,7 @@ local capiWorkerGroup(name) = template: { spec: { generateType: 'join', - talosVersion: params.talosVersion, + talosVersion: '%(major)s.%(minor)s' % validateTalosVersion(params.talosVersion), hostname: { source: 'InfrastructureName', }, diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml index a94bcd6..04a8005 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -39,13 +39,13 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate metadata: - name: c-green-test-1234-control-plane-e84c4ba30597bd6a + name: c-green-test-1234-control-plane-1a05392e618ad8f5 namespace: syn-cluster-api spec: template: spec: flavor: plus-16-4 - image: custom:talos-v1.13.4-37656798 + image: custom:talos-v1.13-37656798 interfaces: - network: privnet-c-green-test-1234 rootVolumeSize: 50 @@ -64,14 +64,14 @@ spec: hostname: source: InfrastructureName strategicPatches: - - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.4","wipe":true},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' - talosVersion: 1.13.4 + - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.0","wipe":true},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' + talosVersion: '1.13' machineTemplate: spec: infrastructureRef: apiGroup: infrastructure.cluster.x-k8s.io kind: CloudscaleMachineTemplate - name: c-green-test-1234-control-plane-e84c4ba30597bd6a + name: c-green-test-1234-control-plane-1a05392e618ad8f5 replicas: 1 version: v1.36.1 --- diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml index 5ecf8f2..95af19a 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml @@ -19,24 +19,24 @@ spec: configRef: apiGroup: bootstrap.cluster.x-k8s.io kind: TalosConfigTemplate - name: worker-e7cba96fd5f9ccbc + name: worker-b49f8b43afa7f968 clusterName: c-green-test-1234 infrastructureRef: apiGroup: infrastructure.cluster.x-k8s.io kind: CloudscaleMachineTemplate - name: worker-cb7ea8521a7ab803 + name: worker-140736b57cf90932 version: v1.36.1 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: CloudscaleMachineTemplate metadata: - name: worker-cb7ea8521a7ab803 + name: worker-140736b57cf90932 namespace: syn-cluster-api spec: template: spec: flavor: plus-16-4 - image: custom:talos-v1.13.4-37656798 + image: custom:talos-v1.13-37656798 interfaces: - network: privnet-c-green-test-1234 rootVolumeSize: 50 @@ -46,7 +46,7 @@ spec: apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 kind: TalosConfigTemplate metadata: - name: worker-e7cba96fd5f9ccbc + name: worker-b49f8b43afa7f968 namespace: syn-cluster-api spec: template: @@ -55,5 +55,5 @@ spec: hostname: source: InfrastructureName strategicPatches: - - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.4","wipe":true},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' - talosVersion: 1.13.4 + - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.0","wipe":true},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' + talosVersion: '1.13' From f301c8188acccd606dea280690fcd065952116e0 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Mon, 3 Aug 2026 15:12:51 +0200 Subject: [PATCH 22/25] Add logic to render and configure a K8s API server `AuthenticationConfiguration` Tested that this config works on Talos 1.13. Note that on Talos 1.14 we should get a real MachineConfiguration field for `AuthenticationConfiguration`. We can't write to one of the already mounted directories of the K8s API server pods, so we write our config file to `/var/config/kubernetes/kube-apiserver` and mount that directory in the API server pods. --- class/defaults.yml | 3 +++ component/main.jsonnet | 43 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/class/defaults.yml b/class/defaults.yml index 745bf69..090edb6 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -65,3 +65,6 @@ parameters: strategicPatches: {} talosStrategicPatches: {} + + kubernetesApiServer: + authenticationConfigurationJWT: {} diff --git a/component/main.jsonnet b/component/main.jsonnet index 24534e7..3bff45a 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -143,6 +143,17 @@ local strategicPatches = [ std.manifestJsonMinified(talosStrategicPatch), ]; +local authenticationConfiguration = { + apiVersion: 'apiserver.config.k8s.io/v1', + kind: 'AuthenticationConfiguration', + jwt: std.filter( + function(it) it != null, + std.objectValues(params.kubernetesApiServer.authenticationConfigurationJWT) + ), + //TODO(sg): do we want to allow configuring other top-level fields? are + //there even any other top-level fields? +}; + local capiTalosControlPlane = { apiVersion: 'controlplane.cluster.x-k8s.io/v1beta1', kind: 'TalosControlPlane', @@ -174,7 +185,37 @@ local capiTalosControlPlane = { strategicPatches: strategicPatches + [ std.manifestJsonMinified(patch) for patch in std.objectValues(params.talosControlPlane.strategicPatches) - ], + ] + if std.length(authenticationConfiguration.jwt) > 0 then + local filedir = '/var/config/kubernetes/kube-apiserver'; + local filepath = '%s/syn-authentication-configuration.yaml' % filedir; + [ + std.manifestJsonMinified({ + machine: { + files: [ + { + content: std.manifestYamlDoc(authenticationConfiguration), + permissions: std.parseOctal('0644'), + path: filepath, + op: 'create', + }, + ], + }, + cluster: { + apiServer: { + extraArgs: { + 'authentication-config': filepath, + }, + extraVolumes: [ + { + hostPath: filedir, + mountPath: filedir, + readonly: true, + }, + ], + }, + }, + }), + ] else [], }, }, }, From 8a44d78de943602880b28ee52a84b0da0ccb7bc5 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Wed, 12 Aug 2026 11:10:12 +0200 Subject: [PATCH 23/25] Remove machinedeployment `spec.selector` CAPI auto-generates the selector and when switching to server-side apply we can't have an explicit `matchLabels: null` anymore. --- component/main.jsonnet | 3 --- .../talos-capi-cluster-cloudscale/worker_group_worker.yaml | 2 -- 2 files changed, 5 deletions(-) diff --git a/component/main.jsonnet b/component/main.jsonnet index 3bff45a..23ae04c 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -318,9 +318,6 @@ local capiWorkerGroup(name) = // TODO(sg): decide how we want to expose useful config options. order: std.get(params.workerGroups[name], 'deletionOrder', 'Oldest'), }, - selector: { - matchLabels: null, - }, template: std.get(params.workerGroups[name], 'template', {}) { metadata: { labels+: { diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml index 95af19a..24ccf93 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml @@ -8,8 +8,6 @@ spec: deletion: order: Oldest replicas: 1 - selector: - matchLabels: null template: metadata: labels: From 49f9e9a109f103e7d5a227e70cec61d94822ade0 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Wed, 12 Aug 2026 11:04:11 +0200 Subject: [PATCH 24/25] Enable server-side apply in ArgoCD app --- component/app.jsonnet | 1 + .../apps/talos-capi-cluster-cloudscale.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/component/app.jsonnet b/component/app.jsonnet index 18049b0..dacead8 100644 --- a/component/app.jsonnet +++ b/component/app.jsonnet @@ -15,6 +15,7 @@ local app = argocd.App('talos-capi-cluster-cloudscale', params.namespace) { syncPolicy+: { syncOptions+: [ 'RespectIgnoreDifferences=true', + 'ServerSideApply=true', ], }, }, diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/apps/talos-capi-cluster-cloudscale.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/apps/talos-capi-cluster-cloudscale.yaml index 5861bd2..3dc4bb6 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/apps/talos-capi-cluster-cloudscale.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/apps/talos-capi-cluster-cloudscale.yaml @@ -7,3 +7,4 @@ spec: syncPolicy: syncOptions: - RespectIgnoreDifferences=true + - ServerSideApply=true From 8c9446878397312a6b531d41cf9fa123c36b31a0 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Thu, 13 Aug 2026 09:08:33 +0200 Subject: [PATCH 25/25] Enable kubelet server certificate rotation This is required for metrics-server, but requires a mechanism to approve Kubelet CSRs. We currently use https://github.com/alex1989hu/kubelet-serving-cert-approver --- component/main.jsonnet | 8 ++++++++ .../talos-capi-cluster-cloudscale/capi_cluster.yaml | 2 +- .../worker_group_worker.yaml | 6 +++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/component/main.jsonnet b/component/main.jsonnet index 23ae04c..cf48880 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -106,6 +106,14 @@ local talosStrategicPatch = { '%(major)s.%(minor)s.0' % validateTalosVersion(params.talosVersion), }, }, + kubelet: { + extraArgs: { + // NOTE(sg): required for metrics-server, but requires a mechanism to + // approve Kubelet CSRs. We currently use + // https://github.com/alex1989hu/kubelet-serving-cert-approver + 'rotate-server-certificates': true, + }, + }, }, // TODO(sg): figure out if this section is really needed for worker groups. cluster: { diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml index 04a8005..c3e8c1d 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/capi_cluster.yaml @@ -64,7 +64,7 @@ spec: hostname: source: InfrastructureName strategicPatches: - - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.0","wipe":true},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' + - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.0","wipe":true},"kubelet":{"extraArgs":{"rotate-server-certificates":true}},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' talosVersion: '1.13' machineTemplate: spec: diff --git a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml index 24ccf93..8796926 100644 --- a/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml +++ b/tests/golden/defaults/talos-capi-cluster-cloudscale/talos-capi-cluster-cloudscale/worker_group_worker.yaml @@ -17,7 +17,7 @@ spec: configRef: apiGroup: bootstrap.cluster.x-k8s.io kind: TalosConfigTemplate - name: worker-b49f8b43afa7f968 + name: worker-596c13fcc77c1a6b clusterName: c-green-test-1234 infrastructureRef: apiGroup: infrastructure.cluster.x-k8s.io @@ -44,7 +44,7 @@ spec: apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 kind: TalosConfigTemplate metadata: - name: worker-b49f8b43afa7f968 + name: worker-596c13fcc77c1a6b namespace: syn-cluster-api spec: template: @@ -53,5 +53,5 @@ spec: hostname: source: InfrastructureName strategicPatches: - - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.0","wipe":true},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' + - '{"cluster":{"externalCloudProvider":{"enabled":true},"network":{"cni":{"name":"none"}},"proxy":{"disabled":true}},"machine":{"install":{"disk":"/dev/sda","image":"factory.talos.dev/openstack-installer/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba:v1.13.0","wipe":true},"kubelet":{"extraArgs":{"rotate-server-certificates":true}},"network":{"interfaces":[{"deviceSelector":{"physical":true},"dhcp":true}]}}}' talosVersion: '1.13'