diff --git a/pkg/cluster/internal/action.go b/pkg/cluster/internal/action.go new file mode 100644 index 0000000000..a0d97bb1a3 --- /dev/null +++ b/pkg/cluster/internal/action.go @@ -0,0 +1,104 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package actions + +import ( + "sync" + + "sigs.k8s.io/kind/pkg/cluster/nodes" + "sigs.k8s.io/kind/pkg/cluster/nodeutils" + "sigs.k8s.io/kind/pkg/internal/apis/config" + "sigs.k8s.io/kind/pkg/internal/cli" + "sigs.k8s.io/kind/pkg/log" + + "sigs.k8s.io/kind/pkg/cluster/internal/providers" +) + +// Action defines a step of bringing up a kind cluster after initial node +// container creation +type Action interface { + Execute(ctx *ActionContext) error +} + +// ActionContext is data supplied to all actions +type ActionContext struct { + Logger log.Logger + Status *cli.Status + Config *config.Cluster + Provider providers.Provider + cache *cachedData +} + +// NewActionContext returns a new ActionContext +func NewActionContext( + logger log.Logger, + status *cli.Status, + provider providers.Provider, + cfg *config.Cluster, +) *ActionContext { + return &ActionContext{ + Logger: logger, + Status: status, + Provider: provider, + Config: cfg, + cache: &cachedData{}, + } +} + +type cachedData struct { + mu sync.RWMutex + nodes []nodes.Node +} + +func (cd *cachedData) getNodes() []nodes.Node { + cd.mu.RLock() + defer cd.mu.RUnlock() + return cd.nodes +} + +func (cd *cachedData) setNodes(n []nodes.Node) { + cd.mu.Lock() + defer cd.mu.Unlock() + cd.nodes = n +} + +// Nodes returns the list of cluster nodes, this is a cached call +func (ac *ActionContext) Nodes() ([]nodes.Node, error) { + cachedNodes := ac.cache.getNodes() + if cachedNodes != nil { + return cachedNodes, nil + } + n, err := ac.Provider.ListNodes(ac.Config.Name) + if err != nil { + return nil, err + } + ac.cache.setNodes(n) + return n, nil +} + +func (ac *ActionContext) GetNode() (nodes.Node, error) { + allNodes, err := ac.Nodes() + if err != nil { + return nil, err + } + + controlPlanes, err := nodeutils.ControlPlaneNodes(allNodes) + if err != nil { + return nil, err + } + return controlPlanes[0], nil +} diff --git a/pkg/cluster/internal/create/actions/createworker/createworker.go b/pkg/cluster/internal/create/actions/createworker/createworker.go index 3a23a04d4f..fb2e87177f 100644 --- a/pkg/cluster/internal/create/actions/createworker/createworker.go +++ b/pkg/cluster/internal/create/actions/createworker/createworker.go @@ -56,14 +56,6 @@ var allowCommonEgressNetPol string //go:embed files/gcp/rbac-loadbalancing.yaml var rbacInternalLoadBalancing string -// In common with keos installer -// -//go:embed files/aws/deny-all-egress-imds_gnetpol.yaml -var denyallEgressIMDSGNetPol string - -//go:embed files/aws/allow-capa-egress-imds_gnetpol.yaml -var allowCAPAEgressIMDSGNetPol string - // NewAction returns a new action for installing default CAPI func NewAction(vaultPassword string, descriptorPath string, moveManagement bool, avoidCreation bool) actions.Action { return &action{ @@ -90,8 +82,6 @@ func (a *action) Execute(ctx *actions.ActionContext) error { if err != nil { return errors.Wrap(err, "failed to parse cluster descriptor") } - //spec := keosCluster.Spec - // Get the secrets credentialsMap, keosRegistry, githubToken, dockerRegistries, err := commons.GetSecrets(keosCluster.Spec, a.vaultPassword) @@ -466,57 +456,65 @@ func (a *action) Execute(ctx *actions.ActionContext) error { ctx.Status.End(true) // End Installing CAPx in workload cluster + ctx.Status.Start("Installing Network Policy Engine in workload cluster 🚧") + defer ctx.Status.End(false) + // Use Calico as network policy engine in managed systems if provider.capxProvider != "azure" && keosCluster.Spec.ControlPlane.Managed { - ctx.Status.Start("Installing Network Policy Engine in workload cluster 🚧") - defer ctx.Status.End(false) err = installCalico(n, kubeconfigPath, *keosCluster, allowCommonEgressNetPolPath) if err != nil { return errors.Wrap(err, "failed to install Network Policy Engine in workload cluster") } + } - // Create the allow and deny (global) network policy file in the container - if keosCluster.Spec.InfraProvider == "aws" { - denyallEgressIMDSGNetPolPath := "/kind/deny-all-egress-imds_gnetpol.yaml" - allowCAPAEgressIMDSGNetPolPath := "/kind/allow-capa-egress-imds_gnetpol.yaml" + // Create the allow and deny (global) network policy file in the container + denyallEgressIMDSGNetPolPath := "/kind/deny-all-egress-imds_gnetpol.yaml" + allowCAPXEgressIMDSGNetPolPath := "/kind/allow-egress-imds_gnetpol.yaml" - // Allow egress in kube-system Namespace - c = "kubectl --kubeconfig " + kubeconfigPath + " -n kube-system apply -f " + allowCommonEgressNetPolPath - _, err = commons.ExecuteCommand(n, c) - if err != nil { - return errors.Wrap(err, "failed to apply kube-system egress NetworkPolicy") - } + // Allow egress in kube-system Namespace + c = "kubectl --kubeconfig " + kubeconfigPath + " -n kube-system apply -f " + allowCommonEgressNetPolPath + _, err = commons.ExecuteCommand(n, c) + if err != nil { + return errors.Wrap(err, "failed to apply kube-system egress NetworkPolicy") + } + denyEgressIMDSGNetPol, err := provider.getDenyAllEgressIMDSGNetPol() + if err != nil { + return err + } - c = "echo \"" + denyallEgressIMDSGNetPol + "\" > " + denyallEgressIMDSGNetPolPath - _, err = commons.ExecuteCommand(n, c) - if err != nil { - return errors.Wrap(err, "failed to write the deny-all-traffic-to-aws-imds global network policy") - } - c = "echo \"" + allowCAPAEgressIMDSGNetPol + "\" > " + allowCAPAEgressIMDSGNetPolPath - _, err = commons.ExecuteCommand(n, c) - if err != nil { - return errors.Wrap(err, "failed to write the allow-traffic-to-aws-imds-capa global network policy") - } + c = "echo \"" + denyEgressIMDSGNetPol + "\" > " + denyallEgressIMDSGNetPolPath + _, err = commons.ExecuteCommand(n, c) + if err != nil { + return errors.Wrap(err, "failed to write the deny-all-traffic-to-aws-imds global network policy") + } + allowEgressIMDSGNetPol, err := provider.getAllowCAPXEgressIMDSGNetPol() + if err != nil { + return err + } - // Deny CAPA egress to AWS IMDS - c = "kubectl --kubeconfig " + kubeconfigPath + " apply -f " + denyallEgressIMDSGNetPolPath - _, err = commons.ExecuteCommand(n, c) - if err != nil { - return errors.Wrap(err, "failed to apply deny IMDS traffic GlobalNetworkPolicy") - } + c = "echo \"" + allowEgressIMDSGNetPol + "\" > " + allowCAPXEgressIMDSGNetPolPath + _, err = commons.ExecuteCommand(n, c) + if err != nil { + return errors.Wrap(err, "failed to write the allow-traffic-to-aws-imds-capa global network policy") + } - // Allow CAPA egress to AWS IMDS - c = "kubectl --kubeconfig " + kubeconfigPath + " apply -f " + allowCAPAEgressIMDSGNetPolPath - _, err = commons.ExecuteCommand(n, c) - if err != nil { - return errors.Wrap(err, "failed to apply allow CAPA as egress GlobalNetworkPolicy") - } - } + // Deny CAPA egress to AWS IMDS + c = "kubectl --kubeconfig " + kubeconfigPath + " apply -f " + denyallEgressIMDSGNetPolPath + _, err = commons.ExecuteCommand(n, c) + if err != nil { + return errors.Wrap(err, "failed to apply deny IMDS traffic GlobalNetworkPolicy") + } - ctx.Status.End(true) // End Installing Network Policy Engine in workload cluster + // Allow CAPA egress to AWS IMDS + c = "kubectl --kubeconfig " + kubeconfigPath + " apply -f " + allowCAPXEgressIMDSGNetPolPath + _, err = commons.ExecuteCommand(n, c) + if err != nil { + return errors.Wrap(err, "failed to apply allow CAPX as egress GlobalNetworkPolicy") } + ctx.Status.End(true) // End Installing Network Policy Engine in workload cluster + if keosCluster.Spec.DeployAutoscaler && !(keosCluster.Spec.InfraProvider == "azure" && keosCluster.Spec.ControlPlane.Managed) { ctx.Status.Start("Adding Cluster-Autoescaler 🗚") defer ctx.Status.End(false) diff --git a/pkg/cluster/internal/create/actions/createworker/files/aws/allow-capa-egress-imds_gnetpol.yaml b/pkg/cluster/internal/create/actions/createworker/files/aws/allow-egress-imds_gnetpol.yaml similarity index 100% rename from pkg/cluster/internal/create/actions/createworker/files/aws/allow-capa-egress-imds_gnetpol.yaml rename to pkg/cluster/internal/create/actions/createworker/files/aws/allow-egress-imds_gnetpol.yaml diff --git a/pkg/cluster/internal/create/actions/createworker/files/azure/allow-egress-imds_gnetpol.yaml b/pkg/cluster/internal/create/actions/createworker/files/azure/allow-egress-imds_gnetpol.yaml new file mode 100644 index 0000000000..894777ddfe --- /dev/null +++ b/pkg/cluster/internal/create/actions/createworker/files/azure/allow-egress-imds_gnetpol.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: crd.projectcalico.org/v1 +kind: GlobalNetworkPolicy +metadata: + name: allow-traffic-to-azure-imds-capz +spec: + egress: + - action: Allow + destination: + nets: + - 169.254.169.254/32 + protocol: TCP + order: 0 + namespaceSelector: kubernetes.io/metadata.name in { 'kube-system', 'capz-system' } + selector: app.kubernetes.io/name == 'azuredisk-csi-driver' || cluster.x-k8s.io/provider == 'infrastructure-azure' + types: + - Egress diff --git a/pkg/cluster/internal/create/actions/createworker/files/azure/deny-all-egress-imds_gnetpol.yaml b/pkg/cluster/internal/create/actions/createworker/files/azure/deny-all-egress-imds_gnetpol.yaml new file mode 100644 index 0000000000..5ee7c9087d --- /dev/null +++ b/pkg/cluster/internal/create/actions/createworker/files/azure/deny-all-egress-imds_gnetpol.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: crd.projectcalico.org/v1 +kind: GlobalNetworkPolicy +metadata: + name: deny-all-traffic-to-azure-imds +spec: + egress: + - action: Deny + destination: + nets: + - 169.254.169.254/32 + protocol: TCP + order: 10 + selector: all() + types: + - Egress + + + + \ No newline at end of file diff --git a/pkg/cluster/internal/create/actions/createworker/files/gcp-compute-persistent-disk-csi-driver.yaml b/pkg/cluster/internal/create/actions/createworker/files/gcp-compute-persistent-disk-csi-driver.yaml index 679fb1fc2a..8b6a5ddf7d 100644 --- a/pkg/cluster/internal/create/actions/createworker/files/gcp-compute-persistent-disk-csi-driver.yaml +++ b/pkg/cluster/internal/create/actions/createworker/files/gcp-compute-persistent-disk-csi-driver.yaml @@ -2,13 +2,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver + namespace: kube-system --- apiVersion: v1 kind: ServiceAccount metadata: name: csi-gce-pd-node-sa - namespace: gce-pd-csi-driver + namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -16,7 +16,7 @@ metadata: labels: k8s-app: gcp-compute-persistent-disk-csi-driver name: csi-gce-pd-leaderelection-role - namespace: gce-pd-csi-driver + namespace: kube-system rules: - apiGroups: - coordination.k8s.io @@ -285,7 +285,7 @@ metadata: labels: k8s-app: gcp-compute-persistent-disk-csi-driver name: csi-gce-pd-controller-leaderelection-binding - namespace: gce-pd-csi-driver + namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -293,7 +293,7 @@ roleRef: subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver + namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -306,7 +306,7 @@ roleRef: subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver + namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -319,7 +319,7 @@ roleRef: subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver + namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -332,7 +332,7 @@ roleRef: subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver + namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -345,7 +345,7 @@ roleRef: subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver + namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -358,7 +358,7 @@ roleRef: subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver + namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -371,7 +371,7 @@ roleRef: subjects: - kind: ServiceAccount name: csi-gce-pd-node-sa - namespace: gce-pd-csi-driver + namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -384,7 +384,7 @@ roleRef: subjects: - kind: ServiceAccount name: csi-gce-pd-controller-sa - namespace: gce-pd-csi-driver + namespace: kube-system --- apiVersion: scheduling.k8s.io/v1 description: This priority class should be used for the GCE PD CSI driver controller @@ -408,9 +408,9 @@ apiVersion: apps/v1 kind: Deployment metadata: name: csi-gce-pd-controller - namespace: gce-pd-csi-driver + namespace: kube-system spec: - replicas: 1 + replicas: 2 selector: matchLabels: app: gcp-compute-persistent-disk-csi-driver @@ -418,7 +418,19 @@ spec: metadata: labels: app: gcp-compute-persistent-disk-csi-driver - spec: + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - gcp-compute-persistent-disk-csi-driver + topologyKey: "kubernetes.io/hostname" + weight: 100 containers: - args: - --v=5 @@ -560,7 +572,7 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: csi-gce-pd-node - namespace: gce-pd-csi-driver + namespace: kube-system spec: selector: matchLabels: diff --git a/pkg/cluster/internal/create/actions/createworker/files/gcp/allow-egress-imds_gnetpol.yaml b/pkg/cluster/internal/create/actions/createworker/files/gcp/allow-egress-imds_gnetpol.yaml new file mode 100644 index 0000000000..3d4d38283f --- /dev/null +++ b/pkg/cluster/internal/create/actions/createworker/files/gcp/allow-egress-imds_gnetpol.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: crd.projectcalico.org/v1 +kind: GlobalNetworkPolicy +metadata: + name: allow-traffic-to-gcp-imds-capg +spec: + egress: + - action: Allow + destination: + nets: + - 169.254.169.254/32 + protocol: TCP + order: 0 + namespaceSelector: kubernetes.io/metadata.name in { 'kube-system', 'capg-system' } + selector: app == 'gcp-compute-persistent-disk-csi-driver' || cluster.x-k8s.io/provider == 'infrastructure-gcp' + types: + - Egress diff --git a/pkg/cluster/internal/create/actions/createworker/files/gcp/deny-all-egress-imds_gnetpol.yaml b/pkg/cluster/internal/create/actions/createworker/files/gcp/deny-all-egress-imds_gnetpol.yaml new file mode 100644 index 0000000000..81319d14b6 --- /dev/null +++ b/pkg/cluster/internal/create/actions/createworker/files/gcp/deny-all-egress-imds_gnetpol.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: crd.projectcalico.org/v1 +kind: GlobalNetworkPolicy +metadata: + name: deny-all-traffic-to-gcp-imds +spec: + egress: + - action: Deny + destination: + nets: + - 169.254.169.254/32 + protocol: TCP + order: 10 + selector: all() + types: + - Egress + + + \ No newline at end of file diff --git a/pkg/cluster/internal/create/actions/createworker/gcp.go b/pkg/cluster/internal/create/actions/createworker/gcp.go index 4de74e47cb..122cbd5781 100644 --- a/pkg/cluster/internal/create/actions/createworker/gcp.go +++ b/pkg/cluster/internal/create/actions/createworker/gcp.go @@ -69,7 +69,7 @@ func (b *GCPBuilder) setCapx(managed bool) { } else { b.capxManaged = false b.capxTemplate = "gcp.tmpl" - b.csiNamespace = "gce-pd-csi-driver" + b.csiNamespace = "" } } @@ -136,16 +136,9 @@ func (b *GCPBuilder) installCSI(n nodes.Node, k string) error { var err error var cmd exec.Cmd - // Create CSI namespace - c = "kubectl --kubeconfig " + k + " create namespace " + b.csiNamespace - _, err = commons.ExecuteCommand(n, c) - if err != nil { - return errors.Wrap(err, "failed to create CSI namespace") - } - // Create CSI secret in CSI namespace secret, _ := b64.StdEncoding.DecodeString(strings.Split(b.capxEnvVars[0], "GCP_B64ENCODED_CREDENTIALS=")[1]) - c = "kubectl --kubeconfig " + k + " -n " + b.csiNamespace + " create secret generic cloud-sa --from-literal=cloud-sa.json='" + string(secret) + "'" + c = "kubectl --kubeconfig " + k + " -n kube-system create secret generic cloud-sa --from-literal=cloud-sa.json='" + string(secret) + "'" _, err = commons.ExecuteCommand(n, c) if err != nil { return errors.Wrap(err, "failed to create CSI secret in CSI namespace") diff --git a/pkg/cluster/internal/create/actions/createworker/provider.go b/pkg/cluster/internal/create/actions/createworker/provider.go index ba5dcdd482..fc8a4720a7 100644 --- a/pkg/cluster/internal/create/actions/createworker/provider.go +++ b/pkg/cluster/internal/create/actions/createworker/provider.go @@ -20,6 +20,7 @@ import ( "bytes" "embed" "encoding/base64" + "io/ioutil" "reflect" "strings" "text/template" @@ -33,6 +34,12 @@ import ( //go:embed templates/* var ctel embed.FS +//go:embed files/*/deny-all-egress-imds_gnetpol.yaml +var denyAllEgressIMDSgnpFiles embed.FS + +//go:embed files/*/allow-egress-imds_gnetpol.yaml +var allowEgressIMDSgnpFiles embed.FS + const ( CAPICoreProvider = "cluster-api:v1.4.3" CAPIBootstrapProvider = "kubeadm:v1.4.3" @@ -181,6 +188,36 @@ func (i *Infra) getAzs(networks commons.Networks) ([]string, error) { return azs, nil } +func (p *Provider) getDenyAllEgressIMDSGNetPol() (string, error) { + denyAllEgressIMDSGNetPolLocalPath := "files/" + p.capxProvider + "/deny-all-egress-imds_gnetpol.yaml" + denyAllEgressIMDSgnpFile, err := denyAllEgressIMDSgnpFiles.Open(denyAllEgressIMDSGNetPolLocalPath) + if err != nil { + return "", errors.Wrap(err, "error opening the deny all egress IMDS file") + } + defer denyAllEgressIMDSgnpFile.Close() + denyAllEgressIMDSgnpContent, err := ioutil.ReadAll(denyAllEgressIMDSgnpFile) + if err != nil { + return "", err + } + + return string(denyAllEgressIMDSgnpContent), nil +} + +func (p *Provider) getAllowCAPXEgressIMDSGNetPol() (string, error) { + allowEgressIMDSGNetPolLocalPath := "files/" + p.capxProvider + "/allow-egress-imds_gnetpol.yaml" + allowEgressIMDSgnpFile, err := allowEgressIMDSgnpFiles.Open(allowEgressIMDSGNetPolLocalPath) + if err != nil { + return "", errors.Wrap(err, "error opening the allow egress IMDS file") + } + defer allowEgressIMDSgnpFile.Close() + allowEgressIMDSgnpContent, err := ioutil.ReadAll(allowEgressIMDSgnpFile) + if err != nil { + return "", err + } + + return string(allowEgressIMDSgnpContent), nil +} + func installCalico(n nodes.Node, k string, keosCluster commons.KeosCluster, allowCommonEgressNetPolPath string) error { var c string var cmd exec.Cmd diff --git a/pkg/cmd/kind/create/cluster/validation/gcpvalidator.go b/pkg/cmd/kind/create/cluster/validation/gcpvalidator.go index e439cee813..ee91f45202 100644 --- a/pkg/cmd/kind/create/cluster/validation/gcpvalidator.go +++ b/pkg/cmd/kind/create/cluster/validation/gcpvalidator.go @@ -2,6 +2,7 @@ package validation import ( "errors" + "fmt" "regexp" "strconv" "strings" @@ -109,6 +110,8 @@ func (v *GCPValidator) storageClassParametersValidation(spec commons.Spec) error sc := spec.StorageClass k8s_version := spec.K8SVersion minor, _ := strconv.Atoi(strings.Split(k8s_version, ".")[1]) + fstypes := []string{"ext4", "ext3", "ext2", "xfs", "btrfs", "ntfs"} + err := verifyFields(spec) if err != nil { return err @@ -126,6 +129,9 @@ func (v *GCPValidator) storageClassParametersValidation(spec commons.Spec) error if sc.Parameters.Type != "pd-extreme" && sc.Parameters.ProvisionedIopsOnCreate != "" { return errors.New("Parameter provisioned_iops_on_create only can be supported for type pd-extreme") } + if sc.Parameters.FsType != "" && !slices.Contains(fstypes, sc.Parameters.FsType) { + return errors.New("Unsupported fsType: " + sc.Parameters.FsType + ". Supported types: " + fmt.Sprint(strings.Join(fstypes, ", "))) + } if sc.Parameters.ProvisionedIopsOnCreate != "" { _, err = strconv.Atoi(sc.Parameters.ProvisionedIopsOnCreate)