diff --git a/CHANGELOG.md b/CHANGELOG.md index 59739109fd..4704618225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * [Azure] Bump cluster-api-provider-azure to v1.11.4 * [Azure] Add priority class to NMI * [Core] Bump cluster api to v1.5.3 +* [Azure] Restric imds access * [Core] Enable scale from zero for node groups * [Core] Added new CR ClusterConfig for cluster configurations * [Core] Support OCI helm repositories diff --git a/pkg/cluster/internal/create/actions/createworker/createworker.go b/pkg/cluster/internal/create/actions/createworker/createworker.go index b7e98395aa..a646f34004 100644 --- a/pkg/cluster/internal/create/actions/createworker/createworker.go +++ b/pkg/cluster/internal/create/actions/createworker/createworker.go @@ -621,66 +621,60 @@ func (a *action) Execute(ctx *actions.ActionContext) error { ctx.Status.End(true) // End Installing CAPx in workload cluster // Use Calico as network policy engine in managed systems - if provider.capxProvider != "azure" && !isMachinePool { - ctx.Status.Start("Configuring Network Policy Engine in workload cluster 🚧") - defer ctx.Status.End(false) - - // Use Calico as network policy engine in managed systems - if a.keosCluster.Spec.ControlPlane.Managed { + ctx.Status.Start("Configuring Network Policy Engine in workload cluster 🚧") + defer ctx.Status.End(false) - err = installCalico(n, kubeconfigPath, privateParams, allowCommonEgressNetPolPath) - if err != nil { - return errors.Wrap(err, "failed to install Network Policy Engine in workload cluster") - } - } + // Use Calico as network policy engine in managed systems + if awsEKSEnabled { + err = installCalico(n, kubeconfigPath, privateParams, allowCommonEgressNetPolPath) + if err != nil { + return errors.Wrap(err, "failed to install Network Policy Engine in workload cluster") + } + } - // Create the allow and deny (global) network policy file in the container - denyallEgressIMDSGNetPolPath := "/kind/deny-all-egress-imds_gnetpol.yaml" - allowCAPXEgressIMDSGNetPolPath := "/kind/allow-egress-imds_gnetpol.yaml" + // Create the allow and deny (global) network policy file in the container + denyallEgressIMDSGNetPolPath := "/kind/deny-all-egress-imds_gnetpol.yaml" + allowCAPXEgressIMDSGNetPolPath := "/kind/allow-egress-imds_gnetpol.yaml" - // Allow egress in kube-system Namespace - c = "kubectl --kubeconfig " + kubeconfigPath + " -n kube-system apply -f " + allowCommonEgressNetPolPath - _, err = commons.ExecuteCommand(n, c, 5) - if err != nil { - return errors.Wrap(err, "failed to apply kube-system egress NetworkPolicy") - } - denyEgressIMDSGNetPol, err := provider.getDenyAllEgressIMDSGNetPol() - if err != nil { - return err - } + // Allow egress in kube-system Namespace + c = "kubectl --kubeconfig " + kubeconfigPath + " -n kube-system apply -f " + allowCommonEgressNetPolPath + _, err = commons.ExecuteCommand(n, c, 5) + if err != nil { + return errors.Wrap(err, "failed to apply kube-system egress NetworkPolicy") + } + denyEgressIMDSGNetPol, err := provider.getDenyAllEgressIMDSGNetPol() + if err != nil { + return err + } - c = "echo \"" + denyEgressIMDSGNetPol + "\" > " + denyallEgressIMDSGNetPolPath - _, err = commons.ExecuteCommand(n, c, 5) - if err != nil { - return errors.Wrap(err, "failed to write the deny-all-traffic-to-aws-imds global network policy") - } - allowEgressIMDSGNetPol, err := provider.getAllowCAPXEgressIMDSGNetPol() - if err != nil { - return err - } + c = "echo \"" + denyEgressIMDSGNetPol + "\" > " + denyallEgressIMDSGNetPolPath + _, err = commons.ExecuteCommand(n, c, 5) + if err != nil { + return errors.Wrap(err, "failed to write the deny-all-traffic-to-aws-imds global network policy") + } + allowEgressIMDSGNetPol, err := provider.getAllowCAPXEgressIMDSGNetPol() + if err != nil { + return err + } - c = "echo \"" + allowEgressIMDSGNetPol + "\" > " + allowCAPXEgressIMDSGNetPolPath - _, err = commons.ExecuteCommand(n, c, 5) - if err != nil { - return errors.Wrap(err, "failed to write the allow-traffic-to-aws-imds-capa global network policy") - } + c = "echo \"" + allowEgressIMDSGNetPol + "\" > " + allowCAPXEgressIMDSGNetPolPath + _, err = commons.ExecuteCommand(n, c, 5) + if err != nil { + return errors.Wrap(err, "failed to write the allow-traffic-to-aws-imds-capa global network policy") + } - // Deny CAPA egress to AWS IMDS - c = "kubectl --kubeconfig " + kubeconfigPath + " apply -f " + denyallEgressIMDSGNetPolPath - _, err = commons.ExecuteCommand(n, c, 5) - if err != nil { - return errors.Wrap(err, "failed to apply deny IMDS traffic GlobalNetworkPolicy") - } + // Deny CAPA egress to AWS IMDS + c = "kubectl --kubeconfig " + kubeconfigPath + " apply -f " + denyallEgressIMDSGNetPolPath + _, err = commons.ExecuteCommand(n, c, 5) + if err != nil { + return errors.Wrap(err, "failed to apply deny IMDS traffic GlobalNetworkPolicy") + } - // Allow CAPA egress to AWS IMDS - c = "kubectl --kubeconfig " + kubeconfigPath + " apply -f " + allowCAPXEgressIMDSGNetPolPath - _, err = commons.ExecuteCommand(n, c, 5) - if err != nil { - return errors.Wrap(err, "failed to apply allow CAPX as egress GlobalNetworkPolicy") - } + // Allow CAPA egress to AWS IMDS + c = "kubectl --kubeconfig " + kubeconfigPath + " apply -f " + allowCAPXEgressIMDSGNetPolPath + _, err = commons.ExecuteCommand(n, c, 5) - ctx.Status.End(true) // End Installing Network Policy Engine in workload cluster - } + ctx.Status.End(true) // End Installing Network Policy Engine in workload cluster if a.keosCluster.Spec.DeployAutoscaler && !isMachinePool { ctx.Status.Start("Installing cluster-autoscaler in workload cluster 🗚") @@ -904,4 +898,4 @@ func (a *action) Execute(ctx *actions.ActionContext) error { ctx.Status.End(true) // End Generating KEOS descriptor return nil -} +} \ No newline at end of file diff --git a/pkg/cluster/internal/create/actions/createworker/files/aws/allow-egress-imds_gnetpol.yaml b/pkg/cluster/internal/create/actions/createworker/files/aws/allow-egress-imds_gnetpol.yaml index 9e9ba4b345..876e6a9eb6 100644 --- a/pkg/cluster/internal/create/actions/createworker/files/aws/allow-egress-imds_gnetpol.yaml +++ b/pkg/cluster/internal/create/actions/createworker/files/aws/allow-egress-imds_gnetpol.yaml @@ -5,6 +5,7 @@ metadata: name: allow-traffic-to-aws-imds-capa spec: egress: + - action: Log - action: Allow destination: nets: @@ -14,4 +15,4 @@ spec: namespaceSelector: kubernetes.io/metadata.name in { 'kube-system', 'capa-system' } selector: app.kubernetes.io/name in {'aws-ebs-csi-driver', 'aws-load-balancer-controller' } || cluster.x-k8s.io/provider == 'infrastructure-aws' || k8s-app == 'aws-cloud-controller-manager' types: - - Egress + - Egress \ No newline at end of file diff --git a/pkg/cluster/internal/create/actions/createworker/files/aws/deny-all-egress-imds_gnetpol.yaml b/pkg/cluster/internal/create/actions/createworker/files/aws/deny-all-egress-imds_gnetpol.yaml index 83a82f591b..1e5df36c9d 100644 --- a/pkg/cluster/internal/create/actions/createworker/files/aws/deny-all-egress-imds_gnetpol.yaml +++ b/pkg/cluster/internal/create/actions/createworker/files/aws/deny-all-egress-imds_gnetpol.yaml @@ -5,10 +5,13 @@ metadata: name: deny-all-traffic-to-aws-imds spec: egress: + - action: Log - action: Deny destination: nets: - - 169.254.169.254/32 + - 169.254.169.254/32 + ports: + - 80 protocol: TCP order: 10 selector: all() diff --git a/pkg/cluster/internal/create/actions/createworker/files/azure/deny-all-egress-imds_gnetpol.yaml b/pkg/cluster/internal/create/actions/createworker/files/azure/deny-all-egress-imds_gnetpol.yaml new file mode 100644 index 0000000000..9c68e6672b --- /dev/null +++ b/pkg/cluster/internal/create/actions/createworker/files/azure/deny-all-egress-imds_gnetpol.yaml @@ -0,0 +1,23 @@ +# NMI intercepts all traffic and redirects it to 127.0.0.1:2579 +# target prot opt source destination +# DNAT tcp -- !localhost 169.254.169.254 tcp dpt:http to:127.0.0.1:2579 +# RETURN all -- anywhere anywhere +--- +apiVersion: crd.projectcalico.org/v1 +kind: GlobalNetworkPolicy +metadata: + name: deny-all-traffic-to-az-imds +spec: + egress: + - action: Log + - action: Deny + destination: + nets: + - 127.0.0.1/32 + ports: + - 2579 + protocol: TCP + order: 10 + selector: all() + types: + - Egress \ No newline at end of file diff --git a/pkg/cluster/internal/create/actions/createworker/files/gcp/allow-egress-imds_gnetpol.yaml b/pkg/cluster/internal/create/actions/createworker/files/gcp/allow-egress-imds_gnetpol.yaml index 3d4d38283f..27aca927c2 100644 --- a/pkg/cluster/internal/create/actions/createworker/files/gcp/allow-egress-imds_gnetpol.yaml +++ b/pkg/cluster/internal/create/actions/createworker/files/gcp/allow-egress-imds_gnetpol.yaml @@ -5,6 +5,7 @@ metadata: name: allow-traffic-to-gcp-imds-capg spec: egress: + - action: Log - action: Allow destination: nets: @@ -14,4 +15,4 @@ spec: namespaceSelector: kubernetes.io/metadata.name in { 'kube-system', 'capg-system' } selector: app == 'gcp-compute-persistent-disk-csi-driver' || cluster.x-k8s.io/provider == 'infrastructure-gcp' types: - - Egress + - Egress \ No newline at end of file diff --git a/pkg/cluster/internal/create/actions/createworker/files/gcp/deny-all-egress-imds_gnetpol.yaml b/pkg/cluster/internal/create/actions/createworker/files/gcp/deny-all-egress-imds_gnetpol.yaml index 6d8ca7d6e8..11c9886e65 100644 --- a/pkg/cluster/internal/create/actions/createworker/files/gcp/deny-all-egress-imds_gnetpol.yaml +++ b/pkg/cluster/internal/create/actions/createworker/files/gcp/deny-all-egress-imds_gnetpol.yaml @@ -5,10 +5,13 @@ metadata: name: deny-all-traffic-to-gcp-imds spec: egress: + - action: Log - action: Deny destination: nets: - - 169.254.169.254/32 + - 169.254.169.254/32 + ports: + - 80 protocol: TCP order: 10 selector: all() diff --git a/pkg/cluster/internal/create/actions/createworker/provider.go b/pkg/cluster/internal/create/actions/createworker/provider.go index 4163aaa448..b277aa6c88 100644 --- a/pkg/cluster/internal/create/actions/createworker/provider.go +++ b/pkg/cluster/internal/create/actions/createworker/provider.go @@ -23,6 +23,7 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "path/filepath" "regexp" "strconv" @@ -239,18 +240,33 @@ func (p *Provider) getDenyAllEgressIMDSGNetPol() (string, error) { } func (p *Provider) getAllowCAPXEgressIMDSGNetPol() (string, error) { - allowEgressIMDSGNetPolLocalPath := "files/" + p.capxProvider + "/allow-egress-imds_gnetpol.yaml" - allowEgressIMDSgnpFile, err := allowEgressIMDSgnpFiles.Open(allowEgressIMDSGNetPolLocalPath) - if err != nil { - return "", errors.Wrap(err, "error opening the allow egress IMDS file") - } - defer allowEgressIMDSgnpFile.Close() - allowEgressIMDSgnpContent, err := io.ReadAll(allowEgressIMDSgnpFile) - if err != nil { - return "", err - } - - return string(allowEgressIMDSgnpContent), nil + var allowEgressIMDSgnpContent string + var err error + + if p.capxProvider == "azure" { + azureParams := struct { + Managed bool + }{ + Managed: p.capxManaged, + } + allowEgressIMDSgnpContent, err = getManifest("azure", "allow-egress-imds_gnetpol.yaml.tmpl", azureParams) + if err != nil { + return "", errors.Wrap(err, "error opening the allow egress IMDS file") + } + } else { + allowEgressIMDSGNetPolLocalPath := "files/" + p.capxProvider + "/allow-egress-imds_gnetpol.yaml" + allowEgressIMDSgnpFile, err := allowEgressIMDSgnpFiles.Open(allowEgressIMDSGNetPolLocalPath) + if err != nil { + return "", errors.Wrap(err, "error opening the allow egress IMDS file") + } + defer allowEgressIMDSgnpFile.Close() + allowEgressIMDSgnpContentBytes, err := ioutil.ReadAll(allowEgressIMDSgnpFile) + if err != nil { + return "", err + } + allowEgressIMDSgnpContent = string(allowEgressIMDSgnpContentBytes) + } + return allowEgressIMDSgnpContent, nil } func getcapxPDB(commonsPDBLocalPath string) (string, error) { diff --git a/pkg/cluster/internal/create/actions/createworker/templates/azure/allow-egress-imds_gnetpol.yaml.tmpl b/pkg/cluster/internal/create/actions/createworker/templates/azure/allow-egress-imds_gnetpol.yaml.tmpl new file mode 100644 index 0000000000..d894f3ccec --- /dev/null +++ b/pkg/cluster/internal/create/actions/createworker/templates/azure/allow-egress-imds_gnetpol.yaml.tmpl @@ -0,0 +1,25 @@ +--- +apiVersion: crd.projectcalico.org/v1 +kind: GlobalNetworkPolicy +metadata: + name: allow-traffic-to-az-imds-capz +spec: + egress: + - action: Log + - action: Allow + destination: + nets: + - 127.0.0.1/32 + ports: + - 2579 + protocol: TCP + order: 0 +{{- if $.Managed }} + namespaceSelector: kubernetes.io/metadata.name == 'capz-system' + selector: cluster.x-k8s.io/provider == 'infrastructure-azure' +{{- else }} + namespaceSelector: kubernetes.io/metadata.name in { 'kube-system', 'capz-system' } + selector: component == 'cloud-controller-manager' || app in { 'csi-azuredisk-controller', 'csi-azurefile-controller' } || cluster.x-k8s.io/provider == 'infrastructure-azure' +{{- end }} + types: + - Egress