kcp e2e tests

.prow.yaml

@@ -191,3 +191,22 @@ presubmits:
           # docker-in-docker needs privileged mode
           securityContext:
             privileged: true
+
+  - name: pull-kcp-operator-test-kcp-e2e
+    always_run: true
+    decorate: true
+    clone_uri: "https://github.com/kcp-dev/kcp-operator"
+    labels:
+      preset-goproxy: "true"
+    spec:
+      containers:
+        - image: ghcr.io/kcp-dev/infra/build:1.26.2-1
+          command:
+            - hack/ci/run-kcp-e2e-tests.sh
+          resources:
+            requests:
+              memory: 4Gi
+              cpu: 2
+          # docker-in-docker needs privileged mode
+          securityContext:
+            privileged: true

hack/ci/run-kcp-e2e-tests.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+# Copyright 2026 The kcp Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+cd "$(dirname "$0")/../.."
+source hack/lib.sh
+
+# build the image(s)
+export IMAGE_TAG=local
+
+export CI_ARCH="$(go env GOARCH)"
+
+echo "Building container images…"
+ARCHITECTURES=$CI_ARCH DRY_RUN=yes ./hack/ci/build-image.sh
+
+export KCP_E2E_TEST_IMAGE="ghcr.io/kcp-dev/kcp:e2e"
+buildah build-using-dockerfile \
+  --file test/kcp/Dockerfile \
+  --tag "$KCP_E2E_TEST_IMAGE-$CI_ARCH" \
+  --arch "$CI_ARCH" \
+  --override-arch "$CI_ARCH" \
+  --squash \
+  --build-arg "TARGETOS=linux" \
+  --build-arg "TARGETARCH=$CI_ARCH" \
+  --format=docker \
+  .
+
+echo "Creating manifest $KCP_E2E_TEST_IMAGE..."
+buildah manifest create "$KCP_E2E_TEST_IMAGE"
+buildah manifest add "$KCP_E2E_TEST_IMAGE" "$KCP_E2E_TEST_IMAGE-$CI_ARCH"
+
+# start docker so we can run kind
+start_docker_daemon_ci
+
+# create a local kind cluster
+KIND_CLUSTER_NAME=e2e
+
+echo "Preloading the kindest/node image…"
+docker load --input /kindest.tar
+
+export KUBECONFIG=$(mktemp)
+echo "Creating kind cluster $KIND_CLUSTER_NAME…"
+create_kind_cluster "$KIND_CLUSTER_NAME" kindest/node:v1.32.2
+chmod 600 "$KUBECONFIG"
+
+# apply kernel limits job first and wait for completion
+echo "Applying kernel limits job…"
+KUBECTL="$(UGET_PRINT_PATH=absolute make --no-print-directory install-kubectl)"
+"$KUBECTL" apply --filename hack/ci/kernel.yaml
+"$KUBECTL" wait --for=condition=Complete job/kernel-limits --timeout=300s
+echo "Kernel limits job completed."
+
+# store logs as artifacts
+PROTOKOL="$(UGET_PRINT_PATH=absolute make --no-print-directory install-protokol)"
+"$PROTOKOL" --output "$ARTIFACTS/logs" --namespace 'kcp-*' --namespace 'e2e-*' >/dev/null 2>&1 &
+
+# load the operator image into the kind cluster
+image="ghcr.io/kcp-dev/kcp-operator:$IMAGE_TAG"
+archive=operator.tar
+
+echo "Loading operator image into kind…"
+buildah manifest push "$image" "oci-archive:$archive:$image"
+retry_linear 1 5 kind load image-archive "$archive" --name "$KIND_CLUSTER_NAME"
+
+# load the tester image
+echo "Loading tester image into kind…"
+archive=tester.tar
+buildah manifest push "$KCP_E2E_TEST_IMAGE" "oci-archive:$archive:$KCP_E2E_TEST_IMAGE"
+retry_linear 1 5 kind load image-archive "$archive" --name "$KIND_CLUSTER_NAME"
+
+# deploy the operator
+
+echo "Deploying operator…"
+"$KUBECTL" kustomize hack/ci/testdata | "$KUBECTL" apply --filename -
+"$KUBECTL" --namespace kcp-operator-system wait deployment kcp-operator-controller-manager --for condition=Available
+"$KUBECTL" --namespace kcp-operator-system wait pod --all --for condition=Ready
+
+# deploying cert-manager
+echo "Deploying cert-manager…"
+
+HELM="$(UGET_PRINT_PATH=absolute make --no-print-directory install-helm)"
+
+"$HELM" repo add jetstack https://charts.jetstack.io --force-update
+"$HELM" repo update
+
+"$HELM" upgrade \
+  --install \
+  --namespace cert-manager \
+  --create-namespace \
+  --version v1.20.2 \
+  --set crds.enabled=true \
+  cert-manager jetstack/cert-manager
+
+"$KUBECTL" apply --filename hack/ci/testdata/clusterissuer.yaml
+
+# Increase file descriptor limit for CI environments
+ulimit -n 65536
+
+echo "Running kcp e2e tests…"
+
+export KUBECTL_BINARY="$KUBECTL"
+export HELM_BINARY="$HELM"
+export ETCD_HELM_CHART="$(realpath hack/ci/testdata/etcd)"
+
+(set -x; go test -tags kcpe2e -timeout 2h -v ./test/kcp/...)
+
+echo "Done. :-)"

test/kcp/Dockerfile

@@ -0,0 +1,18 @@
+# This Dockerfile creates a container image with kcp's source code in it, so
+# we can deploy it as a Pod inside a kind cluster and then run the kcp e2e
+# tests against the kcp-operator managed environment.
+
+FROM docker.io/golang:1.26.2
+
+ENV HTTEST_VERSION="0.3.4"
+RUN curl --fail -LO https://codeberg.org/xrstf/httest/releases/download/v${HTTEST_VERSION}/httest_${HTTEST_VERSION}_linux_$(dpkg --print-architecture).tar.gz && \
+    tar xzf httest_*.tar.gz && \
+    mv httest_*/httest /usr/local/bin
+
+WORKDIR /apps/kcp
+RUN git clone --depth 1 https://github.com/kcp-dev/kcp . && \
+    go build -v ./test/...
+
+ENV NO_GORUN=1
+
+CMD [ "bash", "-c", "go test -parallel 1 ./test/e2e/... -args --kcp-kubeconfig $KUBECONFIG" ]

test/kcp/kcp_test.go

@@ -0,0 +1,183 @@
+//go:build kcpe2e
+
+/*
+Copyright 2026 The KCP Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kcp
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/go-logr/logr"
+	"github.com/kcp-dev/logicalcluster/v3"
+	kcpcorev1alpha1 "github.com/kcp-dev/sdk/apis/core/v1alpha1"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrlruntime "sigs.k8s.io/controller-runtime"
+
+	operatorv1alpha1 "github.com/kcp-dev/kcp-operator/sdk/apis/operator/v1alpha1"
+	"github.com/kcp-dev/kcp-operator/test/utils"
+)
+
+func TestKcpTestSuite(t *testing.T) {
+	const (
+		externalHostname = "example.localhost"
+	)
+
+	testImage := os.Getenv("KCP_E2E_TEST_IMAGE")
+	if testImage == "" {
+		t.Skip("No $KCP_E2E_TEST_IMAGE defined.")
+	}
+
+	ctrlruntime.SetLogger(logr.Discard())
+
+	client := utils.GetKubeClient(t)
+	ctx := context.Background()
+
+	// create namspace
+	namespace := utils.CreateSelfDestructingNamespace(t, ctx, client, "kcp")
+
+	// deploy a root shard incl. etcd
+	rootShard := utils.DeployRootShard(ctx, t, client, namespace.Name, externalHostname)
+
+	// deploy a 2nd shard incl. etcd
+	shardName := "aadvark"
+	utils.DeployShard(ctx, t, client, namespace.Name, shardName, rootShard.Name)
+
+	// deploy front-proxy
+	utils.DeployFrontProxy(ctx, t, client, namespace.Name, rootShard.Name, externalHostname)
+
+	// create a kubeconfig to access the root shard
+	rsConfigSecretName := fmt.Sprintf("%s-shard-kubeconfig", rootShard.Name)
+
+	rsConfig := operatorv1alpha1.Kubeconfig{}
+	rsConfig.Name = rsConfigSecretName
+	rsConfig.Namespace = namespace.Name
+
+	rsConfig.Spec = operatorv1alpha1.KubeconfigSpec{
+		Target: operatorv1alpha1.KubeconfigTarget{
+			RootShardRef: &corev1.LocalObjectReference{
+				Name: rootShard.Name,
+			},
+		},
+		Username: "e2e",
+		Validity: metav1.Duration{Duration: 2 * time.Hour},
+		SecretRef: corev1.LocalObjectReference{
+			Name: rsConfigSecretName,
+		},
+		Groups: []string{"system:masters"},
+	}
+
+	t.Log("Creating kubeconfig for RootShard…")
+	if err := client.Create(ctx, &rsConfig); err != nil {
+		t.Fatal(err)
+	}
+	utils.WaitForObject(t, ctx, client, &corev1.Secret{}, types.NamespacedName{Namespace: rsConfig.Namespace, Name: rsConfig.Spec.SecretRef.Name})
+
+	t.Log("Connecting to RootShard…")
+	rootShardClient := utils.ConnectWithKubeconfig(t, ctx, client, namespace.Name, rsConfig.Name, logicalcluster.None)
+
+	// wait until the 2nd shard has registered itself successfully at the root shard
+	shardKey := types.NamespacedName{Name: shardName}
+	t.Log("Waiting for Shard to register itself on the RootShard…")
+	utils.WaitForObject(t, ctx, rootShardClient, &kcpcorev1alpha1.Shard{}, shardKey)
+
+	// create a kubeconfig to access the shard
+	shardConfigSecretName := fmt.Sprintf("%s-shard-kubeconfig", shardName)
+
+	shardConfig := operatorv1alpha1.Kubeconfig{}
+	shardConfig.Name = shardConfigSecretName
+	shardConfig.Namespace = namespace.Name
+
+	shardConfig.Spec = operatorv1alpha1.KubeconfigSpec{
+		Target: operatorv1alpha1.KubeconfigTarget{
+			ShardRef: &corev1.LocalObjectReference{
+				Name: shardName,
+			},
+		},
+		Username: "e2e",
+		Validity: metav1.Duration{Duration: 2 * time.Hour},
+		SecretRef: corev1.LocalObjectReference{
+			Name: shardConfigSecretName,
+		},
+		Groups: []string{"system:masters"},
+	}
+
+	t.Log("Creating kubeconfig for Shard…")
+	if err := client.Create(ctx, &shardConfig); err != nil {
+		t.Fatal(err)
+	}
+	utils.WaitForObject(t, ctx, client, &corev1.Secret{}, types.NamespacedName{Namespace: shardConfig.Namespace, Name: shardConfig.Spec.SecretRef.Name})
+
+	t.Log("Connecting to Shard…")
+	kcpClient := utils.ConnectWithKubeconfig(t, ctx, client, namespace.Name, shardConfig.Name, logicalcluster.None)
+
+	// proof of life: list something every logicalcluster in kcp has
+	t.Log("Should be able to list Secrets.")
+	secrets := &corev1.SecretList{}
+	if err := kcpClient.List(ctx, secrets); err != nil {
+		t.Fatalf("Failed to list secrets in kcp: %v", err)
+	}
+	// deploy kcp e2e test container into the cluster
+	testPod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:    namespace.Name,
+			GenerateName: "kcp-e2e-",
+			Labels: map[string]string{
+				"test": "kcp-e2e",
+			},
+		},
+		Spec: corev1.PodSpec{
+			RestartPolicy: corev1.RestartPolicyNever,
+			Containers: []corev1.Container{{
+				Name:            "e2e",
+				Image:           testImage,
+				ImagePullPolicy: corev1.PullNever,
+				Env: []corev1.EnvVar{{
+					Name:  "KUBECONFIG",
+					Value: "/opt/rootshard-kubeconfig/kubeconfig",
+				}},
+				VolumeMounts: []corev1.VolumeMount{{
+					Name:      "rootshard-kubeconfig",
+					ReadOnly:  true,
+					MountPath: "/opt/rootshard-kubeconfig",
+				}},
+			}},
+			Volumes: []corev1.Volume{{
+				Name: "rootshard-kubeconfig",
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName: rsConfigSecretName,
+					},
+				},
+			}},
+		},
+	}
+
+	t.Log("Creating kcp e2e test pod…")
+	if err := client.Create(ctx, testPod); err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log("Sleeping for 10 minutes...")
+	time.Sleep(10 * time.Minute)
+}

test/utils/deploy.go

@@ -43,7 +43,11 @@ func DeployEtcd(t *testing.T, name, namespace string) string {
 		helmCommand = "helm"
 	}
 
-	if err := exec.Command(helmCommand, args...).Run(); err != nil {
+	cmd := exec.Command(helmCommand, args...)
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+
+	if err := cmd.Run(); err != nil {
 		t.Fatalf("Failed to deploy etcd: %v", err)
 	}