diff --git a/.prow.yaml b/.prow.yaml index a5446dd3..3c194541 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -191,3 +191,22 @@ presubmits: # docker-in-docker needs privileged mode securityContext: privileged: true + + - name: pull-kcp-operator-test-kcp-e2e + always_run: true + decorate: true + clone_uri: "https://github.com/kcp-dev/kcp-operator" + labels: + preset-goproxy: "true" + spec: + containers: + - image: ghcr.io/kcp-dev/infra/build:1.26.2-1 + command: + - hack/ci/run-kcp-e2e-tests.sh + resources: + requests: + memory: 4Gi + cpu: 2 + # docker-in-docker needs privileged mode + securityContext: + privileged: true diff --git a/hack/ci/run-kcp-e2e-tests.sh b/hack/ci/run-kcp-e2e-tests.sh new file mode 100755 index 00000000..d79dc6a8 --- /dev/null +++ b/hack/ci/run-kcp-e2e-tests.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +# Copyright 2026 The kcp Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +cd "$(dirname "$0")/../.." +source hack/lib.sh + +# build the image(s) +export IMAGE_TAG=local + +export CI_ARCH="$(go env GOARCH)" + +echo "Building container images…" +ARCHITECTURES=$CI_ARCH DRY_RUN=yes ./hack/ci/build-image.sh + +export KCP_E2E_TEST_IMAGE="ghcr.io/kcp-dev/kcp:e2e" +buildah build-using-dockerfile \ + --file test/kcp/Dockerfile \ + --tag "$KCP_E2E_TEST_IMAGE-$CI_ARCH" \ + --arch "$CI_ARCH" \ + --override-arch "$CI_ARCH" \ + --squash \ + --build-arg "TARGETOS=linux" \ + --build-arg "TARGETARCH=$CI_ARCH" \ + --format=docker \ + . + +echo "Creating manifest $KCP_E2E_TEST_IMAGE..." +buildah manifest create "$KCP_E2E_TEST_IMAGE" +buildah manifest add "$KCP_E2E_TEST_IMAGE" "$KCP_E2E_TEST_IMAGE-$CI_ARCH" + +# start docker so we can run kind +start_docker_daemon_ci + +# create a local kind cluster +KIND_CLUSTER_NAME=e2e + +echo "Preloading the kindest/node image…" +docker load --input /kindest.tar + +export KUBECONFIG=$(mktemp) +echo "Creating kind cluster $KIND_CLUSTER_NAME…" +create_kind_cluster "$KIND_CLUSTER_NAME" kindest/node:v1.32.2 +chmod 600 "$KUBECONFIG" + +# apply kernel limits job first and wait for completion +echo "Applying kernel limits job…" +KUBECTL="$(UGET_PRINT_PATH=absolute make --no-print-directory install-kubectl)" +"$KUBECTL" apply --filename hack/ci/kernel.yaml +"$KUBECTL" wait --for=condition=Complete job/kernel-limits --timeout=300s +echo "Kernel limits job completed." + +# store logs as artifacts +PROTOKOL="$(UGET_PRINT_PATH=absolute make --no-print-directory install-protokol)" +"$PROTOKOL" --output "$ARTIFACTS/logs" --namespace 'kcp-*' --namespace 'e2e-*' >/dev/null 2>&1 & + +# load the operator image into the kind cluster +image="ghcr.io/kcp-dev/kcp-operator:$IMAGE_TAG" +archive=operator.tar + +echo "Loading operator image into kind…" +buildah manifest push "$image" "oci-archive:$archive:$image" +retry_linear 1 5 kind load image-archive "$archive" --name "$KIND_CLUSTER_NAME" + +# load the tester image +echo "Loading tester image into kind…" +archive=tester.tar +buildah manifest push "$KCP_E2E_TEST_IMAGE" "oci-archive:$archive:$KCP_E2E_TEST_IMAGE" +retry_linear 1 5 kind load image-archive "$archive" --name "$KIND_CLUSTER_NAME" + +# deploy the operator + +echo "Deploying operator…" +"$KUBECTL" kustomize hack/ci/testdata | "$KUBECTL" apply --filename - +"$KUBECTL" --namespace kcp-operator-system wait deployment kcp-operator-controller-manager --for condition=Available +"$KUBECTL" --namespace kcp-operator-system wait pod --all --for condition=Ready + +# deploying cert-manager +echo "Deploying cert-manager…" + +HELM="$(UGET_PRINT_PATH=absolute make --no-print-directory install-helm)" + +"$HELM" repo add jetstack https://charts.jetstack.io --force-update +"$HELM" repo update + +"$HELM" upgrade \ + --install \ + --namespace cert-manager \ + --create-namespace \ + --version v1.20.2 \ + --set crds.enabled=true \ + cert-manager jetstack/cert-manager + +"$KUBECTL" apply --filename hack/ci/testdata/clusterissuer.yaml + +# Increase file descriptor limit for CI environments +ulimit -n 65536 + +echo "Running kcp e2e tests…" + +export KUBECTL_BINARY="$KUBECTL" +export HELM_BINARY="$HELM" +export ETCD_HELM_CHART="$(realpath hack/ci/testdata/etcd)" + +(set -x; go test -tags kcpe2e -timeout 2h -v ./test/kcp/...) + +echo "Done. :-)" diff --git a/test/kcp/Dockerfile b/test/kcp/Dockerfile new file mode 100644 index 00000000..637604c6 --- /dev/null +++ b/test/kcp/Dockerfile @@ -0,0 +1,18 @@ +# This Dockerfile creates a container image with kcp's source code in it, so +# we can deploy it as a Pod inside a kind cluster and then run the kcp e2e +# tests against the kcp-operator managed environment. + +FROM docker.io/golang:1.26.2 + +ENV HTTEST_VERSION="0.3.4" +RUN curl --fail -LO https://codeberg.org/xrstf/httest/releases/download/v${HTTEST_VERSION}/httest_${HTTEST_VERSION}_linux_$(dpkg --print-architecture).tar.gz && \ + tar xzf httest_*.tar.gz && \ + mv httest_*/httest /usr/local/bin + +WORKDIR /apps/kcp +RUN git clone --depth 1 https://github.com/kcp-dev/kcp . && \ + go build -v ./test/... + +ENV NO_GORUN=1 + +CMD [ "bash", "-c", "go test -parallel 1 ./test/e2e/... -args --kcp-kubeconfig $KUBECONFIG" ] diff --git a/test/kcp/kcp_test.go b/test/kcp/kcp_test.go new file mode 100644 index 00000000..ce4c4714 --- /dev/null +++ b/test/kcp/kcp_test.go @@ -0,0 +1,183 @@ +//go:build kcpe2e + +/* +Copyright 2026 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kcp + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/kcp-dev/logicalcluster/v3" + kcpcorev1alpha1 "github.com/kcp-dev/sdk/apis/core/v1alpha1" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrlruntime "sigs.k8s.io/controller-runtime" + + operatorv1alpha1 "github.com/kcp-dev/kcp-operator/sdk/apis/operator/v1alpha1" + "github.com/kcp-dev/kcp-operator/test/utils" +) + +func TestKcpTestSuite(t *testing.T) { + const ( + externalHostname = "example.localhost" + ) + + testImage := os.Getenv("KCP_E2E_TEST_IMAGE") + if testImage == "" { + t.Skip("No $KCP_E2E_TEST_IMAGE defined.") + } + + ctrlruntime.SetLogger(logr.Discard()) + + client := utils.GetKubeClient(t) + ctx := context.Background() + + // create namspace + namespace := utils.CreateSelfDestructingNamespace(t, ctx, client, "kcp") + + // deploy a root shard incl. etcd + rootShard := utils.DeployRootShard(ctx, t, client, namespace.Name, externalHostname) + + // deploy a 2nd shard incl. etcd + shardName := "aadvark" + utils.DeployShard(ctx, t, client, namespace.Name, shardName, rootShard.Name) + + // deploy front-proxy + utils.DeployFrontProxy(ctx, t, client, namespace.Name, rootShard.Name, externalHostname) + + // create a kubeconfig to access the root shard + rsConfigSecretName := fmt.Sprintf("%s-shard-kubeconfig", rootShard.Name) + + rsConfig := operatorv1alpha1.Kubeconfig{} + rsConfig.Name = rsConfigSecretName + rsConfig.Namespace = namespace.Name + + rsConfig.Spec = operatorv1alpha1.KubeconfigSpec{ + Target: operatorv1alpha1.KubeconfigTarget{ + RootShardRef: &corev1.LocalObjectReference{ + Name: rootShard.Name, + }, + }, + Username: "e2e", + Validity: metav1.Duration{Duration: 2 * time.Hour}, + SecretRef: corev1.LocalObjectReference{ + Name: rsConfigSecretName, + }, + Groups: []string{"system:masters"}, + } + + t.Log("Creating kubeconfig for RootShard…") + if err := client.Create(ctx, &rsConfig); err != nil { + t.Fatal(err) + } + utils.WaitForObject(t, ctx, client, &corev1.Secret{}, types.NamespacedName{Namespace: rsConfig.Namespace, Name: rsConfig.Spec.SecretRef.Name}) + + t.Log("Connecting to RootShard…") + rootShardClient := utils.ConnectWithKubeconfig(t, ctx, client, namespace.Name, rsConfig.Name, logicalcluster.None) + + // wait until the 2nd shard has registered itself successfully at the root shard + shardKey := types.NamespacedName{Name: shardName} + t.Log("Waiting for Shard to register itself on the RootShard…") + utils.WaitForObject(t, ctx, rootShardClient, &kcpcorev1alpha1.Shard{}, shardKey) + + // create a kubeconfig to access the shard + shardConfigSecretName := fmt.Sprintf("%s-shard-kubeconfig", shardName) + + shardConfig := operatorv1alpha1.Kubeconfig{} + shardConfig.Name = shardConfigSecretName + shardConfig.Namespace = namespace.Name + + shardConfig.Spec = operatorv1alpha1.KubeconfigSpec{ + Target: operatorv1alpha1.KubeconfigTarget{ + ShardRef: &corev1.LocalObjectReference{ + Name: shardName, + }, + }, + Username: "e2e", + Validity: metav1.Duration{Duration: 2 * time.Hour}, + SecretRef: corev1.LocalObjectReference{ + Name: shardConfigSecretName, + }, + Groups: []string{"system:masters"}, + } + + t.Log("Creating kubeconfig for Shard…") + if err := client.Create(ctx, &shardConfig); err != nil { + t.Fatal(err) + } + utils.WaitForObject(t, ctx, client, &corev1.Secret{}, types.NamespacedName{Namespace: shardConfig.Namespace, Name: shardConfig.Spec.SecretRef.Name}) + + t.Log("Connecting to Shard…") + kcpClient := utils.ConnectWithKubeconfig(t, ctx, client, namespace.Name, shardConfig.Name, logicalcluster.None) + + // proof of life: list something every logicalcluster in kcp has + t.Log("Should be able to list Secrets.") + secrets := &corev1.SecretList{} + if err := kcpClient.List(ctx, secrets); err != nil { + t.Fatalf("Failed to list secrets in kcp: %v", err) + } + // deploy kcp e2e test container into the cluster + testPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace.Name, + GenerateName: "kcp-e2e-", + Labels: map[string]string{ + "test": "kcp-e2e", + }, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{{ + Name: "e2e", + Image: testImage, + ImagePullPolicy: corev1.PullNever, + Env: []corev1.EnvVar{{ + Name: "KUBECONFIG", + Value: "/opt/rootshard-kubeconfig/kubeconfig", + }}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "rootshard-kubeconfig", + ReadOnly: true, + MountPath: "/opt/rootshard-kubeconfig", + }}, + }}, + Volumes: []corev1.Volume{{ + Name: "rootshard-kubeconfig", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: rsConfigSecretName, + }, + }, + }}, + }, + } + + t.Log("Creating kcp e2e test pod…") + if err := client.Create(ctx, testPod); err != nil { + t.Fatal(err) + } + + t.Log("Sleeping for 10 minutes...") + time.Sleep(10 * time.Minute) +} diff --git a/test/utils/deploy.go b/test/utils/deploy.go index ba980b1b..704ef2c6 100644 --- a/test/utils/deploy.go +++ b/test/utils/deploy.go @@ -43,7 +43,11 @@ func DeployEtcd(t *testing.T, name, namespace string) string { helmCommand = "helm" } - if err := exec.Command(helmCommand, args...).Run(); err != nil { + cmd := exec.Command(helmCommand, args...) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + + if err := cmd.Run(); err != nil { t.Fatalf("Failed to deploy etcd: %v", err) }