From 1f86bb63c07ea0bbdbbfc74b75c8d56ce6c2e603 Mon Sep 17 00:00:00 2001 From: Brandi McCall Date: Fri, 10 Apr 2026 15:24:59 -0600 Subject: [PATCH 1/5] update testing.mdx --- site/src/content/docs/contribute/testing.mdx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/site/src/content/docs/contribute/testing.mdx b/site/src/content/docs/contribute/testing.mdx index ffaf3b8579..a94ce29a19 100644 --- a/site/src/content/docs/contribute/testing.mdx +++ b/site/src/content/docs/contribute/testing.mdx @@ -22,6 +22,9 @@ There are several ways to run tests depending on your specific situation, such a # The default way, from the root directory of the repo. Will run all of the unit tests that are currently defined. make test-unit +# To get a single line readable summary of all the unit tests currently defined: +make test-unit 2>&1 | grep -E "^(ok|FAIL)\s" + # If you already have everything built, you can run this inside this folder. This lets you customize the test run. go test ./src/pkg/... -v @@ -29,6 +32,8 @@ go test ./src/pkg/... -v go test ./src/pkg/... -v -run TestFooBarBaz ``` +All unit tests should pass locally before submitting a PR. + ### Adding New Unit Tests To create a unit test, search for or create a file that ends with `_test.go` in the package of the file that requires testing, such as `auth.go` -> `auth_test.go`. Import the testing library and create test functions as necessary. Generic unit test helper functions can be found in [src/test/testutil](https://github.com/zarf-dev/zarf/tree/main/src/test/testutil). From bcb978b223f9bdab3b11d9f6fce955ae57e0ca37 Mon Sep 17 00:00:00 2001 From: Brandi McCall Date: Fri, 24 Apr 2026 16:01:54 -0600 Subject: [PATCH 2/5] add e2e no-cluster error --- site/src/content/docs/contribute/testing.mdx | 11 ++++++++--- src/test/e2e/20_zarf_init_test.go | 6 ++++++ src/test/e2e/main_test.go | 10 ++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/site/src/content/docs/contribute/testing.mdx b/site/src/content/docs/contribute/testing.mdx index a94ce29a19..137df5a06c 100644 --- a/site/src/content/docs/contribute/testing.mdx +++ b/site/src/content/docs/contribute/testing.mdx @@ -12,7 +12,9 @@ To run the end-to-end tests locally, you must meet the same prerequisites as tho 1. GoLang version >= the version specified in the [go.mod](https://github.com/zarf-dev/zarf/blob/main/go.mod) 2. Make -3. Any clean K8s cluster (local or remote) or Linux with `root` if you want to use the Zarf-installed K3s cluster +3. Docker +4. Docker Buildx +5. Any clean K8s cluster (local or remote) or Linux with `root` if you want to use the Zarf-installed K3s cluster ## Unit Tests @@ -22,7 +24,7 @@ There are several ways to run tests depending on your specific situation, such a # The default way, from the root directory of the repo. Will run all of the unit tests that are currently defined. make test-unit -# To get a single line readable summary of all the unit tests currently defined: +# To get a single-line readable summary of all the unit tests currently defined: make test-unit 2>&1 | grep -E "^(ok|FAIL)\s" # If you already have everything built, you can run this inside this folder. This lets you customize the test run. @@ -56,10 +58,13 @@ There are several ways to run tests depending on your specific situation, such a # The default way, from the root directory of the repo. Will run all of the tests against your chosen k8s distro. Will automatically build any binary dependencies that don't already exist. make test-e2e ARCH="[amd64|arm64]" +# To get a single-line readable summary of all the e2e tests currently defined: +cd src/test/e2e && go test ./main_test.go ./[01]* -failfast -v -timeout 35m 2>&1 | grep -E "^(--- FAIL|--- PASS|FAIL|ok)" + # To test against a Zarf-created cluster (on Linux with sudo) APPLIANCE_MODE=true make test-e2e ARCH="[amd64|arm64]" -# If you already have everything build, you can run this inside this folder. This lets you customize the test run. +# If you already have everything built, you can run this inside this folder. This lets you customize the test run. go test ./src/test/... -v -failfast -count=1 # Let's say you only want to run one test. You would run: diff --git a/src/test/e2e/20_zarf_init_test.go b/src/test/e2e/20_zarf_init_test.go index ce8cc4087e..87c8e25ff1 100644 --- a/src/test/e2e/20_zarf_init_test.go +++ b/src/test/e2e/20_zarf_init_test.go @@ -19,6 +19,12 @@ import ( func TestZarfInit(t *testing.T) { t.Log("E2E: Zarf init") + // Verify cluster connectivity before running with-cluster tests + if !e2e.ApplianceMode { + _, _, err := e2e.Kubectl(t, "cluster-info") + require.NoError(t, err, "No cluster found. Ensure a valid kubeconfig is available and cluster is running.") + } + initComponents := "git-server" if e2e.ApplianceMode { initComponents = "k3s,git-server" diff --git a/src/test/e2e/main_test.go b/src/test/e2e/main_test.go index 50ec10291d..13b0f33e84 100644 --- a/src/test/e2e/main_test.go +++ b/src/test/e2e/main_test.go @@ -5,6 +5,7 @@ package test import ( + // "context" "log" "os" "path/filepath" @@ -12,6 +13,7 @@ import ( "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/test" + // "github.com/zarf-dev/zarf/src/pkg/utils/exec" ) var ( @@ -45,5 +47,13 @@ func TestMain(m *testing.M) { if _, err := os.Stat(e2e.ZarfBinPath); err != nil { log.Fatalf("zarf binary %s not found: %v", e2e.ZarfBinPath, err) } + + // // If running with-cluster tests validate cluster connectivity early + // if os.Getenv(skipK8sEnvVar) != "true" && !e2e.ApplianceMode { + // _, _, err := exec.CmdWithContext(context.Background(), exec.Config{}, e2e.ZarfBinPath, "tools", "kubectl", "cluster-info") + // if err != nil { + // log.Fatalf("No cluster found. Ensure a valid kubeconfig is available and cluster is running: %v", err) + // } + // } os.Exit(m.Run()) } From 09b4781ef0ffcf6f031c9d3dd1e5bf2b0532c872 Mon Sep 17 00:00:00 2001 From: Brandi McCall Date: Fri, 22 May 2026 11:42:21 -0600 Subject: [PATCH 3/5] comment out 20 test addition --- src/test/e2e/20_zarf_init_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/e2e/20_zarf_init_test.go b/src/test/e2e/20_zarf_init_test.go index 87c8e25ff1..493223ab8f 100644 --- a/src/test/e2e/20_zarf_init_test.go +++ b/src/test/e2e/20_zarf_init_test.go @@ -19,11 +19,11 @@ import ( func TestZarfInit(t *testing.T) { t.Log("E2E: Zarf init") - // Verify cluster connectivity before running with-cluster tests - if !e2e.ApplianceMode { - _, _, err := e2e.Kubectl(t, "cluster-info") - require.NoError(t, err, "No cluster found. Ensure a valid kubeconfig is available and cluster is running.") - } + // // Verify cluster connectivity before running with-cluster tests + // if !e2e.ApplianceMode { + // _, _, err := e2e.Kubectl(t, "cluster-info") + // require.NoError(t, err, "No cluster found. Ensure a valid kubeconfig is available and cluster is running.") + // } initComponents := "git-server" if e2e.ApplianceMode { From b661ae53a718af5eab920a4562babb2959c9ee01 Mon Sep 17 00:00:00 2001 From: Brandi McCall Date: Fri, 22 May 2026 11:46:26 -0600 Subject: [PATCH 4/5] update testing.mdx --- site/src/content/docs/contribute/testing.mdx | 29 ++++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/site/src/content/docs/contribute/testing.mdx b/site/src/content/docs/contribute/testing.mdx index 137df5a06c..dceb58269b 100644 --- a/site/src/content/docs/contribute/testing.mdx +++ b/site/src/content/docs/contribute/testing.mdx @@ -6,6 +6,7 @@ Currently, we primarily test Zarf through a series of [end-to-end tests](https:/ In addition, Zarf implements unit tests for specific functions where edge cases prove difficult to cover through end-to-end testing alone. Unit tests follow standard Go convention and are `*_test.go` files. +# TODO: update any more dependencies ## Dependencies To run the end-to-end tests locally, you must meet the same prerequisites as those required for building and running Zarf, which include: @@ -16,6 +17,7 @@ To run the end-to-end tests locally, you must meet the same prerequisites as tho 4. Docker Buildx 5. Any clean K8s cluster (local or remote) or Linux with `root` if you want to use the Zarf-installed K3s cluster +# TODO: DONE these have been tested on mac and linux so this section is GTG ## Unit Tests There are several ways to run tests depending on your specific situation, such as: @@ -30,10 +32,11 @@ make test-unit 2>&1 | grep -E "^(ok|FAIL)\s" # If you already have everything built, you can run this inside this folder. This lets you customize the test run. go test ./src/pkg/... -v -# Let's say you only want to run one test. You would run: +# To run a single test: go test ./src/pkg/... -v -run TestFooBarBaz ``` +# TODO: make new unit test and test instructions All unit tests should pass locally before submitting a PR. ### Adding New Unit Tests @@ -61,8 +64,8 @@ make test-e2e ARCH="[amd64|arm64]" # To get a single-line readable summary of all the e2e tests currently defined: cd src/test/e2e && go test ./main_test.go ./[01]* -failfast -v -timeout 35m 2>&1 | grep -E "^(--- FAIL|--- PASS|FAIL|ok)" -# To test against a Zarf-created cluster (on Linux with sudo) -APPLIANCE_MODE=true make test-e2e ARCH="[amd64|arm64]" +# To test against a Zarf-created cluster (Linux only with sudo, not compatible with macOS) +APPLIANCE_MODE=true make test-e2e ARCH="amd64" # If you already have everything built, you can run this inside this folder. This lets you customize the test run. go test ./src/test/... -v -failfast -count=1 @@ -156,3 +159,23 @@ The end-to-end tests are run sequentially and the naming convention is set inten - 22 is reserved for tests required the git-server, which is removed at the end of the test. - 23-98 are for the remaining tests that only require a basic Zarf cluster without the git-server. - 99 is reserved for the `zarf destroy` and [YOLO Mode](/ref/examples/yolo/) test. + +### Troubleshooting ### + +Potential e2e testing errors and causes: +Docker/Buildx: + +`unknown flag: --load` → Buildx not installed or broken symlink + +Cluster: + +dial tcp [::1]:8080: connect: connection refused → No cluster configured or kubeconfig not set +error loading config file: duplicate name in list → Duplicate entries in ~/.kube/config + +Git/1Password: + +error: 1Password: Could not connect to socket → 1Password closed during rebase commit signing + +Architecture: + +no compatible component named k3s found → Either wrong ARCH value or trying to run Linux-only components on macOS From 7b4402984a7f67b40920c82c4ea961d3e8794cbb Mon Sep 17 00:00:00 2001 From: Brandi McCall Date: Fri, 22 May 2026 14:28:06 -0600 Subject: [PATCH 5/5] add todo notes for myself, add WRN if APPLIANCE_MODE set on macOS or if cluster not found --- site/src/content/docs/contribute/testing.mdx | 9 ++-- src/test/e2e/20_zarf_init_test.go | 6 --- src/test/e2e/main_test.go | 51 +++++++++++++++++--- 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/site/src/content/docs/contribute/testing.mdx b/site/src/content/docs/contribute/testing.mdx index dceb58269b..62252a12d7 100644 --- a/site/src/content/docs/contribute/testing.mdx +++ b/site/src/content/docs/contribute/testing.mdx @@ -6,8 +6,7 @@ Currently, we primarily test Zarf through a series of [end-to-end tests](https:/ In addition, Zarf implements unit tests for specific functions where edge cases prove difficult to cover through end-to-end testing alone. Unit tests follow standard Go convention and are `*_test.go` files. -# TODO: update any more dependencies -## Dependencies +## Dependencies # TODO: update any more dependencies To run the end-to-end tests locally, you must meet the same prerequisites as those required for building and running Zarf, which include: @@ -17,8 +16,7 @@ To run the end-to-end tests locally, you must meet the same prerequisites as tho 4. Docker Buildx 5. Any clean K8s cluster (local or remote) or Linux with `root` if you want to use the Zarf-installed K3s cluster -# TODO: DONE these have been tested on mac and linux so this section is GTG -## Unit Tests +## Unit Tests # TODO: DONE these have been tested on mac and linux so this section is GTG There are several ways to run tests depending on your specific situation, such as: @@ -36,10 +34,9 @@ go test ./src/pkg/... -v go test ./src/pkg/... -v -run TestFooBarBaz ``` -# TODO: make new unit test and test instructions All unit tests should pass locally before submitting a PR. -### Adding New Unit Tests +### Adding New Unit Tests # TODO: make new unit test and test instructions To create a unit test, search for or create a file that ends with `_test.go` in the package of the file that requires testing, such as `auth.go` -> `auth_test.go`. Import the testing library and create test functions as necessary. Generic unit test helper functions can be found in [src/test/testutil](https://github.com/zarf-dev/zarf/tree/main/src/test/testutil). diff --git a/src/test/e2e/20_zarf_init_test.go b/src/test/e2e/20_zarf_init_test.go index 493223ab8f..ce8cc4087e 100644 --- a/src/test/e2e/20_zarf_init_test.go +++ b/src/test/e2e/20_zarf_init_test.go @@ -19,12 +19,6 @@ import ( func TestZarfInit(t *testing.T) { t.Log("E2E: Zarf init") - // // Verify cluster connectivity before running with-cluster tests - // if !e2e.ApplianceMode { - // _, _, err := e2e.Kubectl(t, "cluster-info") - // require.NoError(t, err, "No cluster found. Ensure a valid kubeconfig is available and cluster is running.") - // } - initComponents := "git-server" if e2e.ApplianceMode { initComponents = "k3s,git-server" diff --git a/src/test/e2e/main_test.go b/src/test/e2e/main_test.go index 13b0f33e84..4d085b3ba2 100644 --- a/src/test/e2e/main_test.go +++ b/src/test/e2e/main_test.go @@ -5,13 +5,16 @@ package test import ( - // "context" + "context" "log" "os" "path/filepath" + "runtime" "testing" "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/pkg/logger" + "github.com/zarf-dev/zarf/src/pkg/utils/exec" "github.com/zarf-dev/zarf/src/test" // "github.com/zarf-dev/zarf/src/pkg/utils/exec" ) @@ -48,12 +51,44 @@ func TestMain(m *testing.M) { log.Fatalf("zarf binary %s not found: %v", e2e.ZarfBinPath, err) } - // // If running with-cluster tests validate cluster connectivity early - // if os.Getenv(skipK8sEnvVar) != "true" && !e2e.ApplianceMode { - // _, _, err := exec.CmdWithContext(context.Background(), exec.Config{}, e2e.ZarfBinPath, "tools", "kubectl", "cluster-info") - // if err != nil { - // log.Fatalf("No cluster found. Ensure a valid kubeconfig is available and cluster is running: %v", err) - // } - // } + // If APPLIANCE_MODE set to true, warn Mac user and run only non-cluster tests + if e2e.ApplianceMode && runtime.GOOS != "linux" { + cfg := logger.Config{ + Level: logger.Info, + Format: logger.FormatConsole, + Destination: logger.DestinationDefault, + Color: true, + } + l, err := logger.New(cfg) + if err != nil { + log.Fatal(err) + } + l.Warn("APPLIANCE_MODE=true is only fully supported on Linux. " + + "On macOS, only tests that do not require a cluster will run. " + + "To run with-cluster tests on macOS, create a local cluster first " + + "and run make test-e2e-with-cluster ARCH=arm64 instead.") + } + + // If not in appliance mode, check for cluster connectivity + // If no cluster is found, warn that only without-cluster tests will run + if !e2e.ApplianceMode { + _, _, err := exec.CmdWithContext(context.Background(), exec.Config{}, e2e.ZarfBinPath, "tools", "kubectl", "cluster-info") + if err != nil { + cfg := logger.Config{ + Level: logger.Warn, + Format: logger.FormatConsole, + Destination: logger.DestinationDefault, + Color: true, + } + l, logErr := logger.New(cfg) + if logErr != nil { + log.Fatal(logErr) + } + l.Warn("No cluster found - with-cluster tests will fail. " + + "To run only tests that do not require a cluster use: make test-e2e-without-cluster. " + + "To run with-cluster tests, ensure a valid cluster is running.") + } + } + os.Exit(m.Run()) }