From 7496a8074813c5cd424d06383a43704b10a3ff2d Mon Sep 17 00:00:00 2001 From: Dag Andersen Date: Sat, 15 Aug 2026 11:39:35 +0200 Subject: [PATCH 1/2] Feat | Add preinstall manifests folder --- .github/ISSUE_TEMPLATE/bug_report.md | 1 + cmd/main.go | 2 +- cmd/options.go | 7 ++ docs/getting-started/pre-installing-crds.md | 83 +++++++++++++++++++++ docs/how-it-works.md | 37 +++++++-- docs/options.md | 1 + mkdocs.yml | 1 + pkg/argocd/argocd.go | 8 +- pkg/argocd/utils.go | 67 ++++++++++++----- pkg/argocd/utils_test.go | 64 ++++++++++++++++ 10 files changed, 242 insertions(+), 29 deletions(-) create mode 100644 docs/getting-started/pre-installing-crds.md create mode 100644 pkg/argocd/utils_test.go diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index e28a6f9f..720b5556 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -56,6 +56,7 @@ spec: ✨ - base-branch: integration-test/branch-3/base ✨ - target-branch: integration-test/branch-3/target ✨ - secrets-folder: ./secrets +✨ - preinstall-folder: ./preinstall ✨ - output-folder: ./output ✨ - argocd-namespace: argocd ✨ - repo: dag-andersen/argocd-diff-preview diff --git a/cmd/main.go b/cmd/main.go index c28bbef8..7084171f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -216,7 +216,7 @@ func run(cfg *Config) error { var argocdInstallationDuration time.Duration if cfg.CreateCluster { // Install Argo CD - duration, err := argocd.Install(cfg.Debug, cfg.SecretsFolder) + duration, err := argocd.Install(cfg.Debug, cfg.PreinstallFolder, cfg.SecretsFolder) if err != nil { log.Error().Msgf("❌ Failed to install Argo CD") return err diff --git a/cmd/options.go b/cmd/options.go index 76d3a648..f3bf7187 100644 --- a/cmd/options.go +++ b/cmd/options.go @@ -54,6 +54,7 @@ var ( DefaultBaseBranch = "main" DefaultOutputFolder = "./output" DefaultSecretsFolder = "./secrets" + DefaultPreinstallFolder = "./preinstall" DefaultCluster = "auto" DefaultClusterName = "argocd-diff-preview" DefaultKindOptions = "" @@ -103,6 +104,7 @@ type RawOptions struct { RepoRegex string `mapstructure:"repo-regex"` OutputFolder string `mapstructure:"output-folder"` SecretsFolder string `mapstructure:"secrets-folder"` + PreinstallFolder string `mapstructure:"preinstall-folder"` CreateCluster bool `mapstructure:"create-cluster"` ClusterType string `mapstructure:"cluster"` ClusterName string `mapstructure:"cluster-name"` @@ -153,6 +155,7 @@ type Config struct { RepoSelector repository.Selector OutputFolder string SecretsFolder string + PreinstallFolder string CreateCluster bool ClusterName string KindOptions string @@ -252,6 +255,7 @@ func Parse() *Config { viper.SetDefault("base-branch", DefaultBaseBranch) viper.SetDefault("output-folder", DefaultOutputFolder) viper.SetDefault("secrets-folder", DefaultSecretsFolder) + viper.SetDefault("preinstall-folder", DefaultPreinstallFolder) viper.SetDefault("create-cluster", DefaultCreateCluster) viper.SetDefault("watch-if-no-watch-pattern-found", DefaultWatchIfNoWatchPatternFound) viper.SetDefault("ignore-invalid-watch-pattern", DefaultIgnoreInvalidWatchPattern) @@ -312,6 +316,7 @@ func Parse() *Config { // Folders rootCmd.Flags().StringP("output-folder", "o", DefaultOutputFolder, "Output folder where the diff will be saved") rootCmd.Flags().StringP("secrets-folder", "s", DefaultSecretsFolder, "Secrets folder where the secrets are read from") + rootCmd.Flags().String("preinstall-folder", DefaultPreinstallFolder, "Folder containing Kubernetes manifests to apply before Argo CD is installed") // Cluster related rootCmd.Flags().Bool("create-cluster", DefaultCreateCluster, "Create a new cluster if it doesn't exist") @@ -397,6 +402,7 @@ func (o *RawOptions) ToConfig() (*Config, error) { TargetBranch: o.TargetBranch, OutputFolder: o.OutputFolder, SecretsFolder: o.SecretsFolder, + PreinstallFolder: o.PreinstallFolder, CreateCluster: o.CreateCluster, ClusterName: o.ClusterName, KindOptions: o.KindOptions, @@ -702,6 +708,7 @@ func (o *Config) LogConfig() { log.Info().Msgf("✨ - base-branch: %s", o.BaseBranch) log.Info().Msgf("✨ - target-branch: %s", o.TargetBranch) log.Info().Msgf("✨ - secrets-folder: %s", o.SecretsFolder) + log.Info().Msgf("✨ - preinstall-folder: %s", o.PreinstallFolder) log.Info().Msgf("✨ - output-folder: %s", o.OutputFolder) log.Info().Msgf("✨ - argocd-namespace: %s", o.ArgocdNamespace) if o.ArgocdConfigPath != DefaultArgocdConfigPath { diff --git a/docs/getting-started/pre-installing-crds.md b/docs/getting-started/pre-installing-crds.md new file mode 100644 index 00000000..3a423a76 --- /dev/null +++ b/docs/getting-started/pre-installing-crds.md @@ -0,0 +1,83 @@ +# Pre-installing CRDs + +Some applications need certain Custom Resource Definitions (CRDs) to be installed before a proper render can happen. The ephemeral cluster created by `argocd-diff-preview` only contains the Kubernetes APIs provided by the cluster itself and the resources installed with Argo CD. It does not automatically contain the CRDs from your destination cluster. + +Use the preinstall folder to apply CRDs and other cluster prerequisites before Argo CD is installed and Applications are rendered. + +## Prepare the preinstall folder + +Create a directory and place the required CRD manifests in it: + +```bash +mkdir -p preinstall +cp external-secrets-crd.yaml preinstall/ +``` + +Every file directly inside the directory is applied in filename order. Files may contain multiple YAML documents. Subdirectories are not traversed. + +If resources depend on one another, use filename prefixes to control their order: + +```text +preinstall/ +├── 00-external-secrets-crd.yaml +├── 10-cert-manager-crds.yaml +└── 20-other-prerequisites.yaml +``` + +## Docker + +Mount the directory at `/preinstall`: + +```bash +docker run \ + --network=host \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v $(pwd)/main:/base-branch \ + -v $(pwd)/pull-request:/target-branch \ + -v $(pwd)/preinstall:/preinstall \ + -v $(pwd)/output:/output \ + -e TARGET_BRANCH=refs/pull/123/merge \ + -e REPO=example/repository \ + dagandersen/argocd-diff-preview:latest +``` + +The default preinstall path inside the container is `/preinstall`, so no additional option is required. + +## Standalone binary + +The standalone binary reads manifests from `./preinstall` by default: + +```bash +argocd-diff-preview \ + --repo example/repository \ + --base-branch main \ + --target-branch feature/my-change +``` + +Use `--preinstall-folder` to select a different directory: + +```bash +argocd-diff-preview \ + --repo example/repository \ + --base-branch main \ + --target-branch feature/my-change \ + --preinstall-folder ./cluster-prerequisites +``` + +The equivalent environment variable is `PREINSTALL_FOLDER`. + +## Why install CRDs? + +Installing the CRDs makes their APIs available to rendering and Kubernetes discovery in the ephemeral cluster. This is useful when: + +- A Helm chart checks `.Capabilities.APIVersions` before rendering custom resources. +- Custom resources use the same name in different namespaces. +- The tool needs to determine whether a custom resource is namespaced or cluster-scoped. +- Other resources must exist before Argo CD or an Application can be rendered correctly. + +Without an installed CRD, an unknown custom resource is treated as namespaced. This safe default prevents namespaces from being removed and same-named resources from being silently combined. Pre-installing the CRD still provides the most accurate result because the tool can discover its actual scope. + +!!! note + Preinstall manifests are applied only to the ephemeral preview cluster. They are not synced as part of an Argo CD Application and do not modify the destination cluster. + +Keep repository credentials and other Argo CD secrets in the separate `/secrets` folder. The preinstall folder is intended for CRDs and other cluster prerequisites. diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 78b1e399..1d7beebb 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -66,8 +66,8 @@ For each Application or ApplicationSet found, it applies the following modificat --- -!!! note "Steps 4-6: Ephemeral Cluster Mode Only" - Steps 4, 5, and 6 only apply when using the default ephemeral cluster mode. If you're connecting to a pre-installed Argo CD instance with `--create-cluster=false`, these steps are skipped and the tool connects directly to your pre-installed Argo CD. +!!! note "Steps 4-7: Ephemeral Cluster Mode Only" + Steps 4, 5, 6, and 7 only apply when using the default ephemeral cluster mode. If you're connecting to a pre-installed Argo CD instance with `--create-cluster=false`, these steps are skipped and the tool connects directly to your pre-installed Argo CD. --- @@ -162,7 +162,28 @@ kubectl apply -f /secrets --- -## Step 7: Generate Applications from ApplicationSets +## Step 7: Apply preinstall manifests + +Some applications need certain Custom Resource Definitions (CRDs) to be installed before a proper render can happen. The ephemeral cluster does not automatically contain the CRDs from your destination cluster. + +Kubernetes manifests in the folder configured by `--preinstall-folder` are applied before Argo CD is installed and before Applications are rendered. The default is `./preinstall` for the standalone binary and `/preinstall` in the container. + +Installing the required CRDs makes their APIs available to rendering and lets the tool discover whether each custom resource is namespaced or cluster-scoped. The folder can also contain other cluster prerequisites needed during rendering. + +See [Pre-installing CRDs](./getting-started/pre-installing-crds.md) for setup instructions, Docker and binary examples, and details about file ordering. + +`argocd-diff-preview` practically performs: + +```bash +kubectl apply -f /preinstall +``` + +!!! note "Actual execution order" + Preinstall manifests are applied before the credentials and Argo CD installation steps described above. This section is placed here to keep the related folder-based setup together. + +--- + +## Step 8: Generate Applications from ApplicationSets For each ApplicationSet found, it generates the applications using the Argo CD CLI (or API depending on the `--render-method`): @@ -176,7 +197,7 @@ The newly generated applications also go through `Step 2` and `Step 3` (filterin --- -## Step 8: Apply applications to the cluster +## Step 9: Apply applications to the cluster The patched applications are applied to the cluster. @@ -190,7 +211,7 @@ At this point, Argo CD starts processing each application - rendering the applic --- -## Step 9: Wait for Argo CD to render +## Step 10: Wait for Argo CD to render The tool will repeatedly check the status of each application and extract the rendered manifests as they become ready. @@ -204,7 +225,7 @@ It practically just waits for the Application to look like this: --- -## Step 10: Extract rendered manifests +## Step 11: Extract rendered manifests Once applications are ready, the tool extracts the rendered manifests using the Argo CD CLI (or API depending on the `--render-method`): @@ -216,7 +237,7 @@ This returns the exact YAML generated by the applications - fully rendered with --- -## Step 11: Generate the diff +## Step 12: Generate the diff

@@ -233,7 +254,7 @@ The diff respects the `--diff-ignore` option to filter out noisy changes. Docume --- -## Step 12: Output the results +## Step 13: Output the results The tool writes several files to the output folder (`./output/` by default): diff --git a/docs/options.md b/docs/options.md index a44f5897..e9db0c66 100644 --- a/docs/options.md +++ b/docs/options.md @@ -63,6 +63,7 @@ argocd-diff-preview [FLAGS] [OPTIONS] (--repo | --repo-regex ) --t | `--log-format ` | `LOG_FORMAT` | `human` | Log format. Options: `human`, `json` | | `--max-diff-length ` | `MAX_DIFF_LENGTH` | `65536` | Max diff message character count (only limits the generated Markdown file) | | `--output-folder `, `-o` | `OUTPUT_FOLDER` | `./output` | Output folder where the diff will be saved | +| `--preinstall-folder ` | `PREINSTALL_FOLDER` | `./preinstall` | Folder containing Kubernetes manifests to apply before Argo CD is installed | | `--redirect-target-revisions ` | `REDIRECT_TARGET_REVISIONS` | - | Comma-separated source targetRevision values to redirect to the target branch. Example: main,HEAD. By default, every targetRevision in matching repositories is redirected | | `--render-method ` | `RENDER_METHOD` | `server-api` | Manifest rendering method. Options: `cli`, `server-api`, `repo-server-api` | | `--repo-regex ` | `REPO_REGEX` | - | Advanced repository matcher for templated Argo CD repoURL values. Mutually exclusive with `--repo` | diff --git a/mkdocs.yml b/mkdocs.yml index 3c56cfaf..85530aed 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,6 +28,7 @@ nav: - getting-started/gitlab-cicd.md - Local Installation: getting-started/installation.md - getting-started/custom-argo-cd-installation.md + - Pre-installing CRDs: getting-started/pre-installing-crds.md - Self-hosted Github Runners: getting-started/self-hosted-gh-runner.md - Reusing clusters: - Connect to cluster with Argo CD pre-installed: reusing-clusters/connecting.md diff --git a/pkg/argocd/argocd.go b/pkg/argocd/argocd.go index ec9eec89..4b477b54 100644 --- a/pkg/argocd/argocd.go +++ b/pkg/argocd/argocd.go @@ -80,7 +80,7 @@ func (a *ArgoCDInstallation) RenderMethod() vars.RenderMethod { return a.renderMode } -func (a *ArgoCDInstallation) Install(debug bool, secretsFolder string) (time.Duration, error) { +func (a *ArgoCDInstallation) Install(debug bool, preinstallFolder string, secretsFolder string) (time.Duration, error) { startTime := time.Now() log.Debug().Msgf("Creating namespace: %s", a.Namespace) @@ -97,6 +97,12 @@ func (a *ArgoCDInstallation) Install(debug bool, secretsFolder string) (time.Dur log.Debug().Msgf("Namespace already exists: %s", a.Namespace) } + // Apply cluster prerequisites before installing Argo CD. This allows users + // to install CRDs and other resources needed during application rendering. + if err := ApplyPreinstallFromFolder(a.K8sClient, preinstallFolder, a.Namespace); err != nil { + return time.Since(startTime), fmt.Errorf("failed to apply manifests from preinstall folder %s: %w", preinstallFolder, err) + } + // Apply secrets before installing ArgoCD if err := ApplySecretsFromFolder(a.K8sClient, secretsFolder, a.Namespace); err != nil { return time.Since(startTime), fmt.Errorf("failed to apply secrets from folder: %s: %w", secretsFolder, err) diff --git a/pkg/argocd/utils.go b/pkg/argocd/utils.go index 063ceb73..9219a1e9 100644 --- a/pkg/argocd/utils.go +++ b/pkg/argocd/utils.go @@ -9,39 +9,68 @@ import ( "github.com/rs/zerolog/log" ) -// ApplySecretsFromFolder applies all secret manifests from a folder using the Kubernetes API +// ApplyPreinstallFromFolder applies Kubernetes manifests needed by the ephemeral +// cluster before Argo CD is installed and applications are rendered. +func ApplyPreinstallFromFolder(client *k8s.Client, preinstallFolder string, namespace string) error { + count, found, err := applyManifestsFromFolder(preinstallFolder, "preinstall", func(path string) (int, error) { + return client.ApplyManifestFromFile(path, namespace) + }) + if err != nil { + return err + } + if !found { + log.Info().Msgf("🤷 No preinstall folder found at %s", preinstallFolder) + } else if count > 0 { + log.Info().Msgf("📦 Applied %d preinstall manifests", count) + } else { + log.Info().Msgf("🤷 No preinstall manifests found in %s", preinstallFolder) + } + return nil +} + +// ApplySecretsFromFolder applies all secret manifests from a folder using the Kubernetes API. func ApplySecretsFromFolder(client *k8s.Client, secretsFolder string, namespace string) error { - // Check if folder exists - if _, err := os.Stat(secretsFolder); os.IsNotExist(err) { + count, found, err := applyManifestsFromFolder(secretsFolder, "secret", func(path string) (int, error) { + return client.ApplyManifestFromFile(path, namespace) + }) + if err != nil { + return err + } + if !found { log.Info().Msgf("🤷 No secrets folder found at %s", secretsFolder) - return nil + } else if count > 0 { + log.Info().Msgf("🤫 Applied %d secrets", count) + } else { + log.Info().Msgf("🤷 No secrets found in %s", secretsFolder) + } + return nil +} + +func applyManifestsFromFolder(folder string, manifestType string, apply func(path string) (int, error)) (int, bool, error) { + if _, err := os.Stat(folder); err != nil { + if os.IsNotExist(err) { + return 0, false, nil + } + return 0, false, fmt.Errorf("failed to access %s folder: %w", manifestType, err) } - // Apply all files in the secrets folder - files, err := os.ReadDir(secretsFolder) + files, err := os.ReadDir(folder) if err != nil { - return fmt.Errorf("failed to read secrets folder: %w", err) + return 0, true, fmt.Errorf("failed to read %s folder: %w", manifestType, err) } - secretCount := 0 + manifestCount := 0 for _, file := range files { if file.IsDir() { continue } - // Use the existing ApplyManifestFromFile method to apply each secret - count, err := client.ApplyManifestFromFile(filepath.Join(secretsFolder, file.Name()), namespace) + count, err := apply(filepath.Join(folder, file.Name())) if err != nil { - return fmt.Errorf("failed to apply secret %s: %w", file.Name(), err) + return manifestCount, true, fmt.Errorf("failed to apply %s %s: %w", manifestType, file.Name(), err) } - secretCount += count + manifestCount += count } - if secretCount > 0 { - log.Info().Msgf("🤫 Applied %d secrets", secretCount) - } else { - log.Info().Msgf("🤷 No secrets found in %s", secretsFolder) - } - - return nil + return manifestCount, true, nil } diff --git a/pkg/argocd/utils_test.go b/pkg/argocd/utils_test.go new file mode 100644 index 00000000..09a9c65d --- /dev/null +++ b/pkg/argocd/utils_test.go @@ -0,0 +1,64 @@ +package argocd + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestApplyManifestsFromFolder(t *testing.T) { + t.Run("missing folder is a no-op", func(t *testing.T) { + count, found, err := applyManifestsFromFolder(filepath.Join(t.TempDir(), "missing"), "preinstall", func(string) (int, error) { + t.Fatal("apply must not be called") + return 0, nil + }) + + require.NoError(t, err) + assert.False(t, found) + assert.Zero(t, count) + }) + + t.Run("applies direct files in filename order and skips directories", func(t *testing.T) { + folder := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(folder, "20-second.yaml"), []byte("second"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(folder, "10-first.yaml"), []byte("first"), 0o600)) + require.NoError(t, os.Mkdir(filepath.Join(folder, "15-skipped"), 0o700)) + + var applied []string + count, found, err := applyManifestsFromFolder(folder, "preinstall", func(path string) (int, error) { + applied = append(applied, filepath.Base(path)) + return 2, nil + }) + + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, 4, count) + assert.Equal(t, []string{"10-first.yaml", "20-second.yaml"}, applied) + }) + + t.Run("stops after the first apply error", func(t *testing.T) { + folder := t.TempDir() + for _, name := range []string{"10-first.yaml", "20-fails.yaml", "30-skipped.yaml"} { + require.NoError(t, os.WriteFile(filepath.Join(folder, name), []byte(name), 0o600)) + } + + var applied []string + count, found, err := applyManifestsFromFolder(folder, "preinstall", func(path string) (int, error) { + name := filepath.Base(path) + applied = append(applied, name) + if name == "20-fails.yaml" { + return 0, fmt.Errorf("apply failed") + } + return 1, nil + }) + + assert.ErrorContains(t, err, "failed to apply preinstall 20-fails.yaml") + assert.True(t, found) + assert.Equal(t, 1, count) + assert.Equal(t, []string{"10-first.yaml", "20-fails.yaml"}, applied) + }) +} From 92b52040aa1486fa6c77fed133e9aeaa8e3a33e1 Mon Sep 17 00:00:00 2001 From: Dag Andersen Date: Sun, 23 Aug 2026 12:23:15 +0200 Subject: [PATCH 2/2] Cleanup --- docs/getting-started/pre-installing-crds.md | 5 ----- docs/how-it-works.md | 2 -- pkg/argocd/utils.go | 7 +++---- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/docs/getting-started/pre-installing-crds.md b/docs/getting-started/pre-installing-crds.md index 3a423a76..93582517 100644 --- a/docs/getting-started/pre-installing-crds.md +++ b/docs/getting-started/pre-installing-crds.md @@ -76,8 +76,3 @@ Installing the CRDs makes their APIs available to rendering and Kubernetes disco - Other resources must exist before Argo CD or an Application can be rendered correctly. Without an installed CRD, an unknown custom resource is treated as namespaced. This safe default prevents namespaces from being removed and same-named resources from being silently combined. Pre-installing the CRD still provides the most accurate result because the tool can discover its actual scope. - -!!! note - Preinstall manifests are applied only to the ephemeral preview cluster. They are not synced as part of an Argo CD Application and do not modify the destination cluster. - -Keep repository credentials and other Argo CD secrets in the separate `/secrets` folder. The preinstall folder is intended for CRDs and other cluster prerequisites. diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 1d7beebb..931b01ac 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -178,8 +178,6 @@ See [Pre-installing CRDs](./getting-started/pre-installing-crds.md) for setup in kubectl apply -f /preinstall ``` -!!! note "Actual execution order" - Preinstall manifests are applied before the credentials and Argo CD installation steps described above. This section is placed here to keep the related folder-based setup together. --- diff --git a/pkg/argocd/utils.go b/pkg/argocd/utils.go index 9219a1e9..30d90469 100644 --- a/pkg/argocd/utils.go +++ b/pkg/argocd/utils.go @@ -18,11 +18,9 @@ func ApplyPreinstallFromFolder(client *k8s.Client, preinstallFolder string, name if err != nil { return err } - if !found { - log.Info().Msgf("🤷 No preinstall folder found at %s", preinstallFolder) - } else if count > 0 { + if count > 0 { log.Info().Msgf("📦 Applied %d preinstall manifests", count) - } else { + } else if found { log.Info().Msgf("🤷 No preinstall manifests found in %s", preinstallFolder) } return nil @@ -46,6 +44,7 @@ func ApplySecretsFromFolder(client *k8s.Client, secretsFolder string, namespace return nil } +// returns the number of manifests applied, whether the folder was found, and an error if any. func applyManifestsFromFolder(folder string, manifestType string, apply func(path string) (int, error)) (int, bool, error) { if _, err := os.Stat(folder); err != nil { if os.IsNotExist(err) {