From da69702791d24a89bb94675894fecac7085624db Mon Sep 17 00:00:00 2001 From: Alexis-Maurer Fortin Date: Fri, 10 Jul 2026 11:40:30 -0400 Subject: [PATCH] qol changes to be able to set an existing file index and better manage default probes and detectors --- cmd/bagel/scan.go | 118 ++---------------- pkg/collector/collector.go | 20 ++- pkg/collector/collector_test.go | 34 +++++ pkg/config/config.go | 211 +++++++++++++++++--------------- pkg/config/defaults_test.go | 33 +++++ pkg/detector/defaults.go | 28 +++++ pkg/detector/defaults_test.go | 46 +++++++ pkg/probe/defaults.go | 35 ++++++ pkg/probe/defaults_test.go | 66 ++++++++++ 9 files changed, 380 insertions(+), 211 deletions(-) create mode 100644 pkg/config/defaults_test.go create mode 100644 pkg/detector/defaults.go create mode 100644 pkg/detector/defaults_test.go create mode 100644 pkg/probe/defaults.go create mode 100644 pkg/probe/defaults_test.go diff --git a/cmd/bagel/scan.go b/cmd/bagel/scan.go index a9eb41c..84d80d0 100644 --- a/cmd/bagel/scan.go +++ b/cmd/bagel/scan.go @@ -115,117 +115,17 @@ func runScan(cmd *cobra.Command, args []string) error { return nil } -// initializeProbes creates and configures all probes +// initializeProbes builds the default probe set and keeps only the enabled ones. +// The collector also filters on IsEnabled at execution time, but filtering here +// preserves the historical probe slice (and probe_count log) exactly. func initializeProbes(cfg *models.Config) []probe.Probe { - var probes []probe.Probe - - // Create detector registry and register all secret detectors - registry := detector.NewRegistry() - registry.Register(detector.NewGitHubPATDetector()) - registry.Register(detector.NewNPMTokenDetector()) - registry.Register(detector.NewSSHPrivateKeyDetector()) - registry.Register(detector.NewAIServiceDetector()) - registry.Register(detector.NewHTTPAuthDetector()) - registry.Register(detector.NewCloudCredentialsDetector()) - registry.Register(detector.NewVaultTokenDetector()) - registry.Register(detector.NewPyPITokenDetector()) - registry.Register(detector.NewWireGuardKeyDetector()) - registry.Register(detector.NewSplunkTokenDetector()) - registry.Register(detector.NewDatabaseConnectionDetector()) - registry.Register(detector.NewSlackTokenDetector()) - registry.Register(detector.NewStripeKeyDetector()) - registry.Register(detector.NewTwilioKeyDetector()) - registry.Register(detector.NewGenericAPIKeyDetector()) - registry.Register(detector.NewJWTDetector()) - // Add more detectors here as they are implemented: - // registry.Register(detector.NewSlackTokenDetector()) - // etc. - - // Git probe - if cfg.Probes.Git.Enabled { - probes = append(probes, probe.NewGitProbe(cfg.Probes.Git, registry)) - } - - // Environment variable probe - if cfg.Probes.Env.Enabled { - probes = append(probes, probe.NewEnvProbe(cfg.Probes.Env, registry)) - } - - // NPM probe - if cfg.Probes.NPM.Enabled { - probes = append(probes, probe.NewNPMProbe(cfg.Probes.NPM, registry)) - } - - // SSH probe - if cfg.Probes.SSH.Enabled { - probes = append(probes, probe.NewSSHProbe(cfg.Probes.SSH, registry)) - } - - // Shell history probe - if cfg.Probes.ShellHistory.Enabled { - probes = append(probes, probe.NewShellHistoryProbe(cfg.Probes.ShellHistory, registry)) - } - - // Cloud credentials probe - if cfg.Probes.Cloud.Enabled { - probes = append(probes, probe.NewCloudProbe(cfg.Probes.Cloud, registry)) - } - - // JetBrains probe - if cfg.Probes.JetBrains.Enabled { - probes = append(probes, probe.NewJetBrainsProbe(cfg.Probes.JetBrains, registry)) - } - - // GitHub CLI probe - if cfg.Probes.GH.Enabled { - probes = append(probes, probe.NewGHProbe(cfg.Probes.GH, registry)) - } - - // AI credentials probe (auth/oauth files; scan only — scrub leaves these alone) - if cfg.Probes.AICredentials.Enabled { - probes = append(probes, probe.NewAICredentialsProbe(cfg.Probes.AICredentials, registry)) - } - - // AI chats probe (conversation history) - if cfg.Probes.AIChats.Enabled { - probes = append(probes, probe.NewAIChatsProbe(cfg.Probes.AIChats, registry)) - } - - // WireGuard probe - if cfg.Probes.WireGuard.Enabled { - probes = append(probes, probe.NewWireGuardProbe(cfg.Probes.WireGuard, registry)) - } - - // PyPI probe - if cfg.Probes.PyPI.Enabled { - probes = append(probes, probe.NewPyPIProbe(cfg.Probes.PyPI, registry)) - } - - // Kubernetes probe — credential extraction from kubeconfig - if cfg.Probes.Kube.Enabled { - probes = append(probes, probe.NewKubeProbe(cfg.Probes.Kube, registry)) - } + registry := detector.NewDefaultRegistry() - // Docker/Podman probe — inline registry credentials - if cfg.Probes.Docker.Enabled { - probes = append(probes, probe.NewDockerProbe(cfg.Probes.Docker, registry)) - } - - // Infrastructure-as-Code probe — Terraform + Helm credential discovery - if cfg.Probes.IaC.Enabled { - probes = append(probes, probe.NewIaCProbe(cfg.Probes.IaC, registry)) - } - - // AI MCP probe — credentials in MCP server configs (scan-only; - // scrub would break agents by replacing real tokens with markers). - if cfg.Probes.AIMCP.Enabled { - probes = append(probes, probe.NewMCPProbe(cfg.Probes.AIMCP, registry)) - } - - // AI context/memory probe — CLAUDE.md / AGENTS.md scanning. - if cfg.Probes.AIContext.Enabled { - probes = append(probes, probe.NewContextProbe(cfg.Probes.AIContext, registry)) + var probes []probe.Probe + for _, p := range probe.DefaultProbes(cfg, registry) { + if p.IsEnabled() { + probes = append(probes, p) + } } - return probes } diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index 0d91d0b..1fbd0b8 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -29,7 +29,8 @@ type Collector struct { config *models.Config noCache bool noProgress bool - cacheStore *cache.Store // Reused across load/save operations + cacheStore *cache.Store // Reused across load/save operations + fileIndex *fileindex.FileIndex // Prebuilt index; when non-nil, Collect skips building/caching its own } // NewInput holds the parameters for creating a new Collector @@ -38,6 +39,10 @@ type NewInput struct { Config *models.Config NoCache bool NoProgress bool + // FileIndex, when non-nil, is used as-is; Collect skips building and + // caching its own index. Lets a caller own the crawl (e.g. share one + // index across bagel's probes and its own). + FileIndex *fileindex.FileIndex } // New creates a new Collector @@ -53,6 +58,7 @@ func New(input NewInput) *Collector { noCache: input.NoCache, noProgress: input.NoProgress, cacheStore: store, + fileIndex: input.FileIndex, } } @@ -74,10 +80,14 @@ func (c *Collector) Collect(ctx context.Context) (*models.ScanResult, error) { return nil, fmt.Errorf("failed to get host info: %w", err) } - // Build file index if enabled - fileIdx, err := c.buildFileIndex(ctx) - if err != nil { - return nil, fmt.Errorf("failed to build file index: %w", err) + // Use a caller-supplied index as-is; otherwise build (and cache) our own. + fileIdx := c.fileIndex + if fileIdx == nil { + var err error + fileIdx, err = c.buildFileIndex(ctx) + if err != nil { + return nil, fmt.Errorf("failed to build file index: %w", err) + } } // Compute fingerprint salt from host identity and propagate to probes diff --git a/pkg/collector/collector_test.go b/pkg/collector/collector_test.go index f7dd9c6..ef779c7 100644 --- a/pkg/collector/collector_test.go +++ b/pkg/collector/collector_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/boostsecurityio/bagel/pkg/fileindex" "github.com/boostsecurityio/bagel/pkg/models" "github.com/boostsecurityio/bagel/pkg/probe" "github.com/stretchr/testify/assert" @@ -159,6 +160,39 @@ func TestCollect_ProbeErrors(t *testing.T) { assert.Equal(t, "finding1", result.Findings[0].ID, "expected finding1") } +// fileIndexProbe records the file index handed to it via SetFileIndex. +type fileIndexProbe struct { + mockProbe + got *fileindex.FileIndex +} + +func (p *fileIndexProbe) SetFileIndex(idx *fileindex.FileIndex) { p.got = idx } + +// TestCollect_InjectedFileIndex confirms a caller-supplied index is used as-is: +// it reaches FileIndexAware probes and Collect never builds its own (the probe +// receives the exact pointer we injected, which a fresh build could not produce). +func TestCollect_InjectedFileIndex(t *testing.T) { + t.Parallel() + + injected, err := fileindex.BuildIndex(context.TODO(), fileindex.BuildIndexInput{ + BaseDirs: []string{t.TempDir()}, + }) + require.NoError(t, err) + + prb := &fileIndexProbe{mockProbe: mockProbe{name: "fi", enabled: true}} + + // NoCache:false would normally build + touch the cache; injection must skip both. + col := New(NewInput{ + Probes: []probe.Probe{prb}, + Config: &models.Config{Version: 1}, + FileIndex: injected, + }) + + _, err = col.Collect(context.TODO()) + require.NoError(t, err) + assert.Same(t, injected, prb.got, "probe should receive the injected index, not a freshly built one") +} + func TestExecuteProbes_ConcurrentExecution(t *testing.T) { t.Parallel() diff --git a/pkg/config/config.go b/pkg/config/config.go index 09152d7..9d5c35d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -169,40 +169,57 @@ func setDefaults(v *viper.Viper) { v.SetDefault("resources.max_concurrent_probes", 0) v.SetDefault("resources.probe_timeout", "30s") - // Common dotfiles and config files - v.SetDefault("file_index.patterns", []map[string]interface{}{ + // Common dotfiles and config files. Single source of truth is + // DefaultPatterns; convert to viper's map shape here. + defaults := DefaultPatterns() + rawPatterns := make([]map[string]interface{}, len(defaults)) + for i, p := range defaults { + rawPatterns[i] = map[string]interface{}{ + "name": p.Name, + "patterns": p.Patterns, + "type": p.Type, + } + } + v.SetDefault("file_index.patterns", rawPatterns) +} + +// DefaultPatterns returns the built-in file-index pattern groups bagel scans by +// default. Callers that want to extend (rather than replace) the defaults can +// append their own PatternConfig entries to the returned slice. +func DefaultPatterns() []models.PatternConfig { + return []models.PatternConfig{ // SSH - {"name": "ssh_config", "patterns": []string{".ssh/config"}, "type": "glob"}, - {"name": "ssh_known_hosts", "patterns": []string{".ssh/known_hosts"}, "type": "glob"}, - {"name": "ssh_keys", "patterns": []string{".ssh/id_*", ".ssh/*.pem"}, "type": "glob"}, - {"name": "ssh_authorized_keys", "patterns": []string{".ssh/authorized_keys"}, "type": "glob"}, + {Name: "ssh_config", Patterns: []string{".ssh/config"}, Type: "glob"}, + {Name: "ssh_known_hosts", Patterns: []string{".ssh/known_hosts"}, Type: "glob"}, + {Name: "ssh_keys", Patterns: []string{".ssh/id_*", ".ssh/*.pem"}, Type: "glob"}, + {Name: "ssh_authorized_keys", Patterns: []string{".ssh/authorized_keys"}, Type: "glob"}, // Git - {"name": "gitconfig", "patterns": []string{".gitconfig", ".config/git/config", ".git/config"}, "type": "glob"}, - {"name": "gitignore_global", "patterns": []string{".gitignore_global", ".config/git/ignore"}, "type": "glob"}, - {"name": "git_credentials", "patterns": []string{".git-credentials", ".config/git/credentials"}, "type": "glob"}, + {Name: "gitconfig", Patterns: []string{".gitconfig", ".config/git/config", ".git/config"}, Type: "glob"}, + {Name: "gitignore_global", Patterns: []string{".gitignore_global", ".config/git/ignore"}, Type: "glob"}, + {Name: "git_credentials", Patterns: []string{".git-credentials", ".config/git/credentials"}, Type: "glob"}, // NPM - {"name": "npmrc", "patterns": []string{".npmrc", ".config/npm/npmrc"}, "type": "glob"}, + {Name: "npmrc", Patterns: []string{".npmrc", ".config/npm/npmrc"}, Type: "glob"}, // Yarn - {"name": "yarnrc", "patterns": []string{".yarnrc", ".yarnrc.yml"}, "type": "glob"}, + {Name: "yarnrc", Patterns: []string{".yarnrc", ".yarnrc.yml"}, Type: "glob"}, // AWS - {"name": "aws_config", "patterns": []string{".aws/config"}, "type": "glob"}, - {"name": "aws_credentials", "patterns": []string{".aws/credentials"}, "type": "glob"}, - {"name": "aws_sso_cache", "patterns": []string{".aws/sso/cache/*.json"}, "type": "glob"}, - {"name": "aws_cli_cache", "patterns": []string{".aws/cli/cache/*.json"}, "type": "glob"}, + {Name: "aws_config", Patterns: []string{".aws/config"}, Type: "glob"}, + {Name: "aws_credentials", Patterns: []string{".aws/credentials"}, Type: "glob"}, + {Name: "aws_sso_cache", Patterns: []string{".aws/sso/cache/*.json"}, Type: "glob"}, + {Name: "aws_cli_cache", Patterns: []string{".aws/cli/cache/*.json"}, Type: "glob"}, // Google Cloud (GCP) - Unix: ~/.config/gcloud, Windows: %APPDATA%\gcloud - {"name": "gcp_config", "patterns": []string{ + {Name: "gcp_config", Patterns: []string{ ".config/gcloud/configurations/config_*", ".config/gcloud/properties", // Windows: %APPDATA%\gcloud "AppData/Roaming/gcloud/configurations/config_*", "AppData/Roaming/gcloud/properties", - }, "type": "glob"}, - {"name": "gcp_credentials", "patterns": []string{ + }, Type: "glob"}, + {Name: "gcp_credentials", Patterns: []string{ ".config/gcloud/credentials.db", ".config/gcloud/legacy_credentials/*", ".config/gcloud/application_default_credentials.json", @@ -214,10 +231,10 @@ func setDefaults(v *viper.Viper) { "AppData/Roaming/gcloud/application_default_credentials.json", "AppData/Roaming/gcloud/adc.json", "AppData/Roaming/gcloud/access_tokens.db", - }, "type": "glob"}, + }, Type: "glob"}, // Azure - Unix: ~/.azure, Windows: %USERPROFILE%\.azure or %APPDATA%\.azure - {"name": "azure_config", "patterns": []string{ + {Name: "azure_config", Patterns: []string{ ".azure/config", ".azure/clouds.config", ".azure/azureProfile.json", @@ -225,82 +242,82 @@ func setDefaults(v *viper.Viper) { "AppData/Roaming/.azure/config", "AppData/Roaming/.azure/clouds.config", "AppData/Roaming/.azure/azureProfile.json", - }, "type": "glob"}, - {"name": "azure_tokens", "patterns": []string{ + }, Type: "glob"}, + {Name: "azure_tokens", Patterns: []string{ ".azure/accessTokens.json", ".azure/msal_token_cache.*", ".azure/msazure.login/*", ".azure/azd/*", "AppData/Roaming/.azure/accessTokens.json", "AppData/Roaming/.azure/msal_token_cache.*", - }, "type": "glob"}, - {"name": "oci_config", "patterns": []string{ + }, Type: "glob"}, + {Name: "oci_config", Patterns: []string{ ".oci/config", ".oci/sessions/*", - }, "type": "glob"}, - {"name": "aliyun_config", "patterns": []string{".aliyun/config.json"}, "type": "glob"}, - {"name": "bluemix_config", "patterns": []string{".bluemix/config.json"}, "type": "glob"}, - {"name": "doctl_config", "patterns": []string{".config/doctl/config.yaml"}, "type": "glob"}, - {"name": "hcloud_config", "patterns": []string{".config/hcloud/cli.toml"}, "type": "glob"}, - {"name": "scw_config", "patterns": []string{".config/scw/config.yaml"}, "type": "glob"}, - {"name": "linode_config", "patterns": []string{".config/linode-cli/*"}, "type": "glob"}, - {"name": "fly_config", "patterns": []string{".fly/config.yml"}, "type": "glob"}, - {"name": "vercel_config", "patterns": []string{".vercel/auth.json"}, "type": "glob"}, - {"name": "railway_config", "patterns": []string{".railway/config.json"}, "type": "glob"}, - {"name": "snowflake_config", "patterns": []string{".snowflake/connections.toml"}, "type": "glob"}, - {"name": "doppler_config", "patterns": []string{".doppler.yaml"}, "type": "glob"}, - {"name": "gh_hosts", "patterns": []string{".config/gh/hosts.yml"}, "type": "glob"}, - {"name": "glab_config", "patterns": []string{".config/glab-cli/config.yml"}, "type": "glob"}, - {"name": "hub_config", "patterns": []string{".config/hub"}, "type": "glob"}, - {"name": "netrc_file", "patterns": []string{".netrc", "_netrc"}, "type": "glob"}, + }, Type: "glob"}, + {Name: "aliyun_config", Patterns: []string{".aliyun/config.json"}, Type: "glob"}, + {Name: "bluemix_config", Patterns: []string{".bluemix/config.json"}, Type: "glob"}, + {Name: "doctl_config", Patterns: []string{".config/doctl/config.yaml"}, Type: "glob"}, + {Name: "hcloud_config", Patterns: []string{".config/hcloud/cli.toml"}, Type: "glob"}, + {Name: "scw_config", Patterns: []string{".config/scw/config.yaml"}, Type: "glob"}, + {Name: "linode_config", Patterns: []string{".config/linode-cli/*"}, Type: "glob"}, + {Name: "fly_config", Patterns: []string{".fly/config.yml"}, Type: "glob"}, + {Name: "vercel_config", Patterns: []string{".vercel/auth.json"}, Type: "glob"}, + {Name: "railway_config", Patterns: []string{".railway/config.json"}, Type: "glob"}, + {Name: "snowflake_config", Patterns: []string{".snowflake/connections.toml"}, Type: "glob"}, + {Name: "doppler_config", Patterns: []string{".doppler.yaml"}, Type: "glob"}, + {Name: "gh_hosts", Patterns: []string{".config/gh/hosts.yml"}, Type: "glob"}, + {Name: "glab_config", Patterns: []string{".config/glab-cli/config.yml"}, Type: "glob"}, + {Name: "hub_config", Patterns: []string{".config/hub"}, Type: "glob"}, + {Name: "netrc_file", Patterns: []string{".netrc", "_netrc"}, Type: "glob"}, // Kiro IDE MCP — same shape as Claude Code's mcpServers; suffix // matching catches both user (~/.kiro/) and project (/.kiro/) forms. - {"name": "kiro_mcp", "patterns": []string{".kiro/settings/mcp.json"}, "type": "glob"}, + {Name: "kiro_mcp", Patterns: []string{".kiro/settings/mcp.json"}, Type: "glob"}, // Salesforce CLIs. .sf is the newer CLI's auth store; // .sfdx/auth/* is the legacy layout. Both hold OAuth refresh tokens. - {"name": "sf_config", "patterns": []string{".sf/*"}, "type": "glob"}, - {"name": "sfdx_config", "patterns": []string{".sfdx/*", ".sfdx/auth/*"}, "type": "glob"}, + {Name: "sf_config", Patterns: []string{".sf/*"}, Type: "glob"}, + {Name: "sfdx_config", Patterns: []string{".sfdx/*", ".sfdx/auth/*"}, Type: "glob"}, // Ansible — top-level files (galaxy_token, vault_password*). // The cp/ socket dir and tmp/ subdirs aren't credentials and // produce no findings on a registry pass. - {"name": "ansible_config", "patterns": []string{".ansible/*"}, "type": "glob"}, + {Name: "ansible_config", Patterns: []string{".ansible/*"}, Type: "glob"}, // Rails / WordPress DB config — project-level files holding // cleartext DB passwords. Suffix matching catches them at any // repo depth without anchoring to home root. - {"name": "rails_database_yml", "patterns": []string{"config/database.yml"}, "type": "glob"}, - {"name": "wp_config", "patterns": []string{"wp-config.php"}, "type": "glob"}, + {Name: "rails_database_yml", Patterns: []string{"config/database.yml"}, Type: "glob"}, + {Name: "wp_config", Patterns: []string{"wp-config.php"}, Type: "glob"}, // Docker - {"name": "docker_config", "patterns": []string{".docker/config.json"}, "type": "glob"}, + {Name: "docker_config", Patterns: []string{".docker/config.json"}, Type: "glob"}, // Podman / containers — same schema as docker config.json, different path. - {"name": "podman_config", "patterns": []string{".config/containers/auth.json"}, "type": "glob"}, + {Name: "podman_config", Patterns: []string{".config/containers/auth.json"}, Type: "glob"}, // Helm OCI registry auth — `helm registry login` writes a // docker-config-shaped JSON here. Same `auths{.auth}` // blob with base64(user:password) that DockerProbe already knows how to parse. - {"name": "helm_oci_registry", "patterns": []string{".config/helm/registry/config.json"}, "type": "glob"}, + {Name: "helm_oci_registry", Patterns: []string{".config/helm/registry/config.json"}, Type: "glob"}, // Docker context TLS material — client cert + key + CA for // connecting to a remote Docker daemon. Only the key.pem is a // secret; cert.pem and ca.pem start with `BEGIN CERTIFICATE` // which the SSH-private-key detector ignores by design. - {"name": "docker_context_keys", "patterns": []string{".docker/contexts/tls/*/*/*.pem"}, "type": "glob"}, + {Name: "docker_context_keys", Patterns: []string{".docker/contexts/tls/*/*/*.pem"}, Type: "glob"}, // Kubernetes - {"name": "kubeconfig", "patterns": []string{".kube/config"}, "type": "glob"}, + {Name: "kubeconfig", Patterns: []string{".kube/config"}, Type: "glob"}, // Shell configs - {"name": "bashrc", "patterns": []string{".bashrc", ".bash_profile", ".profile"}, "type": "glob"}, - {"name": "zshrc", "patterns": []string{".zshrc", ".zprofile"}, "type": "glob"}, + {Name: "bashrc", Patterns: []string{".bashrc", ".bash_profile", ".profile"}, Type: "glob"}, + {Name: "zshrc", Patterns: []string{".zshrc", ".zprofile"}, Type: "glob"}, // Shell history files - Unix shells and PowerShell (Windows). // Also covers DB and language-REPL input history. - {"name": "shell_history", "patterns": []string{ + {Name: "shell_history", Patterns: []string{ ".bash_history", ".zsh_history", ".sh_history", @@ -315,119 +332,119 @@ func setDefaults(v *viper.Viper) { ".python_history", ".node_repl_history", ".irb_history", - }, "type": "glob"}, + }, Type: "glob"}, // Environment files - {"name": "env_files", "patterns": []string{".env", ".env.*"}, "type": "glob"}, + {Name: "env_files", Patterns: []string{".env", ".env.*"}, Type: "glob"}, // JetBrains - {"name": "jetbrains", "patterns": []string{".idea/workspace.xml"}, "type": "glob"}, + {Name: "jetbrains", Patterns: []string{".idea/workspace.xml"}, Type: "glob"}, // AI tools - {"name": "gemini_credentials", "patterns": []string{".gemini/oauth_creds.json"}, "type": "glob"}, - {"name": "codex_credentials", "patterns": []string{".codex/auth.json"}, "type": "glob"}, - {"name": "opencode_credentials", "patterns": []string{".local/share/opencode/auth.json"}, "type": "glob"}, + {Name: "gemini_credentials", Patterns: []string{".gemini/oauth_creds.json"}, Type: "glob"}, + {Name: "codex_credentials", Patterns: []string{".codex/auth.json"}, Type: "glob"}, + {Name: "opencode_credentials", Patterns: []string{".local/share/opencode/auth.json"}, Type: "glob"}, - {"name": "gemini_chats", "patterns": []string{".gemini/tmp/*/chats/*.json"}, "type": "glob"}, - {"name": "codex_chats", "patterns": []string{".codex/sessions/*/*/*/rollout-*.jsonl"}, "type": "glob"}, - {"name": "claude_chats", "patterns": []string{".claude/projects/*/*.jsonl"}, "type": "glob"}, - {"name": "opencode_chats", "patterns": []string{".local/share/opencode/storage/part/msg_*/prt_*.json"}, "type": "glob"}, + {Name: "gemini_chats", Patterns: []string{".gemini/tmp/*/chats/*.json"}, Type: "glob"}, + {Name: "codex_chats", Patterns: []string{".codex/sessions/*/*/*/rollout-*.jsonl"}, Type: "glob"}, + {Name: "claude_chats", Patterns: []string{".claude/projects/*/*.jsonl"}, Type: "glob"}, + {Name: "opencode_chats", Patterns: []string{".local/share/opencode/storage/part/msg_*/prt_*.json"}, Type: "glob"}, // Additional AI agent history / paste / env surfaces beyond // session rollouts. Users routinely paste tokens into prompts; // the paste-cache is literally a record of those pastes. REPL // history files capture every prompt the user submitted at the // top level (separate from per-session rollouts). - {"name": "claude_repl_history", "patterns": []string{".claude/history.jsonl"}, "type": "glob"}, - {"name": "claude_paste_cache", "patterns": []string{".claude/paste-cache/*"}, "type": "glob"}, - {"name": "claude_session_env", "patterns": []string{".claude/session-env/*"}, "type": "glob"}, - {"name": "codex_repl_history", "patterns": []string{".codex/history.jsonl"}, "type": "glob"}, - {"name": "opencode_session_info", "patterns": []string{".local/share/opencode/storage/session/info/*.json"}, "type": "glob"}, - {"name": "opencode_session_message", "patterns": []string{".local/share/opencode/storage/session/message/*/*.json"}, "type": "glob"}, + {Name: "claude_repl_history", Patterns: []string{".claude/history.jsonl"}, Type: "glob"}, + {Name: "claude_paste_cache", Patterns: []string{".claude/paste-cache/*"}, Type: "glob"}, + {Name: "claude_session_env", Patterns: []string{".claude/session-env/*"}, Type: "glob"}, + {Name: "codex_repl_history", Patterns: []string{".codex/history.jsonl"}, Type: "glob"}, + {Name: "opencode_session_info", Patterns: []string{".local/share/opencode/storage/session/info/*.json"}, Type: "glob"}, + {Name: "opencode_session_message", Patterns: []string{".local/share/opencode/storage/session/message/*/*.json"}, Type: "glob"}, // AI agent MCP server configs. mcpServers blocks carry the env // map that holds API tokens for third-party services (GitHub // PATs, Slack tokens, etc.). claude.json is the application // state file; settings.{,local.}json may carry mcpServers too; // .mcp.json is a project-level MCP-only file. - {"name": "claude_app_state", "patterns": []string{".claude/claude.json"}, "type": "glob"}, - {"name": "claude_settings", "patterns": []string{ + {Name: "claude_app_state", Patterns: []string{".claude/claude.json"}, Type: "glob"}, + {Name: "claude_settings", Patterns: []string{ ".claude/settings.json", ".claude/settings.local.json", - }, "type": "glob"}, - {"name": "mcp_project_config", "patterns": []string{".mcp.json"}, "type": "glob"}, + }, Type: "glob"}, + {Name: "mcp_project_config", Patterns: []string{".mcp.json"}, Type: "glob"}, // AI agent context/memory files — pasted secrets get baked // into these by users. Basename match: catch them anywhere // under home (per-repo CLAUDE.md, global ~/.claude/CLAUDE.md, // codex/opencode AGENTS.md, etc.). - {"name": "ai_memory_md", "patterns": []string{ + {Name: "ai_memory_md", Patterns: []string{ "CLAUDE.md", "AGENTS.md", - }, "type": "glob"}, + }, Type: "glob"}, // Claude Code user-level customization. Commands, agents, and // skills are user-authored Markdown that Claude loads as // context — secrets in the prompt body get sent to the model // on every invocation. - {"name": "claude_commands", "patterns": []string{".claude/commands/*.md"}, "type": "glob"}, - {"name": "claude_agents", "patterns": []string{".claude/agents/*.md"}, "type": "glob"}, + {Name: "claude_commands", Patterns: []string{".claude/commands/*.md"}, Type: "glob"}, + {Name: "claude_agents", Patterns: []string{".claude/agents/*.md"}, Type: "glob"}, // Skills usually have SKILL.md at the skill root plus optional // sibling .md docs; the glob catches both shapes. - {"name": "claude_skills", "patterns": []string{".claude/skills/*/*.md"}, "type": "glob"}, + {Name: "claude_skills", Patterns: []string{".claude/skills/*/*.md"}, Type: "glob"}, // Cross-agent skill store (a number of plugins symlink into // ~/.agents/skills/). Worth scanning so the underlying files // surface even when symlinks aren't being followed. - {"name": "agents_skills", "patterns": []string{".agents/skills/*/*.md"}, "type": "glob"}, + {Name: "agents_skills", Patterns: []string{".agents/skills/*/*.md"}, Type: "glob"}, // Codex CLI context/memory. - {"name": "codex_instructions", "patterns": []string{".codex/instructions.md"}, "type": "glob"}, - {"name": "codex_memories", "patterns": []string{".codex/memories/*"}, "type": "glob"}, - {"name": "codex_skills", "patterns": []string{".codex/skills/*/*.md"}, "type": "glob"}, + {Name: "codex_instructions", Patterns: []string{".codex/instructions.md"}, Type: "glob"}, + {Name: "codex_memories", Patterns: []string{".codex/memories/*"}, Type: "glob"}, + {Name: "codex_skills", Patterns: []string{".codex/skills/*/*.md"}, Type: "glob"}, // WireGuard (user-level configs; system paths are checked directly by the probe) - {"name": "wireguard_config", "patterns": []string{".config/wireguard/*.conf"}, "type": "glob"}, + {Name: "wireguard_config", Patterns: []string{".config/wireguard/*.conf"}, Type: "glob"}, // HashiCorp Vault - {"name": "vault_token", "patterns": []string{".vault-token"}, "type": "glob"}, + {Name: "vault_token", Patterns: []string{".vault-token"}, Type: "glob"}, // PyPI - {"name": "pypirc", "patterns": []string{".pypirc"}, "type": "glob"}, - {"name": "pip_config", "patterns": []string{ + {Name: "pypirc", Patterns: []string{".pypirc"}, Type: "glob"}, + {Name: "pip_config", Patterns: []string{ ".pip/pip.conf", ".config/pip/pip.conf", // macOS "Library/Application Support/pip/pip.conf", // Windows "AppData/Roaming/pip/pip.ini", - }, "type": "glob"}, + }, Type: "glob"}, // Terraform — credentials live in either path. The JSON form is // authoritative for `terraform login`; the legacy HCL form is // still common from manual setups. - {"name": "terraform_credentials", "patterns": []string{ + {Name: "terraform_credentials", Patterns: []string{ ".terraform.d/credentials.tfrc.json", ".terraformrc", - }, "type": "glob"}, + }, Type: "glob"}, // Terraform variable / state files. tfvars commonly hold cloud // creds and DB passwords; local-backend state serializes resource // outputs (including sensitive ones) as plaintext JSON. - {"name": "terraform_vars", "patterns": []string{ + {Name: "terraform_vars", Patterns: []string{ "*.tfvars", "*.auto.tfvars", - }, "type": "glob"}, - {"name": "terraform_state", "patterns": []string{ + }, Type: "glob"}, + {Name: "terraform_state", Patterns: []string{ "terraform.tfstate", "terraform.tfstate.backup", - }, "type": "glob"}, + }, Type: "glob"}, // Helm — username/password live under repositories[] in this file. - {"name": "helm_repositories", "patterns": []string{ + {Name: "helm_repositories", Patterns: []string{ ".config/helm/repositories.yaml", // macOS "Library/Preferences/helm/repositories.yaml", - }, "type": "glob"}, - }) + }, Type: "glob"}, + } } // GetConfigDir returns the platform-appropriate configuration directory for bagel. diff --git a/pkg/config/defaults_test.go b/pkg/config/defaults_test.go new file mode 100644 index 0000000..ae978ea --- /dev/null +++ b/pkg/config/defaults_test.go @@ -0,0 +1,33 @@ +// Copyright (C) 2026 boostsecurity.io +// SPDX-License-Identifier: GPL-3.0-or-later + +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDefaultPatterns_MatchesLoadedConfig verifies DefaultPatterns is the single +// source of truth: the patterns a default Load produces must match it exactly, +// by name and order. +func TestDefaultPatterns_MatchesLoadedConfig(t *testing.T) { + t.Parallel() + + defaults := DefaultPatterns() + require.NotEmpty(t, defaults) + + cfg, err := Load("") + require.NoError(t, err) + require.Len(t, cfg.FileIndex.Patterns, len(defaults), + "loaded pattern count must match DefaultPatterns") + + for i, p := range defaults { + loaded := cfg.FileIndex.Patterns[i] + assert.Equal(t, p.Name, loaded.Name, "pattern name mismatch at index %d", i) + assert.Equal(t, p.Type, loaded.Type, "pattern type mismatch for %q", p.Name) + assert.Equal(t, p.Patterns, loaded.Patterns, "globs mismatch for %q", p.Name) + } +} diff --git a/pkg/detector/defaults.go b/pkg/detector/defaults.go new file mode 100644 index 0000000..12b51fd --- /dev/null +++ b/pkg/detector/defaults.go @@ -0,0 +1,28 @@ +// Copyright (C) 2026 boostsecurity.io +// SPDX-License-Identifier: GPL-3.0-or-later + +package detector + +// NewDefaultRegistry returns a Registry with all built-in detectors registered, +// in the same order the bagel CLI uses. Registration order is significant for +// redaction (scrub applies detectors in order), so keep it stable. +func NewDefaultRegistry() *Registry { + r := NewRegistry() + r.Register(NewGitHubPATDetector()) + r.Register(NewNPMTokenDetector()) + r.Register(NewSSHPrivateKeyDetector()) + r.Register(NewAIServiceDetector()) + r.Register(NewHTTPAuthDetector()) + r.Register(NewCloudCredentialsDetector()) + r.Register(NewVaultTokenDetector()) + r.Register(NewPyPITokenDetector()) + r.Register(NewWireGuardKeyDetector()) + r.Register(NewSplunkTokenDetector()) + r.Register(NewDatabaseConnectionDetector()) + r.Register(NewSlackTokenDetector()) + r.Register(NewStripeKeyDetector()) + r.Register(NewTwilioKeyDetector()) + r.Register(NewGenericAPIKeyDetector()) + r.Register(NewJWTDetector()) + return r +} diff --git a/pkg/detector/defaults_test.go b/pkg/detector/defaults_test.go new file mode 100644 index 0000000..207697c --- /dev/null +++ b/pkg/detector/defaults_test.go @@ -0,0 +1,46 @@ +// Copyright (C) 2026 boostsecurity.io +// SPDX-License-Identifier: GPL-3.0-or-later + +package detector + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestNewDefaultRegistry pins the built-in detector set and its registration +// order. A newly added detector that isn't wired into NewDefaultRegistry fails +// this test — order matters because scrub applies detectors in order. +func TestNewDefaultRegistry(t *testing.T) { + t.Parallel() + + want := []string{ + "github-token", + "npm-token", + "ssh-private-key", + "ai-service", + "http-authentication", + "cloud-credentials", + "vault-token", + "pypi-token", + "wireguard-key", + "splunk-token", + "database-connection-string", + "slack-token", + "stripe-key", + "twilio-key", + "generic-api-key", + "jwt", + } + + dets := NewDefaultRegistry().GetDetectors() + require.Len(t, dets, len(want), "default detector count changed") + + got := make([]string, len(dets)) + for i, d := range dets { + got[i] = d.Name() + } + assert.Equal(t, want, got, "default detector set/order changed") +} diff --git a/pkg/probe/defaults.go b/pkg/probe/defaults.go new file mode 100644 index 0000000..b435531 --- /dev/null +++ b/pkg/probe/defaults.go @@ -0,0 +1,35 @@ +// Copyright (C) 2026 boostsecurity.io +// SPDX-License-Identifier: GPL-3.0-or-later + +package probe + +import ( + "github.com/boostsecurityio/bagel/pkg/detector" + "github.com/boostsecurityio/bagel/pkg/models" +) + +// DefaultProbes returns every built-in probe wired to cfg and registry, in the +// same set and order the bagel CLI runs. Probes are returned regardless of their +// enabled flag; callers filter on Probe.IsEnabled (the collector already skips +// disabled probes at execution time). +func DefaultProbes(cfg *models.Config, registry *detector.Registry) []Probe { + return []Probe{ + NewGitProbe(cfg.Probes.Git, registry), + NewEnvProbe(cfg.Probes.Env, registry), + NewNPMProbe(cfg.Probes.NPM, registry), + NewSSHProbe(cfg.Probes.SSH, registry), + NewShellHistoryProbe(cfg.Probes.ShellHistory, registry), + NewCloudProbe(cfg.Probes.Cloud, registry), + NewJetBrainsProbe(cfg.Probes.JetBrains, registry), + NewGHProbe(cfg.Probes.GH, registry), + NewAICredentialsProbe(cfg.Probes.AICredentials, registry), + NewAIChatsProbe(cfg.Probes.AIChats, registry), + NewWireGuardProbe(cfg.Probes.WireGuard, registry), + NewPyPIProbe(cfg.Probes.PyPI, registry), + NewKubeProbe(cfg.Probes.Kube, registry), + NewDockerProbe(cfg.Probes.Docker, registry), + NewIaCProbe(cfg.Probes.IaC, registry), + NewMCPProbe(cfg.Probes.AIMCP, registry), + NewContextProbe(cfg.Probes.AIContext, registry), + } +} diff --git a/pkg/probe/defaults_test.go b/pkg/probe/defaults_test.go new file mode 100644 index 0000000..2ffed78 --- /dev/null +++ b/pkg/probe/defaults_test.go @@ -0,0 +1,66 @@ +// Copyright (C) 2026 boostsecurity.io +// SPDX-License-Identifier: GPL-3.0-or-later + +package probe + +import ( + "testing" + + "github.com/boostsecurityio/bagel/pkg/detector" + "github.com/boostsecurityio/bagel/pkg/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDefaultProbes pins the built-in probe set. A newly added probe that isn't +// wired into DefaultProbes fails this test. +func TestDefaultProbes(t *testing.T) { + t.Parallel() + + want := []string{ + "git", + "env", + "npm", + "ssh", + "shell_history", + "cloud", + "jetbrains", + "gh", + "ai_credentials", + "ai_chats", + "wireguard", + "pypi", + "kube", + "docker", + "iac", + "ai_mcp", + "ai_context", + } + + probes := DefaultProbes(&models.Config{}, detector.NewDefaultRegistry()) + require.Len(t, probes, len(want), "default probe count changed") + + got := make([]string, len(probes)) + for i, p := range probes { + got[i] = p.Name() + } + assert.Equal(t, want, got, "default probe set/order changed") +} + +// TestDefaultProbes_EnabledReflectsConfig confirms each probe's enabled flag is +// wired from cfg (callers filter on IsEnabled). +func TestDefaultProbes_EnabledReflectsConfig(t *testing.T) { + t.Parallel() + + cfg := &models.Config{} + cfg.Probes.Git.Enabled = true + // everything else stays false + + for _, p := range DefaultProbes(cfg, detector.NewDefaultRegistry()) { + if p.Name() == "git" { + assert.True(t, p.IsEnabled(), "git should be enabled") + } else { + assert.False(t, p.IsEnabled(), "%s should be disabled", p.Name()) + } + } +}