diff --git a/pkg/collector/build_index.go b/pkg/collector/build_index.go new file mode 100644 index 0000000..c8df4c0 --- /dev/null +++ b/pkg/collector/build_index.go @@ -0,0 +1,110 @@ +// Copyright (C) 2026 boostsecurity.io +// SPDX-License-Identifier: GPL-3.0-or-later + +package collector + +import ( + "context" + "fmt" + "time" + + "github.com/boostsecurityio/bagel/pkg/cache" + "github.com/boostsecurityio/bagel/pkg/fileindex" + "github.com/boostsecurityio/bagel/pkg/models" + "github.com/boostsecurityio/bagel/pkg/progress" + "github.com/boostsecurityio/bagel/pkg/wsl" + "github.com/rs/zerolog/log" +) + +// BuildFileIndex builds the file index for cfg's file-index settings and, when +// store is non-nil, serves it from (and saves it to) the cache. It is the +// canonical crawl the collector runs internally, exported so a caller can own +// the crawl and inject the result via NewInput.FileIndex — sharing a single +// index across bagel's probes and the caller's own consumers. +// +// reporter observes crawl progress; pass progress.NoOp{} (or nil) for none. A +// cache hit returns without starting the reporter, so no progress is shown when +// no crawl happens. +func BuildFileIndex(ctx context.Context, cfg *models.Config, store *cache.Store, reporter progress.Reporter) (*fileindex.FileIndex, error) { + if reporter == nil { + reporter = progress.NoOp{} + } + logger := log.Ctx(ctx) + + patterns := make([]fileindex.Pattern, 0, len(cfg.FileIndex.Patterns)) + for _, p := range cfg.FileIndex.Patterns { + patterns = append(patterns, fileindex.Pattern{ + Name: p.Name, + Patterns: p.Patterns, + Type: fileindex.PatternType(p.Type), + }) + } + + baseDirs := cfg.FileIndex.BaseDirs + // On Windows, append the home dirs of installed WSL distros so Linux + // secrets behind WSL aren't a blindspot. No-op on other platforms. + if cfg.FileIndex.ScanWSL { + if wslDirs := wsl.Homes(ctx); len(wslDirs) > 0 { + baseDirs = append(append([]string{}, baseDirs...), wslDirs...) + } + } + + // Serve from cache when available. + if store != nil { + ttl, _ := time.ParseDuration(cfg.FileIndex.Cache.TTL) + index, err := store.Load(ctx, cache.LoadInput{ + BaseDirs: baseDirs, + ExcludePaths: cfg.FileIndex.ExcludePaths, + Patterns: patterns, + MaxDepth: cfg.FileIndex.MaxDepth, + FollowSymlinks: cfg.FileIndex.FollowSymlinks, + TTL: ttl, + ValidateFiles: cfg.FileIndex.Cache.ValidateOnLoad, + }) + if err != nil { + logger.Debug().Err(err).Msg("Failed to load file index from cache") + } + if index != nil { + return index, nil + } + } + + // Cache miss (or caching disabled): crawl. + reporter.Start() + indexStartTime := time.Now() + index, err := fileindex.BuildIndex(ctx, fileindex.BuildIndexInput{ + BaseDirs: baseDirs, + ExcludePaths: cfg.FileIndex.ExcludePaths, + Patterns: patterns, + MaxDepth: cfg.FileIndex.MaxDepth, + FollowSymlinks: cfg.FileIndex.FollowSymlinks, + NumWorkers: cfg.Resources.FileIndexWorkers, + ProgressCallback: reporter.Update, + }) + reporter.Done() + if err != nil { + return nil, fmt.Errorf("build file index: %w", err) + } + + logger.Info(). + Dur("duration", time.Since(indexStartTime)). + Int("total_files", index.TotalFiles()). + Msg("File index built successfully") + + // Save to cache (best effort). + if store != nil { + if err := store.Save(ctx, cache.SaveInput{ + BaseDirs: baseDirs, + ExcludePaths: cfg.FileIndex.ExcludePaths, + Patterns: patterns, + MaxDepth: cfg.FileIndex.MaxDepth, + FollowSymlinks: cfg.FileIndex.FollowSymlinks, + Index: index, + SampleSize: cfg.FileIndex.Cache.SampleSize, + }); err != nil { + logger.Warn().Err(err).Msg("Failed to save file index to cache") + } + } + + return index, nil +} diff --git a/pkg/collector/build_index_test.go b/pkg/collector/build_index_test.go new file mode 100644 index 0000000..5b95bd6 --- /dev/null +++ b/pkg/collector/build_index_test.go @@ -0,0 +1,78 @@ +// Copyright (C) 2026 boostsecurity.io +// SPDX-License-Identifier: GPL-3.0-or-later + +package collector + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/boostsecurityio/bagel/pkg/cache" + "github.com/boostsecurityio/bagel/pkg/config" + "github.com/boostsecurityio/bagel/pkg/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// recordingReporter records which Reporter methods were called. +type recordingReporter struct { + started, done bool + last int64 +} + +func (r *recordingReporter) Start() { r.started = true } +func (r *recordingReporter) Update(p int64) { r.last = p } +func (r *recordingReporter) Done() { r.done = true } + +// scopedConfig loads defaults, points the crawl at baseDir, and disables WSL. +func scopedConfig(t *testing.T, baseDir string) *models.Config { + t.Helper() + cfg, err := config.Load("") + require.NoError(t, err) + cfg.FileIndex.BaseDirs = []string{baseDir} + cfg.FileIndex.ScanWSL = false + return cfg +} + +func TestBuildFileIndex_CrawlsAndReports(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, ".npmrc"), []byte("//registry/:_authToken=x\n"), 0o600)) + + r := &recordingReporter{} + idx, err := BuildFileIndex(context.Background(), scopedConfig(t, dir), nil, r) + require.NoError(t, err) + require.NotNil(t, idx) + + assert.True(t, r.started, "a crawl should start the reporter") + assert.True(t, r.done, "a crawl should finish the reporter") +} + +func TestBuildFileIndex_CacheHitSkipsReporter(t *testing.T) { + // Isolate the cache directory across all OS resolutions. + cacheHome := t.TempDir() + t.Setenv("HOME", cacheHome) + t.Setenv("XDG_CACHE_HOME", cacheHome) + t.Setenv("LOCALAPPDATA", cacheHome) + + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, ".npmrc"), []byte("//registry/:_authToken=x\n"), 0o600)) + cfg := scopedConfig(t, dir) + + store, err := cache.NewStore() + require.NoError(t, err) + + // First build populates the cache and crawls. + first := &recordingReporter{} + _, err = BuildFileIndex(context.Background(), cfg, store, first) + require.NoError(t, err) + require.True(t, first.started, "first build should crawl") + + // Second build must be served from cache — no crawl, so no Start. + second := &recordingReporter{} + idx, err := BuildFileIndex(context.Background(), cfg, store, second) + require.NoError(t, err) + require.NotNil(t, idx) + assert.False(t, second.started, "a cache hit must not start the reporter") +} diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index 1fbd0b8..6433a2e 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -14,8 +14,8 @@ import ( "github.com/boostsecurityio/bagel/pkg/fileindex" "github.com/boostsecurityio/bagel/pkg/models" "github.com/boostsecurityio/bagel/pkg/probe" + "github.com/boostsecurityio/bagel/pkg/progress" "github.com/boostsecurityio/bagel/pkg/sysinfo" - "github.com/boostsecurityio/bagel/pkg/wsl" "github.com/mattn/go-isatty" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -128,139 +128,54 @@ func (c *Collector) Collect(ctx context.Context) (*models.ScanResult, error) { }, nil } -// buildFileIndex constructs the file index based on configuration +// buildFileIndex constructs the file index based on configuration, delegating +// to the exported BuildFileIndex and rendering progress with a terminal bar +// when appropriate. func (c *Collector) buildFileIndex(ctx context.Context) (*fileindex.FileIndex, error) { - logger := log.Ctx(ctx) - - // Convert config patterns to fileindex.Pattern - patterns := make([]fileindex.Pattern, 0, len(c.config.FileIndex.Patterns)) - for _, p := range c.config.FileIndex.Patterns { - patterns = append(patterns, fileindex.Pattern{ - Name: p.Name, - Patterns: p.Patterns, - Type: fileindex.PatternType(p.Type), - }) - } - - baseDirs := c.config.FileIndex.BaseDirs - - // On Windows, append the home dirs of installed WSL distros so Linux - // secrets behind WSL aren't a blindspot. No-op on other platforms. - if c.config.FileIndex.ScanWSL { - if wslDirs := wsl.Homes(ctx); len(wslDirs) > 0 { - baseDirs = append(append([]string{}, baseDirs...), wslDirs...) - } - } - - // Try loading from cache (unless disabled) + var store *cache.Store if !c.noCache { - index, err := c.loadFromCache(ctx, baseDirs, patterns) - if err != nil { - logger.Debug().Err(err).Msg("Failed to load file index from cache") - } - if index != nil { - return index, nil - } + store = c.cacheStore } - // Build fresh index - indexStartTime := time.Now() - - // Set up progress callback if progress bars are enabled - var progressCallback func(processed int64) - var bar *progressbar.ProgressBar + var reporter progress.Reporter = progress.NoOp{} if c.shouldShowProgress() { - bar = progressbar.NewOptions(-1, - progressbar.OptionSetDescription("Indexing files"), - progressbar.OptionSetWriter(os.Stderr), - progressbar.OptionSpinnerType(14), - progressbar.OptionShowCount(), - ) - progressCallback = func(processed int64) { - _ = bar.Set64(processed) - } + reporter = newBarReporter("Indexing files") } - input := fileindex.BuildIndexInput{ - BaseDirs: baseDirs, - ExcludePaths: c.config.FileIndex.ExcludePaths, - Patterns: patterns, - MaxDepth: c.config.FileIndex.MaxDepth, - FollowSymlinks: c.config.FileIndex.FollowSymlinks, - NumWorkers: c.config.Resources.FileIndexWorkers, - ProgressCallback: progressCallback, - } - - index, err := fileindex.BuildIndex(ctx, input) - - // Finish progress bar if it was created - if bar != nil { - _ = bar.Finish() - } - - if err != nil { - return nil, fmt.Errorf("build file index: %w", err) - } - - indexDuration := time.Since(indexStartTime) - logger.Info(). - Dur("duration", indexDuration). - Int("total_files", index.TotalFiles()). - Msg("File index built successfully") - - // Save to cache (best effort) - if !c.noCache { - if err := c.saveToCache(ctx, baseDirs, patterns, index); err != nil { - logger.Warn().Err(err).Msg("Failed to save file index to cache") - } - } - - return index, nil + return BuildFileIndex(ctx, c.config, store, reporter) } -// loadFromCache attempts to load the file index from cache -func (c *Collector) loadFromCache(ctx context.Context, baseDirs []string, patterns []fileindex.Pattern) (*fileindex.FileIndex, error) { - if c.cacheStore == nil { - return nil, nil - } +// barReporter renders progress as a terminal spinner via progressbar. The bar +// is created lazily in Start so nothing is shown when a crawl never begins +// (e.g. a cache hit). It implements progress.Reporter. +type barReporter struct { + description string + bar *progressbar.ProgressBar +} - ttl, _ := time.ParseDuration(c.config.FileIndex.Cache.TTL) - - index, err := c.cacheStore.Load(ctx, cache.LoadInput{ - BaseDirs: baseDirs, - ExcludePaths: c.config.FileIndex.ExcludePaths, - Patterns: patterns, - MaxDepth: c.config.FileIndex.MaxDepth, - FollowSymlinks: c.config.FileIndex.FollowSymlinks, - TTL: ttl, - ValidateFiles: c.config.FileIndex.Cache.ValidateOnLoad, - }) - if err != nil { - return nil, fmt.Errorf("load from cache: %w", err) - } +func newBarReporter(description string) *barReporter { + return &barReporter{description: description} +} - return index, nil +func (b *barReporter) Start() { + b.bar = progressbar.NewOptions(-1, + progressbar.OptionSetDescription(b.description), + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSpinnerType(14), + progressbar.OptionShowCount(), + ) } -// saveToCache persists the file index to cache -func (c *Collector) saveToCache(ctx context.Context, baseDirs []string, patterns []fileindex.Pattern, index *fileindex.FileIndex) error { - if c.cacheStore == nil { - return nil +func (b *barReporter) Update(processed int64) { + if b.bar != nil { + _ = b.bar.Set64(processed) } +} - if err := c.cacheStore.Save(ctx, cache.SaveInput{ - BaseDirs: baseDirs, - ExcludePaths: c.config.FileIndex.ExcludePaths, - Patterns: patterns, - MaxDepth: c.config.FileIndex.MaxDepth, - FollowSymlinks: c.config.FileIndex.FollowSymlinks, - Index: index, - SampleSize: c.config.FileIndex.Cache.SampleSize, - }); err != nil { - return fmt.Errorf("save to cache: %w", err) +func (b *barReporter) Done() { + if b.bar != nil { + _ = b.bar.Finish() } - - return nil } // executeProbes runs all enabled probes concurrently with timeouts using errgroup. diff --git a/pkg/progress/progress.go b/pkg/progress/progress.go new file mode 100644 index 0000000..1f50c78 --- /dev/null +++ b/pkg/progress/progress.go @@ -0,0 +1,34 @@ +// Copyright (C) 2026 boostsecurity.io +// SPDX-License-Identifier: GPL-3.0-or-later + +// Package progress decouples long-running operations (such as the file-index +// crawl) from how their progress is surfaced. An operation reports through the +// Reporter interface; the caller supplies an implementation — a terminal +// spinner, a daemon event stream, or NoOp — so the operation never depends on +// a particular UI. +package progress + +// Reporter observes the progress of a long-running operation. +type Reporter interface { + // Start signals the operation has begun. A reporter may lazily create its + // display here, so nothing is shown when the operation is skipped (for + // example a cache hit that never starts a crawl). + Start() + // Update reports the cumulative number of items processed so far. + Update(processed int64) + // Done signals the operation finished; the reporter flushes or closes. + Done() +} + +// NoOp is a Reporter that discards all progress. It is the zero-cost default +// for headless consumers (daemon, tests, machine-readable output). +type NoOp struct{} + +// Start does nothing. +func (NoOp) Start() {} + +// Update does nothing. +func (NoOp) Update(int64) {} + +// Done does nothing. +func (NoOp) Done() {}