Export BuildFileIndex; decouple crawl from progress bar

pkg/collector/build_index.go

@@ -0,0 +1,110 @@
+// Copyright (C) 2026 boostsecurity.io
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package collector
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/boostsecurityio/bagel/pkg/cache"
+	"github.com/boostsecurityio/bagel/pkg/fileindex"
+	"github.com/boostsecurityio/bagel/pkg/models"
+	"github.com/boostsecurityio/bagel/pkg/progress"
+	"github.com/boostsecurityio/bagel/pkg/wsl"
+	"github.com/rs/zerolog/log"
+)
+
+// BuildFileIndex builds the file index for cfg's file-index settings and, when
+// store is non-nil, serves it from (and saves it to) the cache. It is the
+// canonical crawl the collector runs internally, exported so a caller can own
+// the crawl and inject the result via NewInput.FileIndex — sharing a single
+// index across bagel's probes and the caller's own consumers.
+//
+// reporter observes crawl progress; pass progress.NoOp{} (or nil) for none. A
+// cache hit returns without starting the reporter, so no progress is shown when
+// no crawl happens.
+func BuildFileIndex(ctx context.Context, cfg *models.Config, store *cache.Store, reporter progress.Reporter) (*fileindex.FileIndex, error) {
+	if reporter == nil {
+		reporter = progress.NoOp{}
+	}
+	logger := log.Ctx(ctx)
+
+	patterns := make([]fileindex.Pattern, 0, len(cfg.FileIndex.Patterns))
+	for _, p := range cfg.FileIndex.Patterns {
+		patterns = append(patterns, fileindex.Pattern{
+			Name:     p.Name,
+			Patterns: p.Patterns,
+			Type:     fileindex.PatternType(p.Type),
+		})
+	}
+
+	baseDirs := cfg.FileIndex.BaseDirs
+	// On Windows, append the home dirs of installed WSL distros so Linux
+	// secrets behind WSL aren't a blindspot. No-op on other platforms.
+	if cfg.FileIndex.ScanWSL {
+		if wslDirs := wsl.Homes(ctx); len(wslDirs) > 0 {
+			baseDirs = append(append([]string{}, baseDirs...), wslDirs...)
+		}
+	}
+
+	// Serve from cache when available.
+	if store != nil {
+		ttl, _ := time.ParseDuration(cfg.FileIndex.Cache.TTL)
+		index, err := store.Load(ctx, cache.LoadInput{
+			BaseDirs:       baseDirs,
+			ExcludePaths:   cfg.FileIndex.ExcludePaths,
+			Patterns:       patterns,
+			MaxDepth:       cfg.FileIndex.MaxDepth,
+			FollowSymlinks: cfg.FileIndex.FollowSymlinks,
+			TTL:            ttl,
+			ValidateFiles:  cfg.FileIndex.Cache.ValidateOnLoad,
+		})
+		if err != nil {
+			logger.Debug().Err(err).Msg("Failed to load file index from cache")
+		}
+		if index != nil {
+			return index, nil
+		}
+	}
+
+	// Cache miss (or caching disabled): crawl.
+	reporter.Start()
+	indexStartTime := time.Now()
+	index, err := fileindex.BuildIndex(ctx, fileindex.BuildIndexInput{
+		BaseDirs:         baseDirs,
+		ExcludePaths:     cfg.FileIndex.ExcludePaths,
+		Patterns:         patterns,
+		MaxDepth:         cfg.FileIndex.MaxDepth,
+		FollowSymlinks:   cfg.FileIndex.FollowSymlinks,
+		NumWorkers:       cfg.Resources.FileIndexWorkers,
+		ProgressCallback: reporter.Update,
+	})
+	reporter.Done()
+	if err != nil {
+		return nil, fmt.Errorf("build file index: %w", err)
+	}
+
+	logger.Info().
+		Dur("duration", time.Since(indexStartTime)).
+		Int("total_files", index.TotalFiles()).
+		Msg("File index built successfully")
+
+	// Save to cache (best effort).
+	if store != nil {
+		if err := store.Save(ctx, cache.SaveInput{
+			BaseDirs:       baseDirs,
+			ExcludePaths:   cfg.FileIndex.ExcludePaths,
+			Patterns:       patterns,
+			MaxDepth:       cfg.FileIndex.MaxDepth,
+			FollowSymlinks: cfg.FileIndex.FollowSymlinks,
+			Index:          index,
+			SampleSize:     cfg.FileIndex.Cache.SampleSize,
+		}); err != nil {
+			logger.Warn().Err(err).Msg("Failed to save file index to cache")
+		}
+	}
+
+	return index, nil
+}

pkg/collector/build_index_test.go

@@ -0,0 +1,78 @@
+// Copyright (C) 2026 boostsecurity.io
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package collector
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/boostsecurityio/bagel/pkg/cache"
+	"github.com/boostsecurityio/bagel/pkg/config"
+	"github.com/boostsecurityio/bagel/pkg/models"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// recordingReporter records which Reporter methods were called.
+type recordingReporter struct {
+	started, done bool
+	last          int64
+}
+
+func (r *recordingReporter) Start()         { r.started = true }
+func (r *recordingReporter) Update(p int64) { r.last = p }
+func (r *recordingReporter) Done()          { r.done = true }
+
+// scopedConfig loads defaults, points the crawl at baseDir, and disables WSL.
+func scopedConfig(t *testing.T, baseDir string) *models.Config {
+	t.Helper()
+	cfg, err := config.Load("")
+	require.NoError(t, err)
+	cfg.FileIndex.BaseDirs = []string{baseDir}
+	cfg.FileIndex.ScanWSL = false
+	return cfg
+}
+
+func TestBuildFileIndex_CrawlsAndReports(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, ".npmrc"), []byte("//registry/:_authToken=x\n"), 0o600))
+
+	r := &recordingReporter{}
+	idx, err := BuildFileIndex(context.Background(), scopedConfig(t, dir), nil, r)
+	require.NoError(t, err)
+	require.NotNil(t, idx)
+
+	assert.True(t, r.started, "a crawl should start the reporter")
+	assert.True(t, r.done, "a crawl should finish the reporter")
+}
+
+func TestBuildFileIndex_CacheHitSkipsReporter(t *testing.T) {
+	// Isolate the cache directory across all OS resolutions.
+	cacheHome := t.TempDir()
+	t.Setenv("HOME", cacheHome)
+	t.Setenv("XDG_CACHE_HOME", cacheHome)
+	t.Setenv("LOCALAPPDATA", cacheHome)
+
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, ".npmrc"), []byte("//registry/:_authToken=x\n"), 0o600))
+	cfg := scopedConfig(t, dir)
+
+	store, err := cache.NewStore()
+	require.NoError(t, err)
+
+	// First build populates the cache and crawls.
+	first := &recordingReporter{}
+	_, err = BuildFileIndex(context.Background(), cfg, store, first)
+	require.NoError(t, err)
+	require.True(t, first.started, "first build should crawl")
+
+	// Second build must be served from cache — no crawl, so no Start.
+	second := &recordingReporter{}
+	idx, err := BuildFileIndex(context.Background(), cfg, store, second)
+	require.NoError(t, err)
+	require.NotNil(t, idx)
+	assert.False(t, second.started, "a cache hit must not start the reporter")
+}

pkg/collector/collector.go

@@ -14,8 +14,8 @@ import (
 	"github.com/boostsecurityio/bagel/pkg/fileindex"
 	"github.com/boostsecurityio/bagel/pkg/models"
 	"github.com/boostsecurityio/bagel/pkg/probe"
+	"github.com/boostsecurityio/bagel/pkg/progress"
 	"github.com/boostsecurityio/bagel/pkg/sysinfo"
-	"github.com/boostsecurityio/bagel/pkg/wsl"
 	"github.com/mattn/go-isatty"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
@@ -128,139 +128,54 @@ func (c *Collector) Collect(ctx context.Context) (*models.ScanResult, error) {
 	}, nil
 }
 
-// buildFileIndex constructs the file index based on configuration
+// buildFileIndex constructs the file index based on configuration, delegating
+// to the exported BuildFileIndex and rendering progress with a terminal bar
+// when appropriate.
 func (c *Collector) buildFileIndex(ctx context.Context) (*fileindex.FileIndex, error) {
-	logger := log.Ctx(ctx)
-
-	// Convert config patterns to fileindex.Pattern
-	patterns := make([]fileindex.Pattern, 0, len(c.config.FileIndex.Patterns))
-	for _, p := range c.config.FileIndex.Patterns {
-		patterns = append(patterns, fileindex.Pattern{
-			Name:     p.Name,
-			Patterns: p.Patterns,
-			Type:     fileindex.PatternType(p.Type),
-		})
-	}
-
-	baseDirs := c.config.FileIndex.BaseDirs
-
-	// On Windows, append the home dirs of installed WSL distros so Linux
-	// secrets behind WSL aren't a blindspot. No-op on other platforms.
-	if c.config.FileIndex.ScanWSL {
-		if wslDirs := wsl.Homes(ctx); len(wslDirs) > 0 {
-			baseDirs = append(append([]string{}, baseDirs...), wslDirs...)
-		}
-	}
-
-	// Try loading from cache (unless disabled)
+	var store *cache.Store
 	if !c.noCache {
-		index, err := c.loadFromCache(ctx, baseDirs, patterns)
-		if err != nil {
-			logger.Debug().Err(err).Msg("Failed to load file index from cache")
-		}
-		if index != nil {
-			return index, nil
-		}
+		store = c.cacheStore
 	}
 
-	// Build fresh index
-	indexStartTime := time.Now()
-
-	// Set up progress callback if progress bars are enabled
-	var progressCallback func(processed int64)
-	var bar *progressbar.ProgressBar
+	var reporter progress.Reporter = progress.NoOp{}
 	if c.shouldShowProgress() {
-		bar = progressbar.NewOptions(-1,
-			progressbar.OptionSetDescription("Indexing files"),
-			progressbar.OptionSetWriter(os.Stderr),
-			progressbar.OptionSpinnerType(14),
-			progressbar.OptionShowCount(),
-		)
-		progressCallback = func(processed int64) {
-			_ = bar.Set64(processed)
-		}
+		reporter = newBarReporter("Indexing files")
 	}
 
-	input := fileindex.BuildIndexInput{
-		BaseDirs:         baseDirs,
-		ExcludePaths:     c.config.FileIndex.ExcludePaths,
-		Patterns:         patterns,
-		MaxDepth:         c.config.FileIndex.MaxDepth,
-		FollowSymlinks:   c.config.FileIndex.FollowSymlinks,
-		NumWorkers:       c.config.Resources.FileIndexWorkers,
-		ProgressCallback: progressCallback,
-	}
-
-	index, err := fileindex.BuildIndex(ctx, input)
-
-	// Finish progress bar if it was created
-	if bar != nil {
-		_ = bar.Finish()
-	}
-
-	if err != nil {
-		return nil, fmt.Errorf("build file index: %w", err)
-	}
-
-	indexDuration := time.Since(indexStartTime)
-	logger.Info().
-		Dur("duration", indexDuration).
-		Int("total_files", index.TotalFiles()).
-		Msg("File index built successfully")
-
-	// Save to cache (best effort)
-	if !c.noCache {
-		if err := c.saveToCache(ctx, baseDirs, patterns, index); err != nil {
-			logger.Warn().Err(err).Msg("Failed to save file index to cache")
-		}
-	}
-
-	return index, nil
+	return BuildFileIndex(ctx, c.config, store, reporter)
 }
 
-// loadFromCache attempts to load the file index from cache
-func (c *Collector) loadFromCache(ctx context.Context, baseDirs []string, patterns []fileindex.Pattern) (*fileindex.FileIndex, error) {
-	if c.cacheStore == nil {
-		return nil, nil
-	}
+// barReporter renders progress as a terminal spinner via progressbar. The bar
+// is created lazily in Start so nothing is shown when a crawl never begins
+// (e.g. a cache hit). It implements progress.Reporter.
+type barReporter struct {
+	description string
+	bar         *progressbar.ProgressBar
+}
 
-	ttl, _ := time.ParseDuration(c.config.FileIndex.Cache.TTL)
-
-	index, err := c.cacheStore.Load(ctx, cache.LoadInput{
-		BaseDirs:       baseDirs,
-		ExcludePaths:   c.config.FileIndex.ExcludePaths,
-		Patterns:       patterns,
-		MaxDepth:       c.config.FileIndex.MaxDepth,
-		FollowSymlinks: c.config.FileIndex.FollowSymlinks,
-		TTL:            ttl,
-		ValidateFiles:  c.config.FileIndex.Cache.ValidateOnLoad,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("load from cache: %w", err)
-	}
+func newBarReporter(description string) *barReporter {
+	return &barReporter{description: description}
+}
 
-	return index, nil
+func (b *barReporter) Start() {
+	b.bar = progressbar.NewOptions(-1,
+		progressbar.OptionSetDescription(b.description),
+		progressbar.OptionSetWriter(os.Stderr),
+		progressbar.OptionSpinnerType(14),
+		progressbar.OptionShowCount(),
+	)
 }
 
-// saveToCache persists the file index to cache
-func (c *Collector) saveToCache(ctx context.Context, baseDirs []string, patterns []fileindex.Pattern, index *fileindex.FileIndex) error {
-	if c.cacheStore == nil {
-		return nil
+func (b *barReporter) Update(processed int64) {
+	if b.bar != nil {
+		_ = b.bar.Set64(processed)
 	}
+}
 
-	if err := c.cacheStore.Save(ctx, cache.SaveInput{
-		BaseDirs:       baseDirs,
-		ExcludePaths:   c.config.FileIndex.ExcludePaths,
-		Patterns:       patterns,
-		MaxDepth:       c.config.FileIndex.MaxDepth,
-		FollowSymlinks: c.config.FileIndex.FollowSymlinks,
-		Index:          index,
-		SampleSize:     c.config.FileIndex.Cache.SampleSize,
-	}); err != nil {
-		return fmt.Errorf("save to cache: %w", err)
+func (b *barReporter) Done() {
+	if b.bar != nil {
+		_ = b.bar.Finish()
 	}
-
-	return nil
 }
 
 // executeProbes runs all enabled probes concurrently with timeouts using errgroup.