fix: persist snapshot state across restarts to prevent incomplete win…

pkg/emitter/exporter.go

@@ -113,6 +113,11 @@ func (de *defaultExporter) Start(interval time.Duration) bool {
 
 			// wait for all emit tasks to complete before continuing
 			emitTasks.Wait()
+
+			// persist snapshot state after successful export to maintain window continuity across restarts
+			if err := de.snapshotProvider.PersistState(); err != nil {
+				log.Warnf("failed to persist snapshot state: %v", err)
+			}
 		}
 	}()
 

pkg/emitter/exporter_test.go

@@ -172,6 +172,10 @@ func (e *emptySnapshotProvider) SnapshotOf(ds core.DataSource) (*ClusterSnapshot
 	return &ClusterSnapshot{}, nil
 }
 
+func (e *emptySnapshotProvider) PersistState() error {
+	return nil
+}
+
 type emptyDataSource struct{}
 
 func (e *emptyDataSource) OpenCostSource() source.OpenCostDataSource {

pkg/emitter/snapshot.go

@@ -1,8 +1,11 @@
 package emitter
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/hashicorp/go-multierror"
@@ -21,12 +24,22 @@ type SnapshotProvider interface {
 	// SnapshotOf generates a `ClusterSnapshot` from the provided `core.DataSource` and returns it.
 	// If the snapshot generation fails, an error is returned.
 	SnapshotOf(core.DataSource) (*ClusterSnapshot, error)
+
+	// PersistState persists the current snapshot state to disk for recovery after restart.
+	// This should be called after successful export to ensure state consistency.
+	PersistState() error
 }
 
 // FIXME: (bolt) use a metrics summary cache duration of 5 minutes while we're using a prometheus data source.
 // FIXME: (bolt) this should be fine to run on a much faster frequency with a non-promethues metrics querier.
 var metricsSummaryCacheDuration time.Duration = 5 * time.Minute
 
+// SnapshotState represents the persisted state of the snapshot provider to maintain
+// continuity across restarts.
+type SnapshotState struct {
+	LastSnapshot time.Time `json:"lastSnapshot"`
+}
+
 // ConcurrentSnapshotProvider is a struct that implements the `SnapshotProvider` interface and executes the
 // snapshot generation process concurrently.
 type ConcurrentSnapshotProvider struct {
@@ -48,10 +61,76 @@ func NewConcurrentSnapshotProvider(config *SnapshotConfig) SnapshotProvider {
 		now = defaultNow
 	}
 
-	return &ConcurrentSnapshotProvider{
+	csp := &ConcurrentSnapshotProvider{
 		now:    now,
 		config: config,
 	}
+
+	// Attempt to recover persisted state from previous runs
+	csp.recoverState()
+
+	return csp
+}
+
+// persistState writes the current lastSnapshot timestamp to disk for recovery after restart.
+func (csp *ConcurrentSnapshotProvider) persistState() error {
+	if csp.config.ScratchDir == "" {
+		return fmt.Errorf("scratch directory not configured")
+	}
+
+	stateFile := filepath.Join(csp.config.ScratchDir, "snapshot-state.json")
+	state := SnapshotState{
+		LastSnapshot: csp.lastSnapshot,
+	}
+
+	data, err := json.Marshal(state)
+	if err != nil {
+		return fmt.Errorf("failed to marshal snapshot state: %w", err)
+	}
+
+	if err := os.MkdirAll(csp.config.ScratchDir, 0755); err != nil {
+		return fmt.Errorf("failed to create scratch directory: %w", err)
+	}
+
+	if err := os.WriteFile(stateFile, data, 0644); err != nil {
+		return fmt.Errorf("failed to write snapshot state: %w", err)
+	}
+
+	return nil
+}
+
+// recoverState attempts to read the persisted snapshot state from disk.
+// If the state file is missing or corrupt, it treats this as a cold start and logs a warning.
+func (csp *ConcurrentSnapshotProvider) recoverState() {
+	if csp.config.ScratchDir == "" {
+		log.Warnf("Scratch directory not configured, starting with empty snapshot state")
+		return
+	}
+
+	stateFile := filepath.Join(csp.config.ScratchDir, "snapshot-state.json")
+	data, err := os.ReadFile(stateFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Infof("No previous snapshot state found, starting fresh")
+		} else {
+			log.Warnf("Failed to read snapshot state file: %v, starting fresh", err)
+		}
+		return
+	}
+
+	var state SnapshotState
+	if err := json.Unmarshal(data, &state); err != nil {
+		log.Warnf("Failed to unmarshal snapshot state: %v, starting fresh", err)
+		return
+	}
+
+	csp.lastSnapshot = state.LastSnapshot
+	log.Infof("Recovered snapshot state: lastSnapshot=%s", state.LastSnapshot.Format(time.RFC3339))
+}
+
+// PersistState implements the SnapshotProvider interface method to persist state.
+func (csp *ConcurrentSnapshotProvider) PersistState() error {
+	return csp.persistState()
 }
 
 // SnapshotOf generates a `ClusterSnapshot` from the provided `core.DataSource` and returns it.

pkg/emitter/snapshotconfig.go

@@ -1,6 +1,7 @@
 package emitter
 
 import (
+	"os"
 	"time"
 
 	"github.com/ibm/finops-agent/pkg/env"
@@ -181,6 +182,15 @@ func defaultNow() time.Time {
 	return time.Now().UTC()
 }
 
+// getScratchDir returns the scratch directory path from the SCRATCH_DIR environment variable,
+// or the default path if not set.
+func getScratchDir() string {
+	if dir := os.Getenv("SCRATCH_DIR"); dir != "" {
+		return dir
+	}
+	return "/opt/finops-agent"
+}
+
 // SnapshotConfig holds the configuration for general snapshotting options.
 type SnapshotConfig struct {
 	// UseMetricsCache indicates whether or not to use a cache for metrics query results.
@@ -194,6 +204,9 @@ type SnapshotConfig struct {
 	// KubernetesSnapshotConfig holds the configuration for Kubernetes resources to snapshot.
 	KubernetesSnapshot *KubernetesSnapshotConfig
 
+	// ScratchDir is the directory path where snapshot state will be persisted.
+	ScratchDir string
+
 	// Now is the func used to determine the current time.
 	Now Now
 }
@@ -215,6 +228,7 @@ func NewSnapshotConfigFromEnv() *SnapshotConfig {
 	return &SnapshotConfig{
 		UseMetricsCache:        !env.IsCollectorDataSourceEnabled(),
 		MinutelyMetricsEnabled: env.IsMinuteMetricsEnabled(),
+		ScratchDir:             getScratchDir(),
 		Now:                    defaultNow,
 	}
 }
@@ -225,6 +239,7 @@ func DefaultSnapshotConfig() *SnapshotConfig {
 		UseMetricsCache:        false,
 		MinutelyMetricsEnabled: false,
 		KubernetesSnapshot:     NewKubernetesSnapshotConfig().EnableAll(),
+		ScratchDir:             "/opt/finops-agent",
 		Now:                    defaultNow,
 	}
 }