diff --git a/prometheus/counter.go b/prometheus/counter.go index 7d963d3af..5a0ef1114 100644 --- a/prometheus/counter.go +++ b/prometheus/counter.go @@ -15,6 +15,7 @@ package prometheus import ( "errors" + "fmt" "math" "sync/atomic" "time" @@ -70,6 +71,11 @@ type CounterVecOpts struct { // of labels. Each label value will be constrained with the optional Constraint // function, if provided. VariableLabels ConstrainableLabels + + // TTL, if greater than zero, enables per-child expiration for this vector. + // See MetricVecOpts.TTL for semantics, orphaned-handle behavior, and + // cleanup via Registry.Gather / CleanupExpired. + TTL time.Duration } // NewCounter creates a new Counter based on the provided CounterOpts. @@ -211,15 +217,26 @@ func (v2) NewCounterVec(opts CounterVecOpts) *CounterVec { if opts.now == nil { opts.now = time.Now } - return &CounterVec{ - MetricVec: NewMetricVec(desc, func(lvs ...string) Metric { - if len(lvs) != len(desc.variableLabels.names) { - panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.names, lvs)) - } - result := &counter{desc: desc, labelPairs: MakeLabelPairs(desc, lvs), now: opts.now} - result.init(result) // Init self-collection. - result.createdTs = timestamppb.New(opts.now()) + if opts.TTL < 0 { + panic(fmt.Sprintf("invalid negative ttl: %v", opts.TTL)) + } + newMetric := func(lvs ...string) Metric { + if len(lvs) != len(desc.variableLabels.names) { + panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.names, lvs)) + } + result := &counter{desc: desc, labelPairs: MakeLabelPairs(desc, lvs), now: opts.now} + result.init(result) // Init self-collection. + result.createdTs = timestamppb.New(opts.now()) + if opts.TTL <= 0 { return result + } + return newTTLCounter(result) + } + return &CounterVec{ + MetricVec: V2.NewMetricVec(MetricVecOpts{ + Desc: desc, + NewMetric: newMetric, + TTL: opts.TTL, }), } } diff --git a/prometheus/gauge.go b/prometheus/gauge.go index 41e54bf27..7cb5be192 100644 --- a/prometheus/gauge.go +++ b/prometheus/gauge.go @@ -14,6 +14,7 @@ package prometheus import ( + "fmt" "math" "sync/atomic" "time" @@ -65,6 +66,10 @@ type GaugeVecOpts struct { // of labels. Each label value will be constrained with the optional Constraint // function, if provided. VariableLabels ConstrainableLabels + + // TTL, if greater than zero, enables per-child expiration for this vector. + // See MetricVecOpts.TTL for semantics. + TTL time.Duration } // NewGauge creates a new Gauge based on the provided GaugeOpts. @@ -166,14 +171,25 @@ func (v2) NewGaugeVec(opts GaugeVecOpts) *GaugeVec { opts.ConstLabels, WithUnit(opts.Unit), ) - return &GaugeVec{ - MetricVec: NewMetricVec(desc, func(lvs ...string) Metric { - if len(lvs) != len(desc.variableLabels.names) { - panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.names, lvs)) - } - result := &gauge{desc: desc, labelPairs: MakeLabelPairs(desc, lvs)} - result.init(result) // Init self-collection. + if opts.TTL < 0 { + panic(fmt.Sprintf("invalid negative ttl: %v", opts.TTL)) + } + newMetric := func(lvs ...string) Metric { + if len(lvs) != len(desc.variableLabels.names) { + panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.names, lvs)) + } + result := &gauge{desc: desc, labelPairs: MakeLabelPairs(desc, lvs)} + result.init(result) // Init self-collection. + if opts.TTL <= 0 { return result + } + return newTTLGauge(result) + } + return &GaugeVec{ + MetricVec: V2.NewMetricVec(MetricVecOpts{ + Desc: desc, + NewMetric: newMetric, + TTL: opts.TTL, }), } } diff --git a/prometheus/histogram.go b/prometheus/histogram.go index 88bae3b32..f685cbb29 100644 --- a/prometheus/histogram.go +++ b/prometheus/histogram.go @@ -515,6 +515,10 @@ type HistogramVecOpts struct { // of labels. Each label value will be constrained with the optional Constraint // function, if provided. VariableLabels ConstrainableLabels + + // TTL, if greater than zero, enables per-child expiration for this vector. + // See MetricVecOpts.TTL for semantics. + TTL time.Duration } // NewHistogram creates a new Histogram based on the provided HistogramOpts. It @@ -970,7 +974,7 @@ func (h *histogram) maybeReset( // We are using the possibly mocked h.now() rather than // time.Since(h.lastResetTime) to enable testing. if h.nativeHistogramMinResetDuration == 0 || // No reset configured. - h.resetScheduled || // Do not interfere if a reset is already scheduled. + h.resetScheduled || // Do not interefere if a reset is already scheduled. h.now().Sub(h.lastResetTime) < h.nativeHistogramMinResetDuration { return false } @@ -1057,8 +1061,8 @@ func (h *histogram) maybeWidenZeroBucket(hot, cold *histogramCounts) bool { atomic.StoreUint64(&cold.nativeHistogramZeroThresholdBits, math.Float64bits(newZeroThreshold)) // ...and then merge the newly deleted buckets into the wider zero // bucket. - mergeAndDeleteOrAddAndReset := func(hotBuckets, coldBuckets *sync.Map) func(k, v any) bool { - return func(k, v any) bool { + mergeAndDeleteOrAddAndReset := func(hotBuckets, coldBuckets *sync.Map) func(k, v interface{}) bool { + return func(k, v interface{}) bool { key := k.(int) bucket := v.(*int64) if key == smallestKey { @@ -1111,8 +1115,8 @@ func (h *histogram) doubleBucketWidth(hot, cold *histogramCounts) { // ...adjust the schema in the cold counts, too... atomic.StoreInt32(&cold.nativeHistogramSchema, coldSchema) // ...and then merge the cold buckets into the wider hot buckets. - merge := func(hotBuckets *sync.Map) func(k, v any) bool { - return func(k, v any) bool { + merge := func(hotBuckets *sync.Map) func(k, v interface{}) bool { + return func(k, v interface{}) bool { key := k.(int) bucket := v.(*int64) // Adjust key to match the bucket to merge into. @@ -1196,9 +1200,21 @@ func (v2) NewHistogramVec(opts HistogramVecOpts) *HistogramVec { opts.ConstLabels, WithUnit(opts.Unit), ) + if opts.TTL < 0 { + panic(fmt.Sprintf("invalid negative ttl: %v", opts.TTL)) + } + newMetric := func(lvs ...string) Metric { + h := newHistogram(desc, opts.HistogramOpts, lvs...) + if opts.TTL <= 0 { + return h + } + return newTTLHistogram(h) + } return &HistogramVec{ - MetricVec: NewMetricVec(desc, func(lvs ...string) Metric { - return newHistogram(desc, opts.HistogramOpts, lvs...) + MetricVec: V2.NewMetricVec(MetricVecOpts{ + Desc: desc, + NewMetric: newMetric, + TTL: opts.TTL, }), } } @@ -1481,7 +1497,7 @@ func pickSchema(bucketFactor float64) int32 { func makeBuckets(buckets *sync.Map) ([]*dto.BucketSpan, []int64) { var ii []int - buckets.Range(func(k, v any) bool { + buckets.Range(func(k, v interface{}) bool { ii = append(ii, k.(int)) return true }) @@ -1558,8 +1574,8 @@ func addToBucket(buckets *sync.Map, key int, increment int64) bool { // according to the buckets ranged through. It then resets all buckets ranged // through to 0 (but leaves them in place so that they don't need to get // recreated on the next scrape). -func addAndReset(hotBuckets *sync.Map, bucketNumber *uint32) func(k, v any) bool { - return func(k, v any) bool { +func addAndReset(hotBuckets *sync.Map, bucketNumber *uint32) func(k, v interface{}) bool { + return func(k, v interface{}) bool { bucket := v.(*int64) if addToBucket(hotBuckets, k.(int), atomic.LoadInt64(bucket)) { atomic.AddUint32(bucketNumber, 1) @@ -1570,7 +1586,7 @@ func addAndReset(hotBuckets *sync.Map, bucketNumber *uint32) func(k, v any) bool } func deleteSyncMap(m *sync.Map) { - m.Range(func(k, v any) bool { + m.Range(func(k, v interface{}) bool { m.Delete(k) return true }) @@ -1578,7 +1594,7 @@ func deleteSyncMap(m *sync.Map) { func findSmallestKey(m *sync.Map) int { result := math.MaxInt32 - m.Range(func(k, v any) bool { + m.Range(func(k, v interface{}) bool { key := k.(int) if key < result { result = key diff --git a/prometheus/registry.go b/prometheus/registry.go index ed0681c8b..b00dc3f8e 100644 --- a/prometheus/registry.go +++ b/prometheus/registry.go @@ -432,6 +432,10 @@ func (r *Registry) MustGather() []*dto.MetricFamily { } // Gather implements Gatherer. +// +// Before Collect, Gather calls CleanupExpired on registered collectors that +// implement ExpiredCleaner and have TTL enabled, so expired Vec children can be +// reclaimed on scrape without touching non-TTL collectors. func (r *Registry) Gather() ([]*dto.MetricFamily, error) { r.mtx.RLock() @@ -476,8 +480,14 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) { for { select { case collector := <-checkedCollectors: + if cleaner, ok := collector.(ttlEnabledCollector); ok && cleaner.ttlEnabled() { + cleaner.CleanupExpired() + } safeErrs.Append((safeCollect(collector, checkedMetricChan))) case collector := <-uncheckedCollectors: + if cleaner, ok := collector.(ttlEnabledCollector); ok && cleaner.ttlEnabled() { + cleaner.CleanupExpired() + } safeErrs.Append(safeCollect(collector, uncheckedMetricChan)) default: return @@ -641,12 +651,10 @@ func WriteToTextfile(filename string, g Gatherer) error { mfs, err := g.Gather() if err != nil { - tmp.Close() return err } for _, mf := range mfs { if _, err := expfmt.MetricFamilyToText(tmp, mf); err != nil { - tmp.Close() return err } } diff --git a/prometheus/summary.go b/prometheus/summary.go index c12b8d13d..1186f4eb7 100644 --- a/prometheus/summary.go +++ b/prometheus/summary.go @@ -164,6 +164,10 @@ type SummaryVecOpts struct { // of labels. Each label value will be constrained with the optional Constraint // function, if provided. VariableLabels ConstrainableLabels + + // TTL, if greater than zero, enables per-child expiration for this vector. + // See MetricVecOpts.TTL for semantics. + TTL time.Duration } // Problem with the sliding-window decay algorithm... The Merge method of @@ -577,6 +581,9 @@ func (v2) NewSummaryVec(opts SummaryVecOpts) *SummaryVec { panic(errQuantileLabelNotAllowed) } } + if opts.TTL < 0 { + panic(fmt.Sprintf("invalid negative ttl: %v", opts.TTL)) + } desc := V2.NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, @@ -584,9 +591,18 @@ func (v2) NewSummaryVec(opts SummaryVecOpts) *SummaryVec { opts.ConstLabels, WithUnit(opts.Unit), ) + newMetric := func(lvs ...string) Metric { + s := newSummary(desc, opts.SummaryOpts, lvs...) + if opts.TTL <= 0 { + return s + } + return newTTLSummary(s) + } return &SummaryVec{ - MetricVec: NewMetricVec(desc, func(lvs ...string) Metric { - return newSummary(desc, opts.SummaryOpts, lvs...) + MetricVec: V2.NewMetricVec(MetricVecOpts{ + Desc: desc, + NewMetric: newMetric, + TTL: opts.TTL, }), } } diff --git a/prometheus/ttl.go b/prometheus/ttl.go new file mode 100644 index 000000000..ebe153598 --- /dev/null +++ b/prometheus/ttl.go @@ -0,0 +1,180 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheus + +import ( + "sync/atomic" + "time" +) + +// ExpiredCleaner is implemented by collectors that support TTL-based cleanup of +// unused children (for example MetricVec with a non-zero Opts.TTL). +// +// Registry.Gather only invokes CleanupExpired on collectors that also report +// TTL as enabled (see ttlEnabled), so vectors with TTL == 0 are not touched on +// the Gather hot path. +type ExpiredCleaner interface { + CleanupExpired() int +} + +// ttlEnabledCollector is the Gather-time check for automatic TTL cleanup. +// ttlEnabled is unexported so only types in this package (e.g. *MetricVec and +// the built-in *Vec types) can opt into automatic cleanup. +type ttlEnabledCollector interface { + ExpiredCleaner + ttlEnabled() bool +} + +// ttlMetric is implemented by decorator wrappers that track last access time. +type ttlMetric interface { + Metric + lastAccessed() int64 + touch() +} + +func nowUnixMilli() int64 { + return time.Now().UnixMilli() +} + +// --- Counter wrapper --- + +type ttlCounter struct { + Counter + lastAccessedTs atomic.Int64 +} + +func newTTLCounter(c Counter) *ttlCounter { + tc := &ttlCounter{Counter: c} + tc.lastAccessedTs.Store(nowUnixMilli()) + return tc +} + +func (c *ttlCounter) Inc() { + c.Counter.Inc() + c.lastAccessedTs.Store(nowUnixMilli()) +} + +func (c *ttlCounter) Add(v float64) { + c.Counter.Add(v) + c.lastAccessedTs.Store(nowUnixMilli()) +} + +func (c *ttlCounter) AddWithExemplar(v float64, e Labels) { + if ea, ok := c.Counter.(ExemplarAdder); ok { + ea.AddWithExemplar(v, e) + } else { + c.Counter.Add(v) + } + c.lastAccessedTs.Store(nowUnixMilli()) +} + +func (c *ttlCounter) lastAccessed() int64 { return c.lastAccessedTs.Load() } +func (c *ttlCounter) touch() { c.lastAccessedTs.Store(nowUnixMilli()) } + +// --- Gauge wrapper --- + +type ttlGauge struct { + Gauge + lastAccessedTs atomic.Int64 +} + +func newTTLGauge(g Gauge) *ttlGauge { + tg := &ttlGauge{Gauge: g} + tg.lastAccessedTs.Store(nowUnixMilli()) + return tg +} + +func (g *ttlGauge) Set(v float64) { + g.Gauge.Set(v) + g.lastAccessedTs.Store(nowUnixMilli()) +} + +func (g *ttlGauge) Inc() { + g.Gauge.Inc() + g.lastAccessedTs.Store(nowUnixMilli()) +} + +func (g *ttlGauge) Dec() { + g.Gauge.Dec() + g.lastAccessedTs.Store(nowUnixMilli()) +} + +func (g *ttlGauge) Add(v float64) { + g.Gauge.Add(v) + g.lastAccessedTs.Store(nowUnixMilli()) +} + +func (g *ttlGauge) Sub(v float64) { + g.Gauge.Sub(v) + g.lastAccessedTs.Store(nowUnixMilli()) +} + +func (g *ttlGauge) SetToCurrentTime() { + g.Gauge.SetToCurrentTime() + g.lastAccessedTs.Store(nowUnixMilli()) +} + +func (g *ttlGauge) lastAccessed() int64 { return g.lastAccessedTs.Load() } +func (g *ttlGauge) touch() { g.lastAccessedTs.Store(nowUnixMilli()) } + +// --- Histogram wrapper --- + +type ttlHistogram struct { + Histogram + lastAccessedTs atomic.Int64 +} + +func newTTLHistogram(h Histogram) *ttlHistogram { + th := &ttlHistogram{Histogram: h} + th.lastAccessedTs.Store(nowUnixMilli()) + return th +} + +func (h *ttlHistogram) Observe(v float64) { + h.Histogram.Observe(v) + h.lastAccessedTs.Store(nowUnixMilli()) +} + +func (h *ttlHistogram) ObserveWithExemplar(v float64, e Labels) { + if eo, ok := h.Histogram.(ExemplarObserver); ok { + eo.ObserveWithExemplar(v, e) + } else { + h.Histogram.Observe(v) + } + h.lastAccessedTs.Store(nowUnixMilli()) +} + +func (h *ttlHistogram) lastAccessed() int64 { return h.lastAccessedTs.Load() } +func (h *ttlHistogram) touch() { h.lastAccessedTs.Store(nowUnixMilli()) } + +// --- Summary wrapper --- + +type ttlSummary struct { + Summary + lastAccessedTs atomic.Int64 +} + +func newTTLSummary(s Summary) *ttlSummary { + ts := &ttlSummary{Summary: s} + ts.lastAccessedTs.Store(nowUnixMilli()) + return ts +} + +func (s *ttlSummary) Observe(v float64) { + s.Summary.Observe(v) + s.lastAccessedTs.Store(nowUnixMilli()) +} + +func (s *ttlSummary) lastAccessed() int64 { return s.lastAccessedTs.Load() } +func (s *ttlSummary) touch() { s.lastAccessedTs.Store(nowUnixMilli()) } diff --git a/prometheus/vec.go b/prometheus/vec.go index 121d2a963..60acf006a 100644 --- a/prometheus/vec.go +++ b/prometheus/vec.go @@ -16,6 +16,7 @@ package prometheus import ( "fmt" "sync" + "time" "github.com/prometheus/common/model" ) @@ -43,19 +44,76 @@ type MetricVec struct { hashAddByte func(h uint64, b byte) uint64 } -// NewMetricVec returns an initialized metricVec. +// MetricVecOpts bundles the options to create a MetricVec. +type MetricVecOpts struct { + Desc *Desc + NewMetric func(lvs ...string) Metric + // TTL, if greater than zero, enables per-child expiration. Children that + // have not been accessed for longer than TTL are omitted from Collect and + // can be removed via CleanupExpired (also invoked automatically by + // Registry.Gather for collectors that implement ExpiredCleaner). + // + // A negative TTL is invalid and causes a panic. TTL of zero disables + // expiration (identical to NewMetricVec). + // + // Access includes GetMetricWith / GetMetricWithLabelValues and, when using + // the built-in CounterVec / GaugeVec / HistogramVec / SummaryVec with TTL, + // mutating methods on cached children (Inc, Add, Set, Observe, …). Caching + // a child and never calling those methods (nor looking it up again) lets + // the child expire; the cached handle then behaves like after Delete — + // updates are not exported until the label set is looked up again. See + // Delete docs. + // + // When TTL > 0, NewMetric must return a Metric that implements the internal + // TTL touch hooks (the built-in *Vec constructors wrap children for you). + // Passing a plain Metric panics when the child is created. + // + // TTL > 0 adds a small per-child wrapper allocation and a timestamp update + // on each mutating call; TTL == 0 keeps the default Vec path with neither. + // + // If metrics are never scraped, call CleanupExpired periodically (or rely + // on Gather) so expired children can be reclaimed; there is no background + // goroutine. + TTL time.Duration +} + +// NewMetricVec returns an initialized MetricVec with no TTL. func NewMetricVec(desc *Desc, newMetric func(lvs ...string) Metric) *MetricVec { + return V2.NewMetricVec(MetricVecOpts{Desc: desc, NewMetric: newMetric}) +} + +// NewMetricVec returns an initialized MetricVec. See MetricVecOpts. +func (v2) NewMetricVec(opts MetricVecOpts) *MetricVec { + if opts.TTL < 0 { + panic(fmt.Sprintf("invalid negative ttl: %v", opts.TTL)) + } return &MetricVec{ metricMap: &metricMap{ metrics: map[uint64][]metricWithLabelValues{}, - desc: desc, - newMetric: newMetric, + desc: opts.Desc, + newMetric: opts.NewMetric, + ttl: opts.TTL, }, hashAdd: hashAdd, hashAddByte: hashAddByte, } } +// CleanupExpired removes all children that have not been accessed within the +// configured TTL. It returns the number of children removed. If TTL is not +// configured (zero), this is a no-op and returns 0. +// +// Registry.Gather invokes CleanupExpired only for collectors with TTL enabled. +// If scrapes are rare or absent, call CleanupExpired periodically yourself; +// client_golang does not start a background cleaner. +func (m *MetricVec) CleanupExpired() int { + return m.cleanupExpired() +} + +func (m *MetricVec) ttlEnabled() bool { + return m.ttl > 0 +} + // DeleteLabelValues removes the metric where the variable labels are the same // as those passed in as labels (same order as the VariableLabels in Desc). It // returns true if a metric was deleted. @@ -71,6 +129,10 @@ func NewMetricVec(desc *Desc, newMetric func(lvs ...string) Metric) *MetricVec { // latter has a much more readable (albeit more verbose) syntax, but it comes // with a performance overhead (for creating and processing the Labels map). // See also the CounterVec example. +// +// Callers that cache a child and keep using it after deletion (or after +// CleanupExpired under TTL) update a detached metric that is no longer +// exported until the same label set is looked up again. func (m *MetricVec) DeleteLabelValues(lvs ...string) bool { lvs = constrainLabelValues(m.desc, lvs, m.curry) @@ -321,6 +383,7 @@ type metricMap struct { metrics map[uint64][]metricWithLabelValues desc *Desc newMetric func(labelValues ...string) Metric + ttl time.Duration // 0 disables TTL; see MetricVecOpts.TTL. } // Describe implements Collector. It will send exactly one Desc to the provided @@ -334,13 +397,59 @@ func (m *metricMap) Collect(ch chan<- Metric) { m.mtx.RLock() defer m.mtx.RUnlock() + var deadline int64 + if m.ttl > 0 { + deadline = time.Now().Add(-m.ttl).UnixMilli() + } for _, metrics := range m.metrics { for _, metric := range metrics { + if m.ttl > 0 { + if tm, ok := metric.metric.(ttlMetric); ok && tm.lastAccessed() < deadline { + continue + } + } ch <- metric.metric } } } +func (m *metricMap) cleanupExpired() int { + if m.ttl <= 0 { + return 0 + } + deadline := time.Now().Add(-m.ttl).UnixMilli() + m.mtx.Lock() + defer m.mtx.Unlock() + + var numDeleted int + for h, metrics := range m.metrics { + origLen := len(metrics) + remaining := metrics[:0] + for i := range metrics { + if tm, ok := metrics[i].metric.(ttlMetric); ok && tm.lastAccessed() < deadline { + numDeleted++ + continue + } + remaining = append(remaining, metrics[i]) + } + if len(remaining) == 0 { + delete(m.metrics, h) + } else { + for i := len(remaining); i < origLen; i++ { + metrics[i] = metricWithLabelValues{} + } + m.metrics[h] = remaining + } + } + return numDeleted +} + +func touchIfTTL(metric Metric) { + if tm, ok := metric.(ttlMetric); ok { + tm.touch() + } +} + // Reset deletes all metrics in this vector. func (m *metricMap) Reset() { m.mtx.Lock() @@ -495,6 +604,9 @@ func (m *metricMap) getOrCreateMetricWithLabelValues( metric, ok := m.getMetricWithHashAndLabelValues(hash, lvs, curry) m.mtx.RUnlock() if ok { + if m.ttl > 0 { + touchIfTTL(metric) + } return metric } @@ -504,7 +616,10 @@ func (m *metricMap) getOrCreateMetricWithLabelValues( if !ok { inlinedLVs := inlineLabelValues(lvs, curry) metric = m.newMetric(inlinedLVs...) + m.requireTTLMetric(metric) m.metrics[hash] = append(m.metrics[hash], metricWithLabelValues{values: inlinedLVs, metric: metric}) + } else if m.ttl > 0 { + touchIfTTL(metric) } return metric } @@ -520,6 +635,9 @@ func (m *metricMap) getOrCreateMetricWithLabels( metric, ok := m.getMetricWithHashAndLabels(hash, labels, curry) m.mtx.RUnlock() if ok { + if m.ttl > 0 { + touchIfTTL(metric) + } return metric } @@ -529,11 +647,23 @@ func (m *metricMap) getOrCreateMetricWithLabels( if !ok { lvs := extractLabelValues(m.desc, labels, curry) metric = m.newMetric(lvs...) + m.requireTTLMetric(metric) m.metrics[hash] = append(m.metrics[hash], metricWithLabelValues{values: lvs, metric: metric}) + } else if m.ttl > 0 { + touchIfTTL(metric) } return metric } +func (m *metricMap) requireTTLMetric(metric Metric) { + if m.ttl <= 0 { + return + } + if _, ok := metric.(ttlMetric); !ok { + panic("MetricVec with TTL > 0 requires NewMetric to return a TTL-aware Metric; use CounterVec/GaugeVec/HistogramVec/SummaryVec Opts.TTL or wrap the Metric yourself") + } +} + // getMetricWithHashAndLabelValues gets a metric while handling possible // collisions in the hash space. Must be called while holding the read mutex. func (m *metricMap) getMetricWithHashAndLabelValues( @@ -659,7 +789,7 @@ func inlineLabelValues(lvs []string, curry []curriedLabelValue) []string { } var labelsPool = &sync.Pool{ - New: func() any { + New: func() interface{} { return make(Labels) }, } diff --git a/prometheus/vec_test.go b/prometheus/vec_test.go index 03223f2f6..1a6504c06 100644 --- a/prometheus/vec_test.go +++ b/prometheus/vec_test.go @@ -18,6 +18,8 @@ import ( "reflect" "strconv" "testing" + "testing/synctest" + "time" dto "github.com/prometheus/client_model/go" ) @@ -48,11 +50,11 @@ func TestDeleteWithCollisions(t *testing.T) { func TestDeleteWithConstraints(t *testing.T) { vec := V2.NewGaugeVec(GaugeVecOpts{ - GaugeOpts{ + GaugeOpts: GaugeOpts{ Name: "test", Help: "helpless", }, - ConstrainedLabels{ + VariableLabels: ConstrainedLabels{ {Name: "l1"}, {Name: "l2", Constraint: func(s string) string { return "x" + s }}, }, @@ -116,11 +118,11 @@ func TestDeleteLabelValuesWithCollisions(t *testing.T) { func TestDeleteLabelValuesWithConstraints(t *testing.T) { vec := V2.NewGaugeVec(GaugeVecOpts{ - GaugeOpts{ + GaugeOpts: GaugeOpts{ Name: "test", Help: "helpless", }, - ConstrainedLabels{ + VariableLabels: ConstrainedLabels{ {Name: "l1"}, {Name: "l2", Constraint: func(s string) string { return "x" + s }}, }, @@ -168,11 +170,11 @@ func TestDeletePartialMatch(t *testing.T) { func TestDeletePartialMatchWithConstraints(t *testing.T) { vec := V2.NewGaugeVec(GaugeVecOpts{ - GaugeOpts{ + GaugeOpts: GaugeOpts{ Name: "test", Help: "helpless", }, - ConstrainedLabels{ + VariableLabels: ConstrainedLabels{ {Name: "l1"}, {Name: "l2", Constraint: func(s string) string { return "x" + s }}, {Name: "l3"}, @@ -344,11 +346,11 @@ func testMetricVec(t *testing.T, vec *GaugeVec) { func TestMetricVecWithConstraints(t *testing.T) { constraint := func(s string) string { return "x" + s } vec := V2.NewGaugeVec(GaugeVecOpts{ - GaugeOpts{ + GaugeOpts: GaugeOpts{ Name: "test", Help: "helpless", }, - ConstrainedLabels{ + VariableLabels: ConstrainedLabels{ {Name: "l1"}, {Name: "l2", Constraint: constraint}, }, @@ -474,11 +476,11 @@ func TestCurryVecWithConstraints(t *testing.T) { constraint := func(s string) string { return "x" + s } t.Run("constrainedLabels overlap variableLabels", func(t *testing.T) { vec := V2.NewCounterVec(CounterVecOpts{ - CounterOpts{ + CounterOpts: CounterOpts{ Name: "test", Help: "helpless", }, - ConstrainedLabels{ + VariableLabels: ConstrainedLabels{ {Name: "one"}, {Name: "two"}, {Name: "three", Constraint: constraint}, @@ -489,11 +491,11 @@ func TestCurryVecWithConstraints(t *testing.T) { t.Run("constrainedLabels reducing cardinality", func(t *testing.T) { constraint := func(s string) string { return "x" } vec := V2.NewCounterVec(CounterVecOpts{ - CounterOpts{ + CounterOpts: CounterOpts{ Name: "test", Help: "helpless", }, - ConstrainedLabels{ + VariableLabels: ConstrainedLabels{ {Name: "one"}, {Name: "two"}, {Name: "three", Constraint: constraint}, @@ -1004,3 +1006,427 @@ func benchmarkMetricVecWithLabelValues(b *testing.B, labels map[string][]string) vec.WithLabelValues(values...) } } + +func collectCount(c Collector) int { + ch := make(chan Metric, 100) + c.Collect(ch) + close(ch) + n := 0 + for range ch { + n++ + } + return n +} + +func TestTTLCounterVec(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ttl := 100 * time.Millisecond + vec := V2.NewCounterVec(CounterVecOpts{ + CounterOpts: CounterOpts{Name: "test_ttl_counter", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"code"}), + TTL: ttl, + }) + + vec.WithLabelValues("200").Add(1) + vec.WithLabelValues("404").Add(1) + + if n := collectCount(vec); n != 2 { + t.Fatalf("expected 2 metrics, got %d", n) + } + + time.Sleep(ttl + 50*time.Millisecond) + + if n := collectCount(vec); n != 0 { + t.Fatalf("expected 0 metrics after TTL, got %d", n) + } + + cleaned := vec.CleanupExpired() + if cleaned != 2 { + t.Fatalf("expected 2 cleaned, got %d", cleaned) + } + }) +} + +func TestTTLGaugeVec(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ttl := 100 * time.Millisecond + vec := V2.NewGaugeVec(GaugeVecOpts{ + GaugeOpts: GaugeOpts{Name: "test_ttl_gauge", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"method"}), + TTL: ttl, + }) + + vec.WithLabelValues("GET").Set(10) + vec.WithLabelValues("POST").Set(20) + + if n := collectCount(vec); n != 2 { + t.Fatalf("expected 2 metrics, got %d", n) + } + + time.Sleep(ttl + 50*time.Millisecond) + // Touch only one + vec.WithLabelValues("GET").Set(30) + + if n := collectCount(vec); n != 1 { + t.Fatalf("expected 1 metric after partial TTL, got %d", n) + } + + cleaned := vec.CleanupExpired() + if cleaned != 1 { + t.Fatalf("expected 1 cleaned, got %d", cleaned) + } + }) +} + +func TestTTLHistogramVec(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ttl := 100 * time.Millisecond + vec := V2.NewHistogramVec(HistogramVecOpts{ + HistogramOpts: HistogramOpts{Name: "test_ttl_histo", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"status"}), + TTL: ttl, + }) + + vec.WithLabelValues("ok").Observe(0.5) + vec.WithLabelValues("err").Observe(1.5) + + if n := collectCount(vec); n != 2 { + t.Fatalf("expected 2 metrics, got %d", n) + } + + time.Sleep(ttl + 50*time.Millisecond) + + if n := collectCount(vec); n != 0 { + t.Fatalf("expected 0 metrics after TTL, got %d", n) + } + }) +} + +func TestTTLCachedChildKeepsAlive(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ttl := 100 * time.Millisecond + vec := V2.NewHistogramVec(HistogramVecOpts{ + HistogramOpts: HistogramOpts{Name: "test_ttl_cached", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"status"}), + TTL: ttl, + }) + + cached := vec.WithLabelValues("ok") + cached.Observe(0.5) + + // Hot path: only Observe on the cached child (no WithLabelValues). + for i := 0; i < 5; i++ { + time.Sleep(80 * time.Millisecond) + cached.Observe(0.1) + } + + if n := collectCount(vec); n != 1 { + t.Fatalf("expected 1 metric still alive via cached Observe, got %d", n) + } + }) +} + +func TestTTLRefreshPreventsExpiration(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ttl := 150 * time.Millisecond + vec := V2.NewCounterVec(CounterVecOpts{ + CounterOpts: CounterOpts{Name: "test_ttl_refresh", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"code"}), + TTL: ttl, + }) + + cached := vec.WithLabelValues("200") + cached.Add(1) + + for i := 0; i < 5; i++ { + time.Sleep(80 * time.Millisecond) + cached.Add(1) + } + + if n := collectCount(vec); n != 1 { + t.Fatalf("expected 1 metric still alive, got %d", n) + } + }) +} + +func TestMetricVecOptsTTLZeroAndNegative(t *testing.T) { + desc := NewDesc("test", "help", []string{"l"}, nil) + newMetric := func(lvs ...string) Metric { return &counter{} } + + mv0 := V2.NewMetricVec(MetricVecOpts{Desc: desc, NewMetric: newMetric, TTL: 0}) + if mv0.ttl != 0 { + t.Fatal("ttl==0 should leave metricMap.ttl at 0") + } + if cleaned := mv0.CleanupExpired(); cleaned != 0 { + t.Fatalf("expected 0 cleaned, got %d", cleaned) + } + + defer func() { + if recover() == nil { + t.Fatal("expected panic for negative ttl") + } + }() + V2.NewMetricVec(MetricVecOpts{Desc: desc, NewMetric: newMetric, TTL: -time.Second}) +} + +func TestTTLZeroMeansNoExpiration(t *testing.T) { + vec := NewCounterVec(CounterOpts{ + Name: "test_no_ttl", + Help: "test", + }, []string{"code"}) + + vec.WithLabelValues("200").Add(1) + + cleaned := vec.CleanupExpired() + if cleaned != 0 { + t.Fatalf("expected 0 cleaned with no TTL, got %d", cleaned) + } + + if n := collectCount(vec); n != 1 { + t.Fatalf("expected 1 metric, got %d", n) + } +} + +func TestTTLWithGetMetricWith(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ttl := 100 * time.Millisecond + vec := V2.NewGaugeVec(GaugeVecOpts{ + GaugeOpts: GaugeOpts{Name: "test_ttl_getmetricwith", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"method"}), + TTL: ttl, + }) + + g, err := vec.GetMetricWith(Labels{"method": "GET"}) + if err != nil { + t.Fatal(err) + } + g.Set(42) + + time.Sleep(ttl + 50*time.Millisecond) + + if n := collectCount(vec); n != 0 { + t.Fatalf("expected 0 after TTL, got %d", n) + } + + g, _ = vec.GetMetricWith(Labels{"method": "GET"}) + g.Set(99) + + if n := collectCount(vec); n != 1 { + t.Fatalf("expected 1 after re-access, got %d", n) + } + }) +} + +func TestRegistryGatherCleansExpired(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ttl := 80 * time.Millisecond + reg := NewRegistry() + vec := V2.NewCounterVec(CounterVecOpts{ + CounterOpts: CounterOpts{Name: "ttl_reg_gather", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"code"}), + TTL: ttl, + }) + reg.MustRegister(vec) + + vec.WithLabelValues("200").Add(1) + if n := collectCount(vec); n != 1 { + t.Fatalf("expected 1 metric before sleep, got %d", n) + } + + time.Sleep(ttl + 40*time.Millisecond) + + if _, err := reg.Gather(); err != nil { + t.Fatal(err) + } + + if n := collectCount(vec); n != 0 { + t.Fatalf("expected 0 metrics after Gather cleanup, got %d", n) + } + if cleaned := vec.CleanupExpired(); cleaned != 0 { + t.Fatalf("expected nothing left to clean, got %d", cleaned) + } + }) +} + +func TestTTLZeroHasNoWrapper(t *testing.T) { + vec := NewCounterVec(CounterOpts{Name: "ttl_zero_wrap", Help: "test"}, []string{"code"}) + c := vec.WithLabelValues("200") + if _, ok := c.(*ttlCounter); ok { + t.Fatal("TTL==0 must not wrap children in ttlCounter") + } + if _, ok := c.(*counter); !ok { + t.Fatalf("TTL==0 child should be *counter, got %T", c) + } + if vec.ttlEnabled() { + t.Fatal("TTL==0 vector must not report ttlEnabled") + } +} + +func TestTTLWrapsChildren(t *testing.T) { + vec := V2.NewCounterVec(CounterVecOpts{ + CounterOpts: CounterOpts{Name: "ttl_wrap", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"code"}), + TTL: time.Minute, + }) + c := vec.WithLabelValues("200") + if _, ok := c.(*ttlCounter); !ok { + t.Fatalf("TTL>0 child should be *ttlCounter, got %T", c) + } + if !vec.ttlEnabled() { + t.Fatal("TTL>0 vector must report ttlEnabled") + } + + gvec := V2.NewGaugeVec(GaugeVecOpts{ + GaugeOpts: GaugeOpts{Name: "ttl_wrap_g", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"code"}), + TTL: time.Minute, + }) + if _, ok := gvec.WithLabelValues("200").(*ttlGauge); !ok { + t.Fatal("expected *ttlGauge") + } + + hvec := V2.NewHistogramVec(HistogramVecOpts{ + HistogramOpts: HistogramOpts{Name: "ttl_wrap_h", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"code"}), + TTL: time.Minute, + }) + if _, ok := hvec.WithLabelValues("200").(*ttlHistogram); !ok { + t.Fatal("expected *ttlHistogram") + } + + svec := V2.NewSummaryVec(SummaryVecOpts{ + SummaryOpts: SummaryOpts{Name: "ttl_wrap_s", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"code"}), + TTL: time.Minute, + }) + if _, ok := svec.WithLabelValues("200").(*ttlSummary); !ok { + t.Fatal("expected *ttlSummary") + } +} + +func TestTTLCustomMetricVecRequiresTTLMetric(t *testing.T) { + desc := NewDesc("ttl_custom", "help", []string{"l"}, nil) + mv := V2.NewMetricVec(MetricVecOpts{ + Desc: desc, + NewMetric: func(lvs ...string) Metric { return &counter{} }, + TTL: time.Minute, + }) + defer func() { + if recover() == nil { + t.Fatal("expected panic when NewMetric does not return a ttlMetric") + } + }() + _, _ = mv.GetMetricWithLabelValues("x") +} + +func TestTTLOrphanedCachedHandleAfterCleanup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ttl := 50 * time.Millisecond + vec := V2.NewCounterVec(CounterVecOpts{ + CounterOpts: CounterOpts{Name: "ttl_orphan", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"code"}), + TTL: ttl, + }) + + cached := vec.WithLabelValues("200") + cached.Add(1) + + time.Sleep(ttl + 20*time.Millisecond) + if n := vec.CleanupExpired(); n != 1 { + t.Fatalf("expected 1 cleaned, got %d", n) + } + + // Cached handle is detached: updates are not exported. + cached.Add(5) + if n := collectCount(vec); n != 0 { + t.Fatalf("expected 0 exported after orphan update, got %d", n) + } + + // Re-lookup creates a fresh child. + fresh := vec.WithLabelValues("200") + if fresh == cached { + t.Fatal("expected a new child metric after cleanup") + } + fresh.Add(1) + if n := collectCount(vec); n != 1 { + t.Fatalf("expected 1 after re-lookup, got %d", n) + } + }) +} + +func TestTTLSummaryVec(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ttl := 100 * time.Millisecond + vec := V2.NewSummaryVec(SummaryVecOpts{ + SummaryOpts: SummaryOpts{Name: "test_ttl_summary", Help: "test"}, + VariableLabels: UnconstrainedLabels([]string{"code"}), + TTL: ttl, + }) + + cached := vec.WithLabelValues("ok") + cached.Observe(0.5) + + for i := 0; i < 5; i++ { + time.Sleep(80 * time.Millisecond) + cached.Observe(0.1) + } + if n := collectCount(vec); n != 1 { + t.Fatalf("expected 1 metric via cached Observe, got %d", n) + } + + time.Sleep(ttl + 50*time.Millisecond) + if n := collectCount(vec); n != 0 { + t.Fatalf("expected 0 after idle TTL, got %d", n) + } + }) +} + +// cleanupCallSpy tracks whether Gather invoked CleanupExpired. +type cleanupCallSpy struct { + selfCollector + desc *Desc + calls int + enableTTL bool +} + +func (s *cleanupCallSpy) Desc() *Desc { return s.desc } + +func (s *cleanupCallSpy) Write(out *dto.Metric) error { + return populateMetric(GaugeValue, 0, nil, nil, out, nil) +} + +func (s *cleanupCallSpy) CleanupExpired() int { + s.calls++ + return 0 +} + +func (s *cleanupCallSpy) ttlEnabled() bool { return s.enableTTL } + +func TestRegistryGatherSkipsCleanupWhenTTLDisabled(t *testing.T) { + reg := NewRegistry() + + disabled := &cleanupCallSpy{desc: NewDesc("spy_disabled", "help", nil, nil), enableTTL: false} + disabled.init(disabled) + enabled := &cleanupCallSpy{desc: NewDesc("spy_enabled", "help", nil, nil), enableTTL: true} + enabled.init(enabled) + + reg.MustRegister(disabled, enabled) + if _, err := reg.Gather(); err != nil { + t.Fatal(err) + } + if disabled.calls != 0 { + t.Fatalf("Gather must not CleanupExpired when ttlEnabled is false, got %d calls", disabled.calls) + } + if enabled.calls != 1 { + t.Fatalf("Gather must CleanupExpired when ttlEnabled is true, got %d calls", enabled.calls) + } + + // Non-TTL built-in vecs must not report ttlEnabled. + plain := NewCounterVec(CounterOpts{Name: "plain_gather", Help: "test"}, []string{"c"}) + if _, ok := any(plain).(ttlEnabledCollector); !ok { + t.Fatal("CounterVec should satisfy ttlEnabledCollector via MetricVec") + } + if plain.ttlEnabled() { + t.Fatal("plain CounterVec must not be ttlEnabled") + } +}