diff --git a/.changeset/proud-shrimps-clean.md b/.changeset/proud-shrimps-clean.md new file mode 100644 index 0000000000..d28eff301a --- /dev/null +++ b/.changeset/proud-shrimps-clean.md @@ -0,0 +1,6 @@ +--- +"@hyperdx/common-utils": minor +"@hyperdx/otel-collector": minor +--- + +feat: update metrics schema for more efficient PK and time pruning diff --git a/docker/otel-collector/schema/seed/00003_otel_metrics.sql b/docker/otel-collector/schema/seed/00003_otel_metrics.sql index b5e0b1d2a0..9a9622755b 100644 --- a/docker/otel-collector/schema/seed/00003_otel_metrics.sql +++ b/docker/otel-collector/schema/seed/00003_otel_metrics.sql @@ -9,16 +9,16 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_gauge `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), `ScopeSchemaUrl` String CODEC(ZSTD(1)), `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), + `MetricName` LowCardinality(String) CODEC(ZSTD(1)), `MetricDescription` String CODEC(ZSTD(1)), `MetricUnit` String CODEC(ZSTD(1)), `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(1)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(1)), `Value` Float64 CODEC(ZSTD(1)), `Flags` UInt32 CODEC(ZSTD(1)), `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)), + `Exemplars.TimeUnix` Array(DateTime) CODEC(ZSTD(1)), `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)), `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)), `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)), @@ -27,11 +27,12 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_gauge INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${TABLES_TTL} SETTINGS ttl_only_drop_parts = 1; @@ -46,16 +47,16 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_sum `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), `ScopeSchemaUrl` String CODEC(ZSTD(1)), `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), + `MetricName` LowCardinality(String) CODEC(ZSTD(1)), `MetricDescription` String CODEC(ZSTD(1)), `MetricUnit` String CODEC(ZSTD(1)), `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(1)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(1)), `Value` Float64 CODEC(ZSTD(1)), `Flags` UInt32 CODEC(ZSTD(1)), `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)), + `Exemplars.TimeUnix` Array(DateTime) CODEC(ZSTD(1)), `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)), `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)), `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)), @@ -66,11 +67,12 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_sum INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${TABLES_TTL} SETTINGS ttl_only_drop_parts = 1; -- +goose StatementEnd @@ -86,18 +88,18 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_histogram `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), `ScopeSchemaUrl` String CODEC(ZSTD(1)), `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), + `MetricName` LowCardinality(String) CODEC(ZSTD(1)), `MetricDescription` String CODEC(ZSTD(1)), `MetricUnit` String CODEC(ZSTD(1)), `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(1)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(1)), `Count` UInt64 CODEC(Delta(8), ZSTD(1)), `Sum` Float64 CODEC(ZSTD(1)), `BucketCounts` Array(UInt64) CODEC(ZSTD(1)), `ExplicitBounds` Array(Float64) CODEC(ZSTD(1)), `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)), + `Exemplars.TimeUnix` Array(DateTime) CODEC(ZSTD(1)), `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)), `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)), `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)), @@ -110,11 +112,12 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_histogram INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${TABLES_TTL} SETTINGS ttl_only_drop_parts = 1; -- +goose StatementEnd @@ -130,12 +133,12 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_exponential_histogram `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), `ScopeSchemaUrl` String CODEC(ZSTD(1)), `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), + `MetricName` LowCardinality(String) CODEC(ZSTD(1)), `MetricDescription` String CODEC(ZSTD(1)), `MetricUnit` String CODEC(ZSTD(1)), `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(1)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(1)), `Count` UInt64 CODEC(Delta(8), ZSTD(1)), `Sum` Float64 CODEC(ZSTD(1)), `Scale` Int32 CODEC(ZSTD(1)), @@ -145,7 +148,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_exponential_histogram `NegativeOffset` Int32 CODEC(ZSTD(1)), `NegativeBucketCounts` Array(UInt64) CODEC(ZSTD(1)), `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)), + `Exemplars.TimeUnix` Array(DateTime) CODEC(ZSTD(1)), `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)), `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)), `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)), @@ -158,11 +161,12 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_exponential_histogram INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${TABLES_TTL} SETTINGS ttl_only_drop_parts = 1; -- +goose StatementEnd @@ -178,12 +182,12 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_summary `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), `ScopeSchemaUrl` String CODEC(ZSTD(1)), `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), + `MetricName` LowCardinality(String) CODEC(ZSTD(1)), `MetricDescription` String CODEC(ZSTD(1)), `MetricUnit` String CODEC(ZSTD(1)), `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(1)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(1)), `Count` UInt64 CODEC(Delta(8), ZSTD(1)), `Sum` Float64 CODEC(ZSTD(1)), `ValueAtQuantiles.Quantile` Array(Float64) CODEC(ZSTD(1)), @@ -194,11 +198,12 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_summary INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${TABLES_TTL} SETTINGS ttl_only_drop_parts = 1; -- +goose StatementEnd diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 9159687546..edd4e3a641 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -2095,6 +2095,17 @@ describe('renderChartConfig', () => { dateRangeEndInclusive: false, expected: `(toDate(timestamp) >= toDate(fromUnixTimestamp64Milli(${new Date('2025-02-12 03:53:38Z').getTime()})) AND toDate(timestamp) <= toDate(fromUnixTimestamp64Milli(${new Date('2025-02-12 04:08:38Z').getTime()})))AND(timestamp > fromUnixTimestamp64Milli(${new Date('2025-02-12 03:53:38Z').getTime()}) AND timestamp < fromUnixTimestamp64Milli(${new Date('2025-02-12 04:08:38Z').getTime()}))`, }, + { + description: + 'wraps includedDataInterval-expanded bound in toStartOf when PK has toStartOfHour(col) — prevents dropping rows whose hour is before the raw expanded start', + timestampValueExpression: 'timestamp, toStartOfHour(timestamp)', + dateRange: [ + new Date('2025-02-12 04:00:00Z'), + new Date('2025-02-12 04:20:00Z'), + ], + includedDataInterval: '5 minute', + expected: `(timestamp >= toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-12 04:00:00Z').getTime()}), INTERVAL 5 minute) - INTERVAL 5 minute AND timestamp <= toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-12 04:20:00Z').getTime()}), INTERVAL 5 minute) + INTERVAL 5 minute)AND(toStartOfHour(timestamp) >= toStartOfHour(toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-12 04:00:00Z').getTime()}), INTERVAL 5 minute) - INTERVAL 5 minute) AND toStartOfHour(timestamp) <= toStartOfHour(toStartOfInterval(fromUnixTimestamp64Milli(${new Date('2025-02-12 04:20:00Z').getTime()}), INTERVAL 5 minute) + INTERVAL 5 minute))`, + }, ]; beforeEach(() => { diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 4f5c8a48d5..9e288fdd2c 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -896,17 +896,21 @@ export async function timeFilterExpr({ ); } - const startTimeCond = includedDataInterval + const rawStartBound = includedDataInterval ? chSql`toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: startTime }}), INTERVAL ${includedDataInterval}) - INTERVAL ${includedDataInterval}` - : toStartOf - ? chSql`${toStartOf.function}(fromUnixTimestamp64Milli(${{ Int64: startTime }})${toStartOf.formattedRemainingArgs})` - : chSql`fromUnixTimestamp64Milli(${{ Int64: startTime }})`; + : chSql`fromUnixTimestamp64Milli(${{ Int64: startTime }})`; - const endTimeCond = includedDataInterval + const rawEndBound = includedDataInterval ? chSql`toStartOfInterval(fromUnixTimestamp64Milli(${{ Int64: endTime }}), INTERVAL ${includedDataInterval}) + INTERVAL ${includedDataInterval}` - : toStartOf - ? chSql`${toStartOf.function}(fromUnixTimestamp64Milli(${{ Int64: endTime }})${toStartOf.formattedRemainingArgs})` - : chSql`fromUnixTimestamp64Milli(${{ Int64: endTime }})`; + : chSql`fromUnixTimestamp64Milli(${{ Int64: endTime }})`; + + const startTimeCond = toStartOf + ? chSql`${toStartOf.function}(${rawStartBound}${toStartOf.formattedRemainingArgs})` + : rawStartBound; + + const endTimeCond = toStartOf + ? chSql`${toStartOf.function}(${rawEndBound}${toStartOf.formattedRemainingArgs})` + : rawEndBound; const isDateType = columnMeta?.type === 'Date' || isToDateExpr;