Skip to content
Merged
30 changes: 30 additions & 0 deletions packages/app/cypress/component/inference-chart-controls.cy.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,36 @@ describe('Inference ChartControls', () => {
cy.get('@setSelectedYAxisMetric').should('have.been.calledOnce');
});

it('lists and selects the schema-v2 derived axes in the Measured Energy group', () => {
const options = [
{
key: 'y_measuredJPerSuccessfulQuery',
label: 'Measured Joules per Successful Query',
},
{
key: 'y_measuredWhPerSuccessfulQuery',
label: 'Measured Watt-hours per Successful Query',
},
{
key: 'y_measuredPowerPercentTdp',
label: 'Measured Average Power as Percent of TDP',
},
];

for (const option of options) {
cy.get('[data-testid="yaxis-metric-selector"]').click();
cy.contains('Measured Energy')
.parent()
.within(() => {
cy.contains('[role="option"]', option.label)
.scrollIntoView()
.should('be.visible')
.click();
});
cy.get('@setSelectedYAxisMetric').should('have.been.calledWith', option.key);
}
});

it('hides the GPU comparison section when no GPUs are selected', () => {
// Default mock: selectedGPUs = [] — GPU date range pickers should not render
cy.contains('Comparison Date Range').should('not.exist');
Expand Down
49 changes: 49 additions & 0 deletions packages/app/src/app/api/unofficial-run/route.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,55 @@ describe('normalizeArtifactRows', () => {
expect(m.mean_e2el).toBe(1.5);
});

it.each([
{
name: 'boolean verdict and junk-suffixed schema',
input: { power_valid: true, power_metric_schema_version: '2garbage' },
expectedVerdict: 0,
expectedSchema: undefined,
},
{
name: 'garbage verdict and valid numeric schema',
input: { power_valid: 'garbage', power_metric_schema_version: 2 },
expectedVerdict: 0,
expectedSchema: 2,
},
{
name: 'canonical numeric strings',
input: { power_valid: '1', power_metric_schema_version: '2' },
expectedVerdict: 1,
expectedSchema: 2,
},
{
name: 'explicit invalid verdict',
input: { power_valid: 0, power_metric_schema_version: 2 },
expectedVerdict: 0,
expectedSchema: 2,
},
{
name: 'legacy row without power contract fields',
input: {},
expectedVerdict: undefined,
expectedSchema: undefined,
},
])(
'normalizes overlay power contract discriminators: $name',
({ input, expectedVerdict, expectedSchema }) => {
const [row] = normalizeArtifactRows([rawRow(input)], '2026-03-01');

if (expectedVerdict === undefined) {
expect(row.metrics).not.toHaveProperty('power_valid');
} else {
expect(row.metrics.power_valid).toBe(expectedVerdict);
}
if (expectedSchema === undefined) {
expect(row.metrics).not.toHaveProperty('power_metric_schema_version');
} else {
expect(row.metrics.power_metric_schema_version).toBe(expectedSchema);
}
},
);

it('surfaces pipeline-parallelism fields in metrics (auto-capture)', () => {
// pp has no configs-table column: the frontend reads it from the metrics
// JSONB (rowToAggDataEntry), so the overlay route must keep passing it
Expand Down
15 changes: 15 additions & 0 deletions packages/app/src/components/ai-chart/types.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import { describe, expect, it } from 'vitest';

import { validateSpec } from './types';

describe('validateSpec measured power axes', () => {
it.each([
'y_measuredJPerSuccessfulQuery',
'y_measuredWhPerSuccessfulQuery',
'y_measuredPowerPercentTdp',
])('preserves %s as a benchmark Y-axis metric', (yAxisMetric) => {
const spec = validateSpec({ dataSource: 'benchmarks', yAxisMetric });

expect(spec.yAxisMetric).toBe(yAxisMetric);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,30 @@ describe('interpolateMetricAtInteractivity', () => {
expect(result!).toBeLessThan(3.5);
});

it.each([
'measuredJPerSuccessfulQuery',
'measuredWhPerSuccessfulQuery',
'measuredPowerPercentTdp',
] as const)('interpolates the derived measured metric %s', (metricKey) => {
const points = [
makePoint({
x: 20,
tpPerGpu: { y: 800, roof: false },
[metricKey]: { y: 80, roof: false },
}),
makePoint({
x: 60,
tpPerGpu: { y: 400, roof: false },
[metricKey]: { y: 40, roof: false },
}),
];

const result = interpolateMetricAtInteractivity(points, 40, metricKey);
expect(result).not.toBeNull();
expect(result!).toBeGreaterThan(40);
expect(result!).toBeLessThan(80);
});

it('returns null when metric field is missing from data points', () => {
const points = [
makePoint({ x: 20, tpPerGpu: { y: 800, roof: false } }),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {
reciprocalMetricAt,
} from '@/components/calculator/useThroughputData';
import { useBenchmarkHistory } from '@/hooks/api/use-benchmark-history';
import { getHardwareKey } from '@/lib/chart-utils';
import { buildMeasuredPowerChartFields, getHardwareKey } from '@/lib/chart-utils';
import { getGpuSpecs, isKnownGpu } from '@/lib/constants';
import { rowToAggDataEntry } from '@/lib/benchmark-transform';
import type { BenchmarkRow } from '@/lib/api';
Expand Down Expand Up @@ -74,24 +74,7 @@ function rowToLightweightPoint(row: BenchmarkRow): InferenceData | null {
jTotal: wrapMetric(power > 0 && tput ? (power * 1000) / tput : 0),
...(outputTput ? { jOutput: wrapMetric(power > 0 ? (power * 1000) / outputTput : 0) } : {}),
...(inputTput ? { jInput: wrapMetric(power > 0 ? (power * 1000) / inputTput : 0) } : {}),
...(typeof entry.avg_power_w === 'number'
? { measuredAvgPower: { y: entry.avg_power_w, roof: false } }
: {}),
...(typeof entry.joules_per_output_token === 'number'
? { measuredJPerOutputToken: { y: entry.joules_per_output_token, roof: false } }
: {}),
...(typeof entry.joules_per_total_token === 'number'
? { measuredJPerTotalToken: { y: entry.joules_per_total_token, roof: false } }
: {}),
...(typeof entry.prefill_avg_power_w === 'number'
? { measuredPrefillAvgPower: { y: entry.prefill_avg_power_w, roof: false } }
: {}),
...(typeof entry.decode_avg_power_w === 'number'
? { measuredDecodeAvgPower: { y: entry.decode_avg_power_w, roof: false } }
: {}),
...(typeof entry.joules_per_input_token === 'number'
? { measuredJPerInputToken: { y: entry.joules_per_input_token, roof: false } }
: {}),
...buildMeasuredPowerChartFields(entry, specs.tdp),
};
return point;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,24 @@
"y_measuredJPerTotalToken_title": "Measured Joules per Token (incl. prompt)",
"y_measuredJPerTotalToken_titleZh": "每 token 实测焦耳能耗(含提示词)",
"y_measuredJPerTotalToken_roofline": "lower_right",
"y_measuredJPerSuccessfulQuery": "measuredJPerSuccessfulQuery.y",
"y_measuredJPerSuccessfulQuery_label": "Measured J per Successful Query (J/query)",
"y_measuredJPerSuccessfulQuery_labelZh": "每次成功请求实测能耗(J/query)",
"y_measuredJPerSuccessfulQuery_title": "Measured Joules per Successful Query",
"y_measuredJPerSuccessfulQuery_titleZh": "每次成功请求实测焦耳能耗",
"y_measuredJPerSuccessfulQuery_roofline": "lower_right",
"y_measuredWhPerSuccessfulQuery": "measuredWhPerSuccessfulQuery.y",
"y_measuredWhPerSuccessfulQuery_label": "Measured Wh per Successful Query (Wh/query)",
"y_measuredWhPerSuccessfulQuery_labelZh": "每次成功请求实测能耗(Wh/query)",
"y_measuredWhPerSuccessfulQuery_title": "Measured Watt-hours per Successful Query",
"y_measuredWhPerSuccessfulQuery_titleZh": "每次成功请求实测瓦时能耗",
"y_measuredWhPerSuccessfulQuery_roofline": "lower_right",
"y_measuredPowerPercentTdp": "measuredPowerPercentTdp.y",
"y_measuredPowerPercentTdp_label": "Measured Average Power (% TDP)",
"y_measuredPowerPercentTdp_labelZh": "实测平均功耗(TDP 占比)",
"y_measuredPowerPercentTdp_title": "Measured Average Power as Percent of TDP",
"y_measuredPowerPercentTdp_titleZh": "实测平均功耗占 TDP 百分比",
"y_measuredPowerPercentTdp_roofline": "lower_right",
Comment thread
cursor[bot] marked this conversation as resolved.
Outdated
Comment thread
edwingao28 marked this conversation as resolved.
Outdated
"y_cost_limit": 5,
"y_latency_limit": 60
},
Expand Down Expand Up @@ -331,6 +349,24 @@
"y_measuredJPerTotalToken_title": "Measured Joules per Token (incl. prompt)",
"y_measuredJPerTotalToken_titleZh": "每 token 实测焦耳能耗(含提示词)",
"y_measuredJPerTotalToken_roofline": "lower_left",
"y_measuredJPerSuccessfulQuery": "measuredJPerSuccessfulQuery.y",
"y_measuredJPerSuccessfulQuery_label": "Measured J per Successful Query (J/query)",
"y_measuredJPerSuccessfulQuery_labelZh": "每次成功请求实测能耗(J/query)",
"y_measuredJPerSuccessfulQuery_title": "Measured Joules per Successful Query",
"y_measuredJPerSuccessfulQuery_titleZh": "每次成功请求实测焦耳能耗",
"y_measuredJPerSuccessfulQuery_roofline": "lower_left",
"y_measuredWhPerSuccessfulQuery": "measuredWhPerSuccessfulQuery.y",
"y_measuredWhPerSuccessfulQuery_label": "Measured Wh per Successful Query (Wh/query)",
"y_measuredWhPerSuccessfulQuery_labelZh": "每次成功请求实测能耗(Wh/query)",
"y_measuredWhPerSuccessfulQuery_title": "Measured Watt-hours per Successful Query",
"y_measuredWhPerSuccessfulQuery_titleZh": "每次成功请求实测瓦时能耗",
"y_measuredWhPerSuccessfulQuery_roofline": "lower_left",
"y_measuredPowerPercentTdp": "measuredPowerPercentTdp.y",
"y_measuredPowerPercentTdp_label": "Measured Average Power (% TDP)",
"y_measuredPowerPercentTdp_labelZh": "实测平均功耗(TDP 占比)",
"y_measuredPowerPercentTdp_title": "Measured Average Power as Percent of TDP",
"y_measuredPowerPercentTdp_titleZh": "实测平均功耗占 TDP 百分比",
"y_measuredPowerPercentTdp_roofline": "lower_left",
"y_cost_limit": 5,
"y_latency_limit": 60
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ const MEASURED_POWER_METRICS = [
'y_measuredDecodeAvgPower',
] as const;

const DERIVED_POWER_METRICS = [
'y_measuredJPerSuccessfulQuery',
'y_measuredWhPerSuccessfulQuery',
'y_measuredPowerPercentTdp',
] as const;

const defs = chartDefinitions as unknown as ChartDefinition[];
const interactivityDef = defs.find((d) => d.chartType === 'interactivity')!;
const e2eDef = defs.find((d) => d.chartType === 'e2e')!;
Expand Down Expand Up @@ -133,4 +139,13 @@ describe('measured-power Pareto direction', () => {
}
}
});

it.each(DERIVED_POWER_METRICS)('%s is bilingual and lower-is-better', (metric) => {
for (const chartDef of [interactivityDef, e2eDef]) {
expect(chartDef[metric]).toMatch(/\.y$/u);
expect(chartDef[`${metric}_label`]).toBeTruthy();
expect(chartDef[`${metric}_labelZh`]).toBeTruthy();
expect(declaredDirection(chartDef, metric)).toMatch(/^lower_/u);
}
});
});
39 changes: 34 additions & 5 deletions packages/app/src/components/inference/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,21 +137,24 @@ export interface AggDataEntry {
'p99.9_e2el': number;
// Measured GPU telemetry (emitted by runner's aggregate_power.py).
// Optional because historical runs predate the fields.
power_valid?: number;
power_metric_schema_version?: number;
avg_power_w?: number;
joules_per_successful_query?: number;
joules_per_output_token?: number;
joules_per_total_token?: number;
// Multinode / disagg-only measured power. The aggregate_power.py runner
// emits per-role energy splits when the deployment has separate prefill
// and decode workers (single-node disagg or multinode disagg). Single-node
// aggregated configs leave these undefined.
// - prefill_avg_power_w / decode_avg_power_w: mean per-GPU draw (W) within each role
// - joules_per_input_token: prefill_energy / total_input_tokens (prefill GPUs only)
// The disagg decode-only J/output is carried by joules_per_output_token above
// (the runner overrides it to decode_energy / total_output_tokens on disagg) —
// there is no separate _decode field.
// Unprefixed joules fields are whole-deployment metrics in schema version 2.
// Explicit prefill/decode keys retain role-local energy breakdowns.
prefill_avg_power_w?: number;
decode_avg_power_w?: number;
joules_per_input_token?: number;
prefill_joules_per_input_token?: number;
decode_joules_per_output_token?: number;
// Cluster-wide GPU telemetry beyond power (temperature, utilization, memory).
// Emitted by aggregate_power.py when the perfmon CSVs include the matching
// sample columns. Optional because older runs (and runs without the relevant
Expand Down Expand Up @@ -315,6 +318,9 @@ export interface InferenceData extends Partial<Omit<AggDataEntry, AggDataConflic
measuredJPerOutputToken?: { y: number; roof: boolean };
measuredJPerTotalToken?: { y: number; roof: boolean };
measuredJPerInputToken?: { y: number; roof: boolean };
measuredJPerSuccessfulQuery?: { y: number; roof: boolean };
measuredWhPerSuccessfulQuery?: { y: number; roof: boolean };
measuredPowerPercentTdp?: { y: number; roof: boolean };
}

/** Why a chart-ready point was intentionally excluded from the visible plot. */
Expand Down Expand Up @@ -358,7 +364,10 @@ export type YAxisMetricKey =
| 'measuredDecodeAvgPower'
| 'measuredJPerOutputToken'
| 'measuredJPerTotalToken'
| 'measuredJPerInputToken';
| 'measuredJPerInputToken'
| 'measuredJPerSuccessfulQuery'
| 'measuredWhPerSuccessfulQuery'
| 'measuredPowerPercentTdp';

/**
* Defines the configuration and labels for a specific chart.
Expand Down Expand Up @@ -488,6 +497,26 @@ export interface ChartDefinition {
y_measuredJPerTotalToken_label?: string;
y_measuredJPerTotalToken_title?: string;
y_measuredJPerTotalToken_roofline?: 'upper_right' | 'upper_left' | 'lower_left' | 'lower_right';
y_measuredJPerSuccessfulQuery?: string;
y_measuredJPerSuccessfulQuery_label?: string;
y_measuredJPerSuccessfulQuery_title?: string;
y_measuredJPerSuccessfulQuery_roofline?:
| 'upper_right'
| 'upper_left'
| 'lower_left'
| 'lower_right';
y_measuredWhPerSuccessfulQuery?: string;
y_measuredWhPerSuccessfulQuery_label?: string;
y_measuredWhPerSuccessfulQuery_title?: string;
y_measuredWhPerSuccessfulQuery_roofline?:
| 'upper_right'
| 'upper_left'
| 'lower_left'
| 'lower_right';
y_measuredPowerPercentTdp?: string;
y_measuredPowerPercentTdp_label?: string;
y_measuredPowerPercentTdp_title?: string;
y_measuredPowerPercentTdp_roofline?: 'upper_right' | 'upper_left' | 'lower_left' | 'lower_right';
y_cost_limit?: number;
y_latency_limit?: number;
}
Expand Down
3 changes: 3 additions & 0 deletions packages/app/src/components/inference/ui/ChartControls.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,9 @@ const METRIC_GROUPS: {
'y_measuredJPerInputToken',
'y_measuredJPerOutputToken',
'y_measuredJPerTotalToken',
'y_measuredJPerSuccessfulQuery',
'y_measuredWhPerSuccessfulQuery',
'y_measuredPowerPercentTdp',
Comment thread
edwingao28 marked this conversation as resolved.
],
},
{ label: 'Custom User Values', labelZh: '自定义值', metrics: ['y_costUser', 'y_powerUser'] },
Expand Down
Loading