Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/curiocity/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ curiocity validate --source <dir> # discovery dry-run + P10 prefligh
| `--repeats <n>` | override case repeats |
| `--concurrency <n>` | pool size (1 = serial debugging) |
| `--timeout <sec>` | per-trial wall-clock cap |
| `--max-turns <n>` | per-trial interaction turn cap |
| `--evaluate` / `--no-evaluate` | toggle evaluation (D9 default per mode) |
| `--collect-cost` / `--no-collect-cost` | toggle cost collection |
| `--only-evaluator <id>` / `--skip-evaluator <id>` | narrow the eval pipeline |
Expand Down Expand Up @@ -118,6 +119,7 @@ Example (from [`demo/cases/healthcheck/config.json`](./demo/cases/healthcheck/co
{
"agents": ["claude-code", "codex"],
"timeoutSec": 600,
"maxTurns": 100,
"repeats": 1,
"evaluators": [
{ "use": "file-exists", "must": ["HEALTHCHECK.md", "**/HealthController.java"], "gate": true },
Expand Down
1 change: 1 addition & 0 deletions src/curiocity/demo/cases/healthcheck/config.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
{
"agents": ["claude-code", "codex"],
"timeoutSec": 600,
"maxTurns": 100,
"repeats": 1,
"evaluators": [
{
Expand Down
2 changes: 2 additions & 0 deletions src/curiocity/docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,7 @@ Per Curion, in order:
{
"agents": ["claude-code", "codex"],
"timeoutSec": 1800,
"maxTurns": 100,
"repeats": 1,
"provision": { "mcps": [], "plugins": [] },
"setup": ["./setup-fixture.sh"],
Expand Down Expand Up @@ -514,6 +515,7 @@ Key `run` options (all override config; every one usable in CI):
| `--repeats <n>` | override case repeats |
| `--concurrency <n>` | pool size (1 = serial debugging) |
| `--timeout <sec>` | per-trial cap |
| `--max-turns <n>` | per-trial interaction turn cap |
| `--evaluate/--no-evaluate`, `--collect-cost/--no-collect-cost` | feature toggles (D9 defaults) |
| `--only-evaluator <id>` / `--skip-evaluator <id>` | narrow the eval pipeline |
| `--mirror` | stream PTY output live |
Expand Down
4 changes: 4 additions & 0 deletions src/curiocity/src/cli/commands/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { preflightAgentHomes } from '../../orchestrator/preflight';
import { resolveBaseUrls, resolveKeys } from '../../llm/keys';
import { evaluatorRegistry } from '../../evaluators';
import { evaluatorEntrySchema } from '../../config/schema';
import { DEFAULT_MAX_TURNS } from '../../interaction/engine';
import type { ResolvedCaseConfig } from '../../config/merge';
import type { PartialModelRoles } from '../../shared/models';
import { ConfigError } from '../../shared/errors';
Expand All @@ -33,6 +34,7 @@ export interface RunOptions {
repeats?: number;
concurrency?: number;
timeout?: number;
maxTurns?: number;
config?: string;
out?: string;
evaluate?: boolean;
Expand Down Expand Up @@ -139,6 +141,7 @@ function printMatrix(matrix: MatrixEntry[]): void {
out.write(
` - ${cell.case} × ${cell.agent} × repeat ${cell.repeat}` +
` [timeout=${cell.timeoutSec}s, combiner=${cell.combiner}, evaluate=${cell.evaluate}` +
`, maxTurns=${cell.maxTurns ?? DEFAULT_MAX_TURNS}` +
(models ? `, models: ${models}` : '') +
']\n',
);
Expand All @@ -154,6 +157,7 @@ export async function runRun(opts: RunOptions): Promise<number> {
...(opts.agent && opts.agent.length > 0 ? { agents: opts.agent } : {}),
...(opts.repeats !== undefined ? { repeats: opts.repeats } : {}),
...(opts.timeout !== undefined ? { timeoutSec: opts.timeout } : {}),
...(opts.maxTurns !== undefined ? { maxTurns: opts.maxTurns } : {}),
...(opts.concurrency !== undefined ? { concurrency: opts.concurrency } : {}),
...(opts.out !== undefined ? { out: opts.out } : {}),
...(opts.evaluate !== undefined ? { evaluate: opts.evaluate } : {}),
Expand Down
1 change: 1 addition & 0 deletions src/curiocity/src/cli/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ export function buildProgram(): Command {
.option('--repeats <n>', 'override repeats per case', parsePositiveInt)
.option('--concurrency <n>', 'bounded pool size', parsePositiveInt)
.option('--timeout <sec>', 'per-trial wall-clock cap', parsePositiveInt)
.option('--max-turns <n>', 'per-trial interaction turn cap', parsePositiveInt)
.option('--config <file>', 'top-level config path')
.option('--out <dir>', 'results output dir')
.option('--evaluate', 'enable evaluation')
Expand Down
2 changes: 2 additions & 0 deletions src/curiocity/src/config/matrix.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import type { TopLevelConfig } from './schema';

export interface MatrixEntry extends MatrixCell {
timeoutSec: number;
maxTurns?: number;
/** Effective model roles for this cell: top-level < profile < case < CLI (D13). */
models: PartialModelRoles;
combiner: string;
Expand Down Expand Up @@ -40,6 +41,7 @@ export function buildMatrix(args: BuildMatrixArgs): MatrixEntry[] {
agent,
repeat,
timeoutSec: c.timeoutSec,
...(c.maxTurns !== undefined ? { maxTurns: c.maxTurns } : {}),
models: effectiveModels,
combiner: c.combiner,
evaluate: c.evaluate,
Expand Down
4 changes: 4 additions & 0 deletions src/curiocity/src/config/merge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ export interface CliOverrides {
repeats?: number;
/** `--timeout <sec>`. */
timeoutSec?: number;
/** `--max-turns <n>`. */
maxTurns?: number;
/** `--concurrency <n>` (suite-level). */
concurrency?: number;
/** `--out <dir>` (suite-level). */
Expand Down Expand Up @@ -65,6 +67,7 @@ export interface ResolvedCaseConfig {
agents: string[];
repeats: number;
timeoutSec: number;
maxTurns?: number;
provision: ProvisionSpec;
setup: string[];
teardown: string[];
Expand Down Expand Up @@ -172,6 +175,7 @@ export function resolveCaseConfig(args: ResolveCaseArgs): ResolvedCaseConfig {
agents,
repeats: cli.repeats ?? caseConfig.repeats ?? DEFAULT_REPEATS,
timeoutSec: cli.timeoutSec ?? caseConfig.timeoutSec ?? DEFAULT_TIMEOUT_SEC,
maxTurns: cli.maxTurns ?? caseConfig.maxTurns,
provision: mergeProvision(topProvision, caseConfig.provision),
setup: concatScripts(topLevel.setup, caseConfig.setup),
teardown: concatScripts(topLevel.teardown, caseConfig.teardown),
Expand Down
1 change: 1 addition & 0 deletions src/curiocity/src/config/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ export type TopLevelConfig = z.infer<typeof topLevelConfigSchema>;
export const caseConfigSchema = z.object({
agents: z.array(z.string().min(1)).min(1),
timeoutSec: z.number().int().positive().optional(),
maxTurns: z.number().int().positive().optional(),
repeats: z.number().int().positive().optional(),
provision: provisionSchema.optional(),
setup: z.array(z.string()).default([]),
Expand Down
1 change: 1 addition & 0 deletions src/curiocity/src/curion/lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ export async function runTrial(spec: TrialSpec, opts: RunTrialOptions): Promise<
router,
qnaPolicy: spec.qna,
maxWallClockMs: opts.maxWallClockMs ?? spec.timeoutSec * 1000 + 10_000,
...(spec.maxTurns !== undefined ? { maxTurns: spec.maxTurns } : {}),
...(opts.pollIntervalMs !== undefined ? { pollIntervalMs: opts.pollIntervalMs } : {}),
...(opts.onQna ? { onQna: opts.onQna } : {}),
...(spawnedAt !== undefined ? { spawnedAt } : {}),
Expand Down
2 changes: 1 addition & 1 deletion src/curiocity/src/interaction/engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ export interface EngineDeps {
}

const DEFAULT_POLL_MS = 25;
const DEFAULT_MAX_TURNS = 100;
export const DEFAULT_MAX_TURNS = 100;
const TERMINATE_GRACE_MS = 2000;

type CheckAction = { action: 'answered' | 'terminate' | 'none' };
Expand Down
1 change: 1 addition & 0 deletions src/curiocity/src/orchestrator/spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ export function buildTrialSpecs(args: BuildSpecsArgs): BuiltSpecs {
caseName: entry.case,
repeat: entry.repeat,
timeoutSec: entry.timeoutSec,
...(entry.maxTurns !== undefined ? { maxTurns: entry.maxTurns } : {}),
prompt: def.prompt,
qna: def.qna,
...(resolved.evaluate && def.evaluation !== undefined ? { evaluation: def.evaluation } : {}),
Expand Down
1 change: 1 addition & 0 deletions src/curiocity/src/shared/ipc.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ export const trialSpecSchema = z.object({
caseName: z.string(),
repeat: z.number().int().positive(),
timeoutSec: z.number().int().positive(),
maxTurns: z.number().int().positive().optional(),
/** Task prompt (launch argument, D15). */
prompt: z.string(),
/** QnA answering policy (§6). */
Expand Down
2 changes: 2 additions & 0 deletions src/curiocity/test/integration/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ export interface MockSpecArgs {
scene: string;
runDir?: string;
timeoutSec?: number;
maxTurns?: number;
evaluate?: boolean;
keepWorkspace?: boolean;
mirror?: boolean;
Expand All @@ -88,6 +89,7 @@ export function mockSpec(args: MockSpecArgs): TrialSpec {
caseName: args.caseName ?? 'mock-case',
repeat: args.repeat ?? 1,
timeoutSec: args.timeoutSec ?? 20,
...(args.maxTurns !== undefined ? { maxTurns: args.maxTurns } : {}),
prompt: 'Create out.txt containing hello world.',
qna: args.qna ?? 'Answer helpfully and concisely. If unsure, abort.',
models: {},
Expand Down
12 changes: 12 additions & 0 deletions src/curiocity/test/integration/interaction.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,18 @@ describe('§6 interaction engine — trigger table, row by row', () => {
expect(router!.isExhausted()).toBe(true);
});

it('uses the TrialSpec maxTurns cap instead of the engine default', async () => {
const { result, router } = await run({ scene: 'two-gates.json', maxTurns: 1 }, {
entries: [
{ role: 'workhorse', text: 'Yes, approved' },
{ role: 'workhorse', text: 'Yes, approve' },
],
});
expect(result.status).toBe('timeout');
expect(result.turnCount).toBe(2);
expect(router!.isExhausted()).toBe(true);
});

it('row 2 (Stop → fast classifies question) → workhorse free-text answer → typed reply', async () => {
const { result, router } = await run({ scene: 'free-text-question.json' }, {
entries: [
Expand Down
22 changes: 22 additions & 0 deletions src/curiocity/test/unit/merge.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,28 @@ describe('D13: scalar precedence (defaults < top-level < case < CLI)', () => {
expect(resolved.combiner).toBe(DEFAULT_COMBINER);
});

it('maxTurns stays optional and CLI overrides the case value', () => {
const topLevel = topLevelConfigSchema.parse({});
const unset = resolveCaseConfig({
caseName: 'c',
topLevel,
caseConfig: caseConfigSchema.parse({ agents: ['x'] }),
evaluateDefault: true,
});
expect(unset.maxTurns).toBeUndefined();

const resolved = resolveCaseConfig({
caseName: 'c',
topLevel,
caseConfig: caseConfigSchema.parse({ agents: ['x'], maxTurns: 7 }),
cli: { maxTurns: 9 },
evaluateDefault: true,
});
expect(resolved.maxTurns).toBe(9);
expect(buildMatrix({ topLevel, cases: [resolved] })[0]!.maxTurns).toBe(9);
expect(() => caseConfigSchema.parse({ agents: ['x'], maxTurns: 0 })).toThrow();
});

it('--agent narrows the case-declared agent list', () => {
const resolved = resolveCaseConfig({
caseName: 'c',
Expand Down
25 changes: 25 additions & 0 deletions src/curiocity/test/unit/profile-resolution.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,31 @@ describe('buildTrialSpecs (D13 defaults reachable end-to-end)', () => {
openai: 'https://bifrost.example/openai',
});
});

it('propagates the optional case maxTurns into the final TrialSpec', () => {
const topLevel = topLevelConfigSchema.parse({});
const cases = [makeCase('pong', ['claude-code'])];
cases[0]!.config = caseConfigSchema.parse({ agents: ['claude-code'], maxTurns: 7 });
const resolvedCases = cases.map((c) =>
resolveCaseConfig({ caseName: c.name, topLevel, caseConfig: c.config, evaluateDefault: false }),
);
const matrix = buildMatrix({ topLevel, cases: resolvedCases });

const { specs, skipped } = buildTrialSpecs({
topLevel,
cases,
resolvedCases,
matrix,
runDir: '/tmp/does-not-matter',
configDir: process.cwd(),
keepWorkspace: false,
mirror: false,
keys: {},
});

expect(skipped).toHaveLength(0);
expect(specs[0]!.maxTurns).toBe(7);
});
});

describe('buildTrialSpecs — registry-default `models` reach the final TrialSpec (m5-review R1)', () => {
Expand Down