From dd4a969301d7f8766a5057ba1946f564580645b3 Mon Sep 17 00:00:00 2001 From: Shin <128954611+shin4141@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:03:11 +0900 Subject: [PATCH 1/2] feat(curiocity): make max turns configurable Signed-off-by: Shin <128954611+shin4141@users.noreply.github.com> --- src/curiocity/README.md | 2 ++ src/curiocity/src/cli/commands/run.ts | 3 +++ src/curiocity/src/cli/index.ts | 1 + src/curiocity/src/config/matrix.ts | 2 ++ src/curiocity/src/config/merge.ts | 4 +++ src/curiocity/src/config/schema.ts | 1 + src/curiocity/src/curion/lifecycle.ts | 1 + src/curiocity/src/orchestrator/spec.ts | 1 + src/curiocity/src/shared/ipc.ts | 1 + src/curiocity/test/integration/helpers.ts | 2 ++ .../test/integration/interaction.test.ts | 12 +++++++++ src/curiocity/test/unit/merge.test.ts | 22 ++++++++++++++++ .../test/unit/profile-resolution.test.ts | 25 +++++++++++++++++++ 13 files changed, 77 insertions(+) diff --git a/src/curiocity/README.md b/src/curiocity/README.md index fb9deb22f..c8c59ef5f 100644 --- a/src/curiocity/README.md +++ b/src/curiocity/README.md @@ -68,6 +68,7 @@ curiocity validate --source # discovery dry-run + P10 prefligh | `--repeats ` | override case repeats | | `--concurrency ` | pool size (1 = serial debugging) | | `--timeout ` | per-trial wall-clock cap | +| `--max-turns ` | per-trial interaction turn cap | | `--evaluate` / `--no-evaluate` | toggle evaluation (D9 default per mode) | | `--collect-cost` / `--no-collect-cost` | toggle cost collection | | `--only-evaluator ` / `--skip-evaluator ` | narrow the eval pipeline | @@ -118,6 +119,7 @@ Example (from [`demo/cases/healthcheck/config.json`](./demo/cases/healthcheck/co { "agents": ["claude-code", "codex"], "timeoutSec": 600, + "maxTurns": 100, "repeats": 1, "evaluators": [ { "use": "file-exists", "must": ["HEALTHCHECK.md", "**/HealthController.java"], "gate": true }, diff --git a/src/curiocity/src/cli/commands/run.ts b/src/curiocity/src/cli/commands/run.ts index ea1aad01e..32cfd6d6f 100644 --- a/src/curiocity/src/cli/commands/run.ts +++ b/src/curiocity/src/cli/commands/run.ts @@ -33,6 +33,7 @@ export interface RunOptions { repeats?: number; concurrency?: number; timeout?: number; + maxTurns?: number; config?: string; out?: string; evaluate?: boolean; @@ -139,6 +140,7 @@ function printMatrix(matrix: MatrixEntry[]): void { out.write( ` - ${cell.case} × ${cell.agent} × repeat ${cell.repeat}` + ` [timeout=${cell.timeoutSec}s, combiner=${cell.combiner}, evaluate=${cell.evaluate}` + + (cell.maxTurns !== undefined ? `, maxTurns=${cell.maxTurns}` : '') + (models ? `, models: ${models}` : '') + ']\n', ); @@ -154,6 +156,7 @@ export async function runRun(opts: RunOptions): Promise { ...(opts.agent && opts.agent.length > 0 ? { agents: opts.agent } : {}), ...(opts.repeats !== undefined ? { repeats: opts.repeats } : {}), ...(opts.timeout !== undefined ? { timeoutSec: opts.timeout } : {}), + ...(opts.maxTurns !== undefined ? { maxTurns: opts.maxTurns } : {}), ...(opts.concurrency !== undefined ? { concurrency: opts.concurrency } : {}), ...(opts.out !== undefined ? { out: opts.out } : {}), ...(opts.evaluate !== undefined ? { evaluate: opts.evaluate } : {}), diff --git a/src/curiocity/src/cli/index.ts b/src/curiocity/src/cli/index.ts index 5a436c0b0..2f5926ed2 100644 --- a/src/curiocity/src/cli/index.ts +++ b/src/curiocity/src/cli/index.ts @@ -70,6 +70,7 @@ export function buildProgram(): Command { .option('--repeats ', 'override repeats per case', parsePositiveInt) .option('--concurrency ', 'bounded pool size', parsePositiveInt) .option('--timeout ', 'per-trial wall-clock cap', parsePositiveInt) + .option('--max-turns ', 'per-trial interaction turn cap', parsePositiveInt) .option('--config ', 'top-level config path') .option('--out ', 'results output dir') .option('--evaluate', 'enable evaluation') diff --git a/src/curiocity/src/config/matrix.ts b/src/curiocity/src/config/matrix.ts index 76353deb8..3a3fb5860 100644 --- a/src/curiocity/src/config/matrix.ts +++ b/src/curiocity/src/config/matrix.ts @@ -12,6 +12,7 @@ import type { TopLevelConfig } from './schema'; export interface MatrixEntry extends MatrixCell { timeoutSec: number; + maxTurns?: number; /** Effective model roles for this cell: top-level < profile < case < CLI (D13). */ models: PartialModelRoles; combiner: string; @@ -40,6 +41,7 @@ export function buildMatrix(args: BuildMatrixArgs): MatrixEntry[] { agent, repeat, timeoutSec: c.timeoutSec, + ...(c.maxTurns !== undefined ? { maxTurns: c.maxTurns } : {}), models: effectiveModels, combiner: c.combiner, evaluate: c.evaluate, diff --git a/src/curiocity/src/config/merge.ts b/src/curiocity/src/config/merge.ts index 918226dda..91254429e 100644 --- a/src/curiocity/src/config/merge.ts +++ b/src/curiocity/src/config/merge.ts @@ -32,6 +32,8 @@ export interface CliOverrides { repeats?: number; /** `--timeout `. */ timeoutSec?: number; + /** `--max-turns `. */ + maxTurns?: number; /** `--concurrency ` (suite-level). */ concurrency?: number; /** `--out ` (suite-level). */ @@ -65,6 +67,7 @@ export interface ResolvedCaseConfig { agents: string[]; repeats: number; timeoutSec: number; + maxTurns?: number; provision: ProvisionSpec; setup: string[]; teardown: string[]; @@ -172,6 +175,7 @@ export function resolveCaseConfig(args: ResolveCaseArgs): ResolvedCaseConfig { agents, repeats: cli.repeats ?? caseConfig.repeats ?? DEFAULT_REPEATS, timeoutSec: cli.timeoutSec ?? caseConfig.timeoutSec ?? DEFAULT_TIMEOUT_SEC, + maxTurns: cli.maxTurns ?? caseConfig.maxTurns, provision: mergeProvision(topProvision, caseConfig.provision), setup: concatScripts(topLevel.setup, caseConfig.setup), teardown: concatScripts(topLevel.teardown, caseConfig.teardown), diff --git a/src/curiocity/src/config/schema.ts b/src/curiocity/src/config/schema.ts index d308c8b19..680dcd746 100644 --- a/src/curiocity/src/config/schema.ts +++ b/src/curiocity/src/config/schema.ts @@ -138,6 +138,7 @@ export type TopLevelConfig = z.infer; export const caseConfigSchema = z.object({ agents: z.array(z.string().min(1)).min(1), timeoutSec: z.number().int().positive().optional(), + maxTurns: z.number().int().positive().optional(), repeats: z.number().int().positive().optional(), provision: provisionSchema.optional(), setup: z.array(z.string()).default([]), diff --git a/src/curiocity/src/curion/lifecycle.ts b/src/curiocity/src/curion/lifecycle.ts index 0f69fac02..f6bb7f8c7 100644 --- a/src/curiocity/src/curion/lifecycle.ts +++ b/src/curiocity/src/curion/lifecycle.ts @@ -229,6 +229,7 @@ export async function runTrial(spec: TrialSpec, opts: RunTrialOptions): Promise< router, qnaPolicy: spec.qna, maxWallClockMs: opts.maxWallClockMs ?? spec.timeoutSec * 1000 + 10_000, + ...(spec.maxTurns !== undefined ? { maxTurns: spec.maxTurns } : {}), ...(opts.pollIntervalMs !== undefined ? { pollIntervalMs: opts.pollIntervalMs } : {}), ...(opts.onQna ? { onQna: opts.onQna } : {}), ...(spawnedAt !== undefined ? { spawnedAt } : {}), diff --git a/src/curiocity/src/orchestrator/spec.ts b/src/curiocity/src/orchestrator/spec.ts index 4fa05937c..d6e1a87d3 100644 --- a/src/curiocity/src/orchestrator/spec.ts +++ b/src/curiocity/src/orchestrator/spec.ts @@ -129,6 +129,7 @@ export function buildTrialSpecs(args: BuildSpecsArgs): BuiltSpecs { caseName: entry.case, repeat: entry.repeat, timeoutSec: entry.timeoutSec, + ...(entry.maxTurns !== undefined ? { maxTurns: entry.maxTurns } : {}), prompt: def.prompt, qna: def.qna, ...(resolved.evaluate && def.evaluation !== undefined ? { evaluation: def.evaluation } : {}), diff --git a/src/curiocity/src/shared/ipc.ts b/src/curiocity/src/shared/ipc.ts index 496725cc9..d56d2c064 100644 --- a/src/curiocity/src/shared/ipc.ts +++ b/src/curiocity/src/shared/ipc.ts @@ -26,6 +26,7 @@ export const trialSpecSchema = z.object({ caseName: z.string(), repeat: z.number().int().positive(), timeoutSec: z.number().int().positive(), + maxTurns: z.number().int().positive().optional(), /** Task prompt (launch argument, D15). */ prompt: z.string(), /** QnA answering policy (§6). */ diff --git a/src/curiocity/test/integration/helpers.ts b/src/curiocity/test/integration/helpers.ts index c699bf081..0bc25dbdd 100644 --- a/src/curiocity/test/integration/helpers.ts +++ b/src/curiocity/test/integration/helpers.ts @@ -71,6 +71,7 @@ export interface MockSpecArgs { scene: string; runDir?: string; timeoutSec?: number; + maxTurns?: number; evaluate?: boolean; keepWorkspace?: boolean; mirror?: boolean; @@ -88,6 +89,7 @@ export function mockSpec(args: MockSpecArgs): TrialSpec { caseName: args.caseName ?? 'mock-case', repeat: args.repeat ?? 1, timeoutSec: args.timeoutSec ?? 20, + ...(args.maxTurns !== undefined ? { maxTurns: args.maxTurns } : {}), prompt: 'Create out.txt containing hello world.', qna: args.qna ?? 'Answer helpfully and concisely. If unsure, abort.', models: {}, diff --git a/src/curiocity/test/integration/interaction.test.ts b/src/curiocity/test/integration/interaction.test.ts index c755498ba..d1b0d67e8 100644 --- a/src/curiocity/test/integration/interaction.test.ts +++ b/src/curiocity/test/integration/interaction.test.ts @@ -91,6 +91,18 @@ describe('§6 interaction engine — trigger table, row by row', () => { expect(router!.isExhausted()).toBe(true); }); + it('uses the TrialSpec maxTurns cap instead of the engine default', async () => { + const { result, router } = await run({ scene: 'two-gates.json', maxTurns: 1 }, { + entries: [ + { role: 'workhorse', text: 'Yes, approved' }, + { role: 'workhorse', text: 'Yes, approve' }, + ], + }); + expect(result.status).toBe('timeout'); + expect(result.turnCount).toBe(2); + expect(router!.isExhausted()).toBe(true); + }); + it('row 2 (Stop → fast classifies question) → workhorse free-text answer → typed reply', async () => { const { result, router } = await run({ scene: 'free-text-question.json' }, { entries: [ diff --git a/src/curiocity/test/unit/merge.test.ts b/src/curiocity/test/unit/merge.test.ts index 5071af596..6cb72a854 100644 --- a/src/curiocity/test/unit/merge.test.ts +++ b/src/curiocity/test/unit/merge.test.ts @@ -109,6 +109,28 @@ describe('D13: scalar precedence (defaults < top-level < case < CLI)', () => { expect(resolved.combiner).toBe(DEFAULT_COMBINER); }); + it('maxTurns stays optional and CLI overrides the case value', () => { + const topLevel = topLevelConfigSchema.parse({}); + const unset = resolveCaseConfig({ + caseName: 'c', + topLevel, + caseConfig: caseConfigSchema.parse({ agents: ['x'] }), + evaluateDefault: true, + }); + expect(unset.maxTurns).toBeUndefined(); + + const resolved = resolveCaseConfig({ + caseName: 'c', + topLevel, + caseConfig: caseConfigSchema.parse({ agents: ['x'], maxTurns: 7 }), + cli: { maxTurns: 9 }, + evaluateDefault: true, + }); + expect(resolved.maxTurns).toBe(9); + expect(buildMatrix({ topLevel, cases: [resolved] })[0]!.maxTurns).toBe(9); + expect(() => caseConfigSchema.parse({ agents: ['x'], maxTurns: 0 })).toThrow(); + }); + it('--agent narrows the case-declared agent list', () => { const resolved = resolveCaseConfig({ caseName: 'c', diff --git a/src/curiocity/test/unit/profile-resolution.test.ts b/src/curiocity/test/unit/profile-resolution.test.ts index fae2bb101..d81f878a7 100644 --- a/src/curiocity/test/unit/profile-resolution.test.ts +++ b/src/curiocity/test/unit/profile-resolution.test.ts @@ -173,6 +173,31 @@ describe('buildTrialSpecs (D13 defaults reachable end-to-end)', () => { openai: 'https://bifrost.example/openai', }); }); + + it('propagates the optional case maxTurns into the final TrialSpec', () => { + const topLevel = topLevelConfigSchema.parse({}); + const cases = [makeCase('pong', ['claude-code'])]; + cases[0]!.config = caseConfigSchema.parse({ agents: ['claude-code'], maxTurns: 7 }); + const resolvedCases = cases.map((c) => + resolveCaseConfig({ caseName: c.name, topLevel, caseConfig: c.config, evaluateDefault: false }), + ); + const matrix = buildMatrix({ topLevel, cases: resolvedCases }); + + const { specs, skipped } = buildTrialSpecs({ + topLevel, + cases, + resolvedCases, + matrix, + runDir: '/tmp/does-not-matter', + configDir: process.cwd(), + keepWorkspace: false, + mirror: false, + keys: {}, + }); + + expect(skipped).toHaveLength(0); + expect(specs[0]!.maxTurns).toBe(7); + }); }); describe('buildTrialSpecs — registry-default `models` reach the final TrialSpec (m5-review R1)', () => { From 63f79af8cc918693594674db0ab151db2eaf8227 Mon Sep 17 00:00:00 2001 From: Shin <128954611+shin4141@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:36:12 +0900 Subject: [PATCH 2/2] fix(curiocity): show effective max turns in dry-run Signed-off-by: Shin <128954611+shin4141@users.noreply.github.com> --- src/curiocity/demo/cases/healthcheck/config.json | 1 + src/curiocity/docs/architecture.md | 2 ++ src/curiocity/src/cli/commands/run.ts | 3 ++- src/curiocity/src/interaction/engine.ts | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/curiocity/demo/cases/healthcheck/config.json b/src/curiocity/demo/cases/healthcheck/config.json index bac284ab9..900c5e299 100644 --- a/src/curiocity/demo/cases/healthcheck/config.json +++ b/src/curiocity/demo/cases/healthcheck/config.json @@ -1,6 +1,7 @@ { "agents": ["claude-code", "codex"], "timeoutSec": 600, + "maxTurns": 100, "repeats": 1, "evaluators": [ { diff --git a/src/curiocity/docs/architecture.md b/src/curiocity/docs/architecture.md index 2b072c3b6..0ad58fb26 100644 --- a/src/curiocity/docs/architecture.md +++ b/src/curiocity/docs/architecture.md @@ -382,6 +382,7 @@ Per Curion, in order: { "agents": ["claude-code", "codex"], "timeoutSec": 1800, + "maxTurns": 100, "repeats": 1, "provision": { "mcps": [], "plugins": [] }, "setup": ["./setup-fixture.sh"], @@ -514,6 +515,7 @@ Key `run` options (all override config; every one usable in CI): | `--repeats ` | override case repeats | | `--concurrency ` | pool size (1 = serial debugging) | | `--timeout ` | per-trial cap | +| `--max-turns ` | per-trial interaction turn cap | | `--evaluate/--no-evaluate`, `--collect-cost/--no-collect-cost` | feature toggles (D9 defaults) | | `--only-evaluator ` / `--skip-evaluator ` | narrow the eval pipeline | | `--mirror` | stream PTY output live | diff --git a/src/curiocity/src/cli/commands/run.ts b/src/curiocity/src/cli/commands/run.ts index 32cfd6d6f..1efc0b6dc 100644 --- a/src/curiocity/src/cli/commands/run.ts +++ b/src/curiocity/src/cli/commands/run.ts @@ -11,6 +11,7 @@ import { preflightAgentHomes } from '../../orchestrator/preflight'; import { resolveBaseUrls, resolveKeys } from '../../llm/keys'; import { evaluatorRegistry } from '../../evaluators'; import { evaluatorEntrySchema } from '../../config/schema'; +import { DEFAULT_MAX_TURNS } from '../../interaction/engine'; import type { ResolvedCaseConfig } from '../../config/merge'; import type { PartialModelRoles } from '../../shared/models'; import { ConfigError } from '../../shared/errors'; @@ -140,7 +141,7 @@ function printMatrix(matrix: MatrixEntry[]): void { out.write( ` - ${cell.case} × ${cell.agent} × repeat ${cell.repeat}` + ` [timeout=${cell.timeoutSec}s, combiner=${cell.combiner}, evaluate=${cell.evaluate}` + - (cell.maxTurns !== undefined ? `, maxTurns=${cell.maxTurns}` : '') + + `, maxTurns=${cell.maxTurns ?? DEFAULT_MAX_TURNS}` + (models ? `, models: ${models}` : '') + ']\n', ); diff --git a/src/curiocity/src/interaction/engine.ts b/src/curiocity/src/interaction/engine.ts index b53f6b339..d3ee942f1 100644 --- a/src/curiocity/src/interaction/engine.ts +++ b/src/curiocity/src/interaction/engine.ts @@ -87,7 +87,7 @@ export interface EngineDeps { } const DEFAULT_POLL_MS = 25; -const DEFAULT_MAX_TURNS = 100; +export const DEFAULT_MAX_TURNS = 100; const TERMINATE_GRACE_MS = 2000; type CheckAction = { action: 'answered' | 'terminate' | 'none' };