Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions .github/workflows/refresh-model-capabilities.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,18 +28,21 @@ jobs:
- name: Refresh bundled model capabilities snapshot
run: bun run build:model-capabilities

- name: Refresh free opencode models snapshot
run: bun run build:free-models

- name: Validate capability guardrails
run: bun run test:model-capabilities

- name: Create refresh pull request
uses: peter-evans/create-pull-request@v7
with:
commit-message: "chore: refresh model capabilities snapshot"
title: "chore: refresh model capabilities snapshot"
commit-message: "chore: refresh model snapshots"
title: "chore: refresh model snapshots"
body: |
Automated refresh of `src/generated/model-capabilities.generated.json` from `https://models.dev/api.json`.

This keeps the bundled capability snapshot aligned with upstream model metadata without relying on manual refreshes.
Automated refresh of model snapshots from `https://models.dev/api.json`:
- `src/generated/model-capabilities.generated.json` — full model capabilities
- `src/generated/free-opencode-models.generated.json` — free opencode model list
branch: automation/refresh-model-capabilities
delete-branch: true
labels: |
Expand Down
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,13 @@
"build:all": "bun run build && bun run build:binaries",
"build:binaries": "bun run script/build-binaries.ts",
"build:schema": "bun run script/build-schema.ts",
"build:free-models": "bun run script/build-free-models.ts",
"build:model-capabilities": "bun run script/build-model-capabilities.ts",
"clean": "rm -rf dist",
"prepare": "bun run build",
"postinstall": "node postinstall.mjs",
"prepublishOnly": "bun run clean && bun run build:lsp-tools-mcp && bun run build",
"test:model-capabilities": "bun test src/shared/model-capability-aliases.test.ts src/shared/model-capability-guardrails.test.ts src/shared/model-capabilities.test.ts src/cli/doctor/checks/model-resolution.test.ts --bail",
"test:model-capabilities": "bun test src/shared/model-capability-aliases.test.ts src/shared/model-capability-guardrails.test.ts src/shared/model-capabilities.test.ts src/cli/doctor/checks/model-resolution.test.ts script/free-model-extraction.test.ts --bail",
"typecheck": "tsgo --noEmit && bun run typecheck:packages",
"typecheck:packages": "tsgo --noEmit -p packages/rules-engine/tsconfig.json && tsgo --noEmit -p packages/ast-grep-core/tsconfig.json && tsgo --noEmit -p packages/ast-grep-mcp/tsconfig.json && tsgo --noEmit -p packages/utils/tsconfig.json && tsgo --noEmit -p packages/model-core/tsconfig.json && tsgo --noEmit -p packages/comment-checker-core/tsconfig.json && tsgo --noEmit -p packages/hashline-core/tsconfig.json && tsgo --noEmit -p packages/boulder-state/tsconfig.json && tsgo --noEmit -p packages/agents-md-core/tsconfig.json",
"typecheck:script": "tsgo --noEmit -p script/tsconfig.json",
Expand Down
26 changes: 26 additions & 0 deletions script/build-free-models.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { writeFileSync } from "fs"
import { resolve } from "path"
import { MODELS_DEV_SOURCE_URL } from "../src/shared/model-capabilities-cache"
import { extractFreeOpenCodeModelIds } from "./free-model-extraction"

const OUTPUT_PATH = resolve(import.meta.dir, "../src/generated/free-opencode-models.generated.json")

console.log(`Fetching free opencode models from ${MODELS_DEV_SOURCE_URL}...`)
const response = await fetch(MODELS_DEV_SOURCE_URL)
if (!response.ok) {
throw new Error(`models.dev fetch failed with ${response.status}`)
}

const raw = await response.json()
const freeModelIds = extractFreeOpenCodeModelIds(raw)
if (freeModelIds.length === 0) {
throw new Error("No free opencode models found — models.dev schema may have changed")
}
const snapshot = {
generatedAt: new Date().toISOString(),
sourceUrl: MODELS_DEV_SOURCE_URL,
providers: ["opencode"],
models: freeModelIds,
}
writeFileSync(OUTPUT_PATH, `${JSON.stringify(snapshot, null, 2)}\n`)
console.log(`Generated ${OUTPUT_PATH} with ${freeModelIds.length} free opencode models`)
89 changes: 89 additions & 0 deletions script/free-model-extraction.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/// <reference types="bun-types" />

import { describe, expect, test } from "bun:test"
import { extractFreeOpenCodeModelIds } from "./free-model-extraction"

describe("extractFreeOpenCodeModelIds", () => {
test("extracts free model IDs sorted alphabetically", () => {
const raw = {
opencode: {
models: {
"z-model": { status: "active", cost: { input: 0, output: 0 } },
"a-model": { status: "active", cost: { input: 0, output: 0 } },
},
},
}
expect(extractFreeOpenCodeModelIds(raw)).toEqual(["a-model", "z-model"])
})

test("excludes deprecated models", () => {
const raw = {
opencode: {
models: {
"live-model": { cost: { input: 0, output: 0 } },
"dead-model": { status: "deprecated", cost: { input: 0, output: 0 } },
},
},
}
expect(extractFreeOpenCodeModelIds(raw)).toEqual(["live-model"])
})

test("excludes models with paid input cost", () => {
const raw = {
opencode: {
models: {
"free-model": { cost: { input: 0, output: 0 } },
"paid-input": { cost: { input: 5, output: 0 } },
},
},
}
expect(extractFreeOpenCodeModelIds(raw)).toEqual(["free-model"])
})

test("excludes models with paid output cost", () => {
const raw = {
opencode: {
models: {
"free-model": { cost: { input: 0, output: 0 } },
"paid-output": { cost: { input: 0, output: 3 } },
},
},
}
expect(extractFreeOpenCodeModelIds(raw)).toEqual(["free-model"])
})

test("returns empty array for null input", () => {
expect(extractFreeOpenCodeModelIds(null)).toEqual([])
})

test("returns empty array when opencode provider is missing", () => {
expect(extractFreeOpenCodeModelIds({ other: {} })).toEqual([])
})

test("returns empty array when models key is missing", () => {
expect(extractFreeOpenCodeModelIds({ opencode: {} })).toEqual([])
})

test("returns empty array when all models are deprecated", () => {
const raw = {
opencode: {
models: {
m1: { status: "deprecated", cost: { input: 0, output: 0 } },
},
},
}
expect(extractFreeOpenCodeModelIds(raw)).toEqual([])
})

test("skips entries without cost field", () => {
const raw = {
opencode: {
models: {
"no-cost": { status: "active" },
"has-cost": { cost: { input: 0, output: 0 } },
},
},
}
expect(extractFreeOpenCodeModelIds(raw)).toEqual(["has-cost"])
})
})
22 changes: 22 additions & 0 deletions script/free-model-extraction.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)
}

export function extractFreeOpenCodeModelIds(raw: unknown): string[] {
if (!isRecord(raw)) return []
const opencode = raw.opencode
if (!isRecord(opencode)) return []
const models = opencode.models
if (!isRecord(models)) return []

const ids: string[] = []
for (const [id, entry] of Object.entries(models)) {
if (!isRecord(entry)) continue
if (entry.status === "deprecated") continue
const cost = entry.cost
if (!isRecord(cost)) continue
if (cost.input !== 0 || cost.output !== 0) continue
ids.push(id)
}
return ids.sort()
}
14 changes: 14 additions & 0 deletions src/generated/free-opencode-models.generated.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"generatedAt": "2026-05-06T12:37:59.222Z",
"sourceUrl": "https://models.dev/api.json",
"providers": [
"opencode"
],
"models": [
"big-pickle",
"gpt-5-nano",
"hy3-preview-free",
"minimax-m2.5-free",
"nemotron-3-super-free"
]
}
102 changes: 102 additions & 0 deletions src/tools/delegate-task/free-model-fallback.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
/// <reference types="bun-types" />

import { describe, expect, test } from "bun:test"
import {
FREE_ONLY_FALLBACK_CHAIN,
appendFreeModelFallbacks,
isFreeOnlyProviderConfiguration,
isKnownFreeModel,
} from "./free-model-fallback"
import freeModelsSnapshot from "../../generated/free-opencode-models.generated.json"
import { CATEGORY_MODEL_REQUIREMENTS } from "../../shared/model-requirements"

const ultrabrainFallbackChain = CATEGORY_MODEL_REQUIREMENTS.ultrabrain.fallbackChain

describe("free-opencode-models.generated.json", () => {
test("snapshot contains at least one free model", () => {
expect(freeModelsSnapshot.models.length).toBeGreaterThan(0)
})

test("snapshot models are sorted alphabetically", () => {
expect(freeModelsSnapshot.models).toEqual([...freeModelsSnapshot.models].sort())
})
})

describe("FREE_ONLY_FALLBACK_CHAIN", () => {
test("is derived from the generated snapshot", () => {
const chainModelIds = FREE_ONLY_FALLBACK_CHAIN.map((entry) => entry.model)
expect(chainModelIds).toEqual(freeModelsSnapshot.models)
})

test("only contains entries that isKnownFreeModel recognizes", () => {
for (const entry of FREE_ONLY_FALLBACK_CHAIN) {
expect(isKnownFreeModel(entry.model)).toBe(true)
}
})

test("every provider in the chain is recognized as a free-only provider", () => {
for (const entry of FREE_ONLY_FALLBACK_CHAIN) {
for (const provider of entry.providers) {
expect(isFreeOnlyProviderConfiguration([provider])).toBe(true)
}
}
})
})

describe("isFreeOnlyProviderConfiguration", () => {
test("returns true when all providers are in FREE_ONLY_PROVIDER_IDS", () => {
expect(isFreeOnlyProviderConfiguration(["opencode"])).toBe(true)
})

test("returns false when any provider is not in FREE_ONLY_PROVIDER_IDS", () => {
expect(isFreeOnlyProviderConfiguration(["opencode", "anthropic"])).toBe(false)
})

test("returns false for null", () => {
expect(isFreeOnlyProviderConfiguration(null)).toBe(false)
})

test("returns false for empty array", () => {
expect(isFreeOnlyProviderConfiguration([])).toBe(false)
})
})

describe("appendFreeModelFallbacks", () => {
test("returns FREE_ONLY_FALLBACK_CHAIN when original chain is undefined", () => {
const result = appendFreeModelFallbacks(undefined)
expect(result).toEqual(FREE_ONLY_FALLBACK_CHAIN)
})

test("returns FREE_ONLY_FALLBACK_CHAIN when original chain is empty", () => {
const result = appendFreeModelFallbacks([])
expect(result).toEqual(FREE_ONLY_FALLBACK_CHAIN)
})

test("preserves original chain order then appends free models in FREE_ONLY_FALLBACK_CHAIN order", () => {
const result = appendFreeModelFallbacks(ultrabrainFallbackChain)
expect(result.length).toBeGreaterThan(ultrabrainFallbackChain.length)
expect(result.slice(0, ultrabrainFallbackChain.length)).toEqual(ultrabrainFallbackChain)

const appended = result.slice(ultrabrainFallbackChain.length)
const expectedAppended = FREE_ONLY_FALLBACK_CHAIN.filter(
(entry) => !ultrabrainFallbackChain.some((e) => e.model === entry.model),
)
expect(appended).toEqual(expectedAppended)
})

test("does not duplicate free models already present in the chain", () => {
const chainWithFreeEntry = [
...ultrabrainFallbackChain,
FREE_ONLY_FALLBACK_CHAIN[0],
]
const result = appendFreeModelFallbacks(chainWithFreeEntry)

const modelCounts = new Map<string, number>()
for (const entry of result) {
modelCounts.set(entry.model, (modelCounts.get(entry.model) ?? 0) + 1)
}
for (const [model, count] of modelCounts) {
expect(count).toBe(1)
}
})
})
35 changes: 35 additions & 0 deletions src/tools/delegate-task/free-model-fallback.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import type { FallbackEntry } from "../../shared/model-requirements"
import freeModelsSnapshot from "../../generated/free-opencode-models.generated.json"

const FREE_ONLY_PROVIDER_IDS = new Set(freeModelsSnapshot.providers)
const KNOWN_FREE_MODEL_IDS = new Set(freeModelsSnapshot.models)

export const FREE_ONLY_FALLBACK_CHAIN: FallbackEntry[] = freeModelsSnapshot.models.map(
(model) => ({ providers: freeModelsSnapshot.providers, model }),
)

function getModelId(model: string): string {
return model.includes("/") ? model.split("/").slice(1).join("/") : model
}

export function isKnownFreeModel(model: string): boolean {
return KNOWN_FREE_MODEL_IDS.has(getModelId(model))
}

export function isFreeOnlyProviderConfiguration(connectedProviders: string[] | null): boolean {
return connectedProviders !== null
&& connectedProviders.length > 0
&& connectedProviders.every((provider) => FREE_ONLY_PROVIDER_IDS.has(provider))
}

export function appendFreeModelFallbacks(
fallbackChain: FallbackEntry[] | undefined,
): FallbackEntry[] {
if (!fallbackChain || fallbackChain.length === 0) {
return FREE_ONLY_FALLBACK_CHAIN
}

const existingModels = new Set(fallbackChain.map((entry) => entry.model))
const newEntries = FREE_ONLY_FALLBACK_CHAIN.filter((entry) => !existingModels.has(entry.model))
return [...fallbackChain, ...newEntries]
}
40 changes: 40 additions & 0 deletions src/tools/delegate-task/model-selection-input.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import { normalizeModel } from "../../shared/model-normalization"
import { parseModelString, parseVariantFromModelID } from "../../shared/model-string-parser"

export function isExplicitHighModel(model: string): boolean {
return /(?:^|\/)[^/]+-high$/.test(model)
}

export function getExplicitHighBaseModel(model: string): string | null {
return isExplicitHighModel(model) ? model.replace(/-high$/, "") : null
}

export function parseUserFallbackModel(fallbackModel: string): {
baseModel: string
providerHint?: string[]
variant?: string
} | undefined {
const normalizedFallback = normalizeModel(fallbackModel)
if (!normalizedFallback) {
return undefined
}

const parsedFullModel = parseModelString(normalizedFallback)
if (parsedFullModel) {
return {
baseModel: `${parsedFullModel.providerID}/${parsedFullModel.modelID}`,
providerHint: [parsedFullModel.providerID],
variant: parsedFullModel.variant,
}
}

const parsedModel = parseVariantFromModelID(normalizedFallback)
if (!parsedModel.modelID) {
return undefined
}

return {
baseModel: parsedModel.modelID,
variant: parsedModel.variant,
}
}
Loading
Loading