feat(tools): add a browser tool

.changeset/browser-tool.md

@@ -0,0 +1,5 @@
+---
+'@nanocollective/nanocoder': minor
+---
+
+Add a built-in `browser` tool (navigate, click, type, screenshot) on one shared headless Chromium page. Screenshots reach vision models as images. On OpenAI-compatible providers they follow the tool result as an image message. Models that models.dev lists as text-only get the caption without the image. Localhost is allowed. Private network and cloud metadata addresses are blocked. Chromium itself is installed with `npx playwright install chromium`.

docs/features/development-modes.md

@@ -104,7 +104,7 @@ Plan mode removes mutation tools and leaves only read-only and interaction tools
 
 `write_plan` is the one write plan mode allows, and it only ever writes to the session's own artifact directory — never to your project. It exists in plan mode only; the other modes do not have it.
 
-The following are **excluded**: the file mutation tools (`write_file`, `string_replace`, `diff_edit`, `file_op`, `lsp_format_document`), `execute_bash`, the task and walkthrough tools (`write_tasks`, `write_walkthrough`), and the git write tools (`git_add`, `git_commit`, `git_pr`). Custom tools are available only when they declare `approval: never` and `read_only: true`.
+The following are **excluded**: the file mutation tools (`write_file`, `string_replace`, `diff_edit`, `file_op`, `lsp_format_document`), `execute_bash`, `browser`, the task and walkthrough tools (`write_tasks`, `write_walkthrough`), and the git write tools (`git_add`, `git_commit`, `git_pr`). Custom tools are available only when they declare `approval: never` and `read_only: true`.
 
 Subagents started with `agent` in plan mode get the same exclusions, so delegating work does not open a path to a mutation.
 

package.json

@@ -106,6 +106,7 @@
 		"pino": "^10.1.0",
 		"pino-pretty": "^13.1.3",
 		"pino-roll": "^4.0.0",
+		"playwright-core": "^1.63.0",
 		"react": "^19.0.0",
 		"sonic-boom": "^5.0.0",
 		"string-width": "^8.2.2",

source/ai-sdk-client/chat/chat-handler.spec.ts

@@ -1085,3 +1085,64 @@ test('stopOnUnknownTool stops on a call to a tool that does not exist', t => {
 	);
 	t.false(stopOnUnknownTool({steps: []} as never) as boolean);
 });
+
+test('openai-compatible: a tool screenshot reaches the request body as an image_url part', async t => {
+	const {createOpenAICompatible} = await import('@ai-sdk/openai-compatible');
+	let body: {messages: Array<{role: string; content: unknown}>} | undefined;
+	const model = createOpenAICompatible({
+		name: 'Ollama',
+		baseURL: 'http://localhost:11434/v1',
+		fetch: async (_input, init) => {
+			body = JSON.parse(String(init?.body));
+			throw new Error('request captured');
+		},
+	})('llava');
+
+	await handleChat({
+		model,
+		currentModel: 'llava',
+		providerConfig: {
+			name: 'Ollama',
+			type: 'openai',
+			models: ['llava'],
+			config: {baseURL: 'http://localhost:11434/v1'},
+		},
+		messages: [
+			{role: 'user', content: 'check the page'},
+			{
+				role: 'assistant',
+				content: '',
+				tool_calls: [
+					{id: 'call_shot', function: {name: 'browser', arguments: {}}},
+				],
+			},
+			{
+				role: 'tool',
+				content: 'Screenshot of http://localhost:3000/',
+				tool_call_id: 'call_shot',
+				name: 'browser',
+				images: [{data: 'SCREENSHOT_B64', mediaType: 'image/jpeg'}],
+			},
+		],
+		tools: {},
+		callbacks: {},
+		maxRetries: 0,
+	}).catch(() => undefined);
+
+	const [, , toolMessage, imageMessage] = body?.messages ?? [];
+	t.deepEqual(toolMessage, {
+		role: 'tool',
+		tool_call_id: 'call_shot',
+		content: 'Screenshot of http://localhost:3000/',
+	});
+	t.deepEqual(imageMessage, {
+		role: 'user',
+		content: [
+			{type: 'text', text: 'Images from the tool results above.'},
+			{
+				type: 'image_url',
+				image_url: {url: 'data:image/jpeg;base64,SCREENSHOT_B64'},
+			},
+		],
+	});
+});

source/ai-sdk-client/chat/chat-handler.ts

@@ -7,6 +7,7 @@ import {
 	ToolCallRepairError,
 } from 'ai';
 import {MAX_TOOL_STEPS} from '@/constants';
+import {getModelAcceptsImages} from '@/models/models-dev-client';
 
 /**
  * Stop the SDK's step loop at a call to a tool that does not exist.
@@ -53,6 +54,7 @@ import {
 import {getSafeMemory} from '@/utils/logging/safe-process.js';
 import {
 	convertToModelMessages,
+	type ToolImageDelivery,
 	withCacheBreakpoints,
 } from '../converters/message-converter.js';
 import {convertAISDKToolCalls} from '../converters/tool-converter.js';
@@ -201,7 +203,21 @@ export async function handleChat(
 
 			// Convert messages to AI SDK v5 ModelMessage format
 			const promptCaching = isPromptCachingEnabled(providerConfig);
-			const convertedMessages = convertToModelMessages(finalNonSystemMessages);
+			// `*.chat` is the SDK's Chat Completions model: openai-compatible
+			// providers and Copilot's non-GPT-5 route.
+			const hasToolImages = finalNonSystemMessages.some(
+				m => m.role === 'tool' && !!m.images?.length,
+			);
+			const toolImages: ToolImageDelivery =
+				hasToolImages && (await getModelAcceptsImages(currentModel)) === false
+					? 'omit'
+					: typeof model !== 'string' && model.provider.endsWith('.chat')
+						? 'user-message'
+						: 'inline';
+			const convertedMessages = convertToModelMessages(
+				finalNonSystemMessages,
+				toolImages,
+			);
 			const modelMessages = promptCaching
 				? withCacheBreakpoints(convertedMessages, finalSystemContent)
 				: convertedMessages;

source/ai-sdk-client/converters/message-converter.spec.ts

@@ -262,6 +262,112 @@ test('convertToModelMessages converts tool message', t => {
 	t.is(content[0].output?.value, 'Tool result');
 });
 
+test('convertToModelMessages emits image-data parts for tool screenshots', t => {
+	const messages: Message[] = [
+		{
+			role: 'assistant',
+			content: '',
+			tool_calls: [
+				{id: 'call_shot', function: {name: 'browser', arguments: {}}},
+			],
+		},
+		{
+			role: 'tool',
+			content: 'Screenshot of http://localhost:3000/',
+			tool_call_id: 'call_shot',
+			name: 'browser',
+			images: [{data: 'abc', mediaType: 'image/jpeg'}],
+		},
+	];
+
+	const result = convertToModelMessages(messages);
+	const toolMsg = result[1];
+	t.is(toolMsg.role, 'tool');
+	if (toolMsg.role !== 'tool' || !Array.isArray(toolMsg.content)) {
+		t.fail('screenshot tool result must be a tool message');
+		return;
+	}
+	const part = toolMsg.content[0];
+	if (!part || part.type !== 'tool-result' || part.output.type !== 'content') {
+		t.fail('screenshot must use the SDK content tool-result output');
+		return;
+	}
+	t.deepEqual(part.output.value, [
+		{type: 'text', text: 'Screenshot of http://localhost:3000/'},
+		{type: 'image-data', data: 'abc', mediaType: 'image/jpeg'},
+	]);
+});
+
+test('convertToModelMessages moves tool images after the whole run of tool results', t => {
+	const messages: Message[] = [
+		{
+			role: 'assistant',
+			content: '',
+			tool_calls: [
+				{id: 'call_shot', function: {name: 'browser', arguments: {}}},
+				{id: 'call_read', function: {name: 'read_file', arguments: {}}},
+			],
+		},
+		{
+			role: 'tool',
+			content: 'Screenshot',
+			tool_call_id: 'call_shot',
+			name: 'browser',
+			images: [{data: 'abc', mediaType: 'image/jpeg'}],
+		},
+		{
+			role: 'tool',
+			content: 'file body',
+			tool_call_id: 'call_read',
+			name: 'read_file',
+		},
+		{role: 'user', content: 'next'},
+	];
+
+	const result = convertToModelMessages(messages, 'user-message');
+	t.deepEqual(
+		result.map(m => m.role),
+		['assistant', 'tool', 'tool', 'user', 'user'],
+	);
+	t.deepEqual(result[3], {
+		role: 'user',
+		content: [
+			{type: 'text', text: 'Images from the tool results above.'},
+			{type: 'image', image: 'data:image/jpeg;base64,abc', mediaType: 'image/jpeg'},
+		],
+	});
+	t.false(JSON.stringify(result.slice(0, 3)).includes('abc'));
+});
+
+test('convertToModelMessages drops tool images for a model without image input', t => {
+	const messages: Message[] = [
+		{
+			role: 'assistant',
+			content: '',
+			tool_calls: [
+				{id: 'call_shot', function: {name: 'browser', arguments: {}}},
+			],
+		},
+		{
+			role: 'tool',
+			content: 'Screenshot',
+			tool_call_id: 'call_shot',
+			name: 'browser',
+			images: [{data: 'abc', mediaType: 'image/jpeg'}],
+		},
+	];
+
+	const result = convertToModelMessages(messages, 'omit');
+	t.is(result.length, 2);
+	const serialized = JSON.stringify(result);
+	t.false(serialized.includes('abc'));
+	t.true(
+		serialized.includes(
+			'Screenshot\\n[Image not sent: this model does not accept image input.]',
+		),
+	);
+});
+
 test('convertToModelMessages emits a json output for structured tool results', t => {
 	const messages: Message[] = [
 		{

source/ai-sdk-client/converters/message-converter.ts

@@ -6,7 +6,7 @@ import type {
 	ToolCallPart,
 	UserContent,
 } from 'ai';
-import type {Message} from '@/types/index';
+import type {ImageAttachment, Message} from '@/types/index';
 import {getLogger} from '@/utils/logging';
 import {filterModelFacing} from '@/utils/message-visibility';
 import {truncateToolResult} from '@/utils/truncate-tool-result';
@@ -124,6 +124,54 @@ export function withCacheBreakpoints(
 	return marked;
 }
 
+/**
+ * Where tool-result images go.
+ * - inline: inside the tool result (Anthropic, Google, OpenAI Responses).
+ * - user-message: Chat Completions only allows text in a tool message, and
+ *   the SDK would stringify the base64 into it. Send the images in a user
+ *   message after the run of tool results instead.
+ * - omit: the model takes no image input.
+ */
+export type ToolImageDelivery = 'inline' | 'user-message' | 'omit';
+
+function placeToolImages(
+	messages: Message[],
+	delivery: ToolImageDelivery,
+): Message[] {
+	if (delivery === 'inline') {
+		return messages;
+	}
+	const placed: Message[] = [];
+	let pending: ImageAttachment[] = [];
+	messages.forEach((msg, index) => {
+		if (msg.role === 'tool' && msg.images && msg.images.length > 0) {
+			const {images, ...text} = msg;
+			if (delivery === 'omit') {
+				placed.push({
+					...text,
+					content: `${text.content}\n[Image not sent: this model does not accept image input.]`,
+				});
+			} else {
+				placed.push(text);
+				pending.push(...images);
+			}
+		} else {
+			placed.push(msg);
+		}
+		// Every tool result has to follow its assistant turn directly, so the
+		// image message waits until the run of tool results ends.
+		if (pending.length > 0 && messages[index + 1]?.role !== 'tool') {
+			placed.push({
+				role: 'user',
+				content: 'Images from the tool results above.',
+				images: pending,
+			});
+			pending = [];
+		}
+	});
+	return placed;
+}
+
 /**
  * Convert our Message format to AI SDK v6 ModelMessage format
  *
@@ -137,7 +185,10 @@ export function withCacheBreakpoints(
  * carrying tool_calls would take its answering tool results down with it —
  * that combination is a bug, so warn loudly rather than fail silently.
  */
-export function convertToModelMessages(messages: Message[]): ModelMessage[] {
+export function convertToModelMessages(
+	messages: Message[],
+	toolImages: ToolImageDelivery = 'inline',
+): ModelMessage[] {
 	const modelFacing = filterModelFacing(messages);
 	if (modelFacing.length !== messages.length) {
 		for (const msg of messages) {
@@ -152,13 +203,39 @@ export function convertToModelMessages(messages: Message[]): ModelMessage[] {
 			}
 		}
 	}
-	return dropOrphanedToolResults(modelFacing).map((msg): ModelMessage => {
+	const paired = placeToolImages(
+		dropOrphanedToolResults(modelFacing),
+		toolImages,
+	);
+	return paired.map((msg): ModelMessage => {
 		if (msg.role === 'tool') {
-			// Convert to AI SDK tool-result format
-			// AI SDK expects: { role: 'tool', content: [{ type: 'tool-result', toolCallId, toolName, output }] }
-			// where output is { type: 'text', value: string } or { type: 'json', value: JSONValue }.
-			// Structured tool results travel as JSON so the model can reason over
-			// the typed shape; everything else falls back to the text content.
+			// Convert to AI SDK tool-result format.
+			// output is text, json, or content. content is the SDK's multimodal
+			// tool result: text parts plus image-data. Anthropic, Google, and
+			// OpenAI Responses each map image-data onto their own image block.
+			if (msg.images && msg.images.length > 0) {
+				return {
+					role: 'tool',
+					content: [
+						{
+							type: 'tool-result',
+							toolCallId: msg.tool_call_id || '',
+							toolName: msg.name || '',
+							output: {
+								type: 'content',
+								value: [
+									{type: 'text', text: truncateToolResult(msg.content)},
+									...msg.images.map(image => ({
+										type: 'image-data' as const,
+										data: image.data,
+										mediaType: image.mediaType,
+									})),
+								],
+							},
+						},
+					],
+				};
+			}
 			let output;
 			if (msg.structuredContent === undefined) {
 				output = {

source/app/prompts/sections/browser.md

@@ -0,0 +1,6 @@
+## BROWSER
+
+- `browser` drives one shared headless Chromium page for the whole session.
+- `navigate` takes an absolute http or https URL, including localhost. Private network and cloud metadata addresses are blocked.
+- `click` and `type` take a CSS selector. `type` sets the field value.
+- After navigate or a change, call `screenshot`. The image comes back on the tool result. Do not ask the user to paste one.