From 3e7c72f07e08a7ca06382eae267c7a63a9f9cdd2 Mon Sep 17 00:00:00 2001 From: Aari Date: Fri, 7 Aug 2026 09:57:39 +0800 Subject: [PATCH 1/2] feat(frontend): add Browser Live to Custom Agent chats --- README.md | 2 + frontend/AGENTS.md | 2 +- .../[agent_name]/chats/[thread_id]/layout.tsx | 12 +-- .../[agent_name]/chats/[thread_id]/page.tsx | 11 ++- frontend/tests/e2e/agent-chat.spec.ts | 92 +++++++++++++++++++ frontend/tests/e2e/utils/mock-api.ts | 1 + 6 files changed, 108 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 958cd4e5363..a29e517a500 100644 --- a/README.md +++ b/README.md @@ -1009,6 +1009,8 @@ uv run playwright install chromium Then uncomment the `group: browser` tool entries in `config.yaml` (`browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_get_text`, `browser_back`, `browser_screenshot`, `browser_close`). `make dev` / Docker startup detects an enabled `browser_navigate` tool and preserves the `browser` extra on dependency syncs. The Gateway fails startup if browser control is configured but Playwright is missing, and `/api/features` hides the Browser UI unless the backend can actually serve it. Keep `headless: true` and `allow_private_addresses: false` for anything but local, trusted debugging. Attaching to an existing Chrome with `cdp_url` cannot enforce DeerFlow's subresource/redirect SSRF guard and therefore fails closed unless `allow_unguarded_cdp: true` explicitly acknowledges that risk; use it only with a trusted local browser. Browser sessions are process-local; keep `GATEWAY_WORKERS=1` while this tool group is enabled because ordinary uvicorn worker dispatch does not provide thread affinity. +Custom Agent chats expose the same Browser Live controls when browser control is available and the agent either leaves `tool_groups` unrestricted or includes the `browser` group. An explicit tool-group allowlist without `browser` keeps those controls hidden. + The workspace Browser Live client negotiates binary JPEG WebSocket frames, keeps only the newest pending frame per display refresh, and revokes replaced object URLs. Gateway control messages remain JSON, and clients that do not diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md index c71ad74306c..31caf907e92 100644 --- a/frontend/AGENTS.md +++ b/frontend/AGENTS.md @@ -135,7 +135,7 @@ Edit-and-rerun is deliberately latest-turn-only. `core/messages/utils.ts::getLat - `src/app/workspace/chats/[thread_id]/page.tsx` owns composer busy-state wiring. - `src/app/workspace/chats/[thread_id]/page.tsx` owns branch-from-turn submission and navigation; sidecar `MessageList` instances do not receive the branch action. - `src/app/workspace/chats/[thread_id]/page.tsx` and `src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx` own edit-and-rerun submission wiring because the page must preserve normal/custom-agent run context; `MessageList` only detects the latest editable user turn and renders the inline editor. -- `src/app/workspace/chats/[thread_id]/page.tsx` gates the Workspace Browser trigger and browser right panel on `/api/features -> browser_control.enabled`; default/failed feature discovery hides the browser control so optional backend installs do not show a dead Live socket. +- `src/app/workspace/chats/[thread_id]/page.tsx` gates the Workspace Browser trigger and browser right panel on `/api/features -> browser_control.enabled`; `src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx` applies the same capability gate and additionally requires the Custom Agent's tool groups to be unrestricted or include `browser`. Default/failed feature discovery hides the browser control so optional backend installs do not show a dead Live socket. - `src/app/workspace/chats/[thread_id]/page.tsx` and `src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx` own active-goal display state for their composer overlays. - `src/components/workspace/messages/message-list.tsx` owns human-input card answered/latest/pending gating; entry pages only translate a submitted card response into `sendMessage` calls. - `src/components/workspace/browser-view/browser-view-panel.tsx` forwards each physical pointer click as one `click` input; do not also emit `down`/`up` for the same gesture because the remote Playwright click would run twice. diff --git a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/layout.tsx b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/layout.tsx index 68f51b607b7..811be80f09b 100644 --- a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/layout.tsx +++ b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/layout.tsx @@ -1,19 +1,11 @@ "use client"; -import { PromptInputProvider } from "@/components/ai-elements/prompt-input"; -import { ArtifactsProvider } from "@/components/workspace/artifacts"; -import { SubtasksProvider } from "@/core/tasks/context"; +import { ChatProviders } from "@/components/workspace/chats/chat-providers"; export default function AgentChatLayout({ children, }: { children: React.ReactNode; }) { - return ( - - - {children} - - - ); + return {children}; } diff --git a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx index e3f6c80ba17..4070d40d1a6 100644 --- a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx +++ b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx @@ -9,6 +9,7 @@ import { Button } from "@/components/ui/button"; import { SidebarTrigger } from "@/components/ui/sidebar"; import { AgentWelcome } from "@/components/workspace/agent-welcome"; import { ArtifactTrigger } from "@/components/workspace/artifacts"; +import { BrowserTrigger } from "@/components/workspace/browser-view"; import { ChatBox, useThreadChat } from "@/components/workspace/chats"; import { ContextUsageBadge } from "@/components/workspace/context-usage-badge"; import { ExportTrigger } from "@/components/workspace/export-trigger"; @@ -32,6 +33,7 @@ import { TokenUsageIndicator } from "@/components/workspace/token-usage-indicato import { Tooltip } from "@/components/workspace/tooltip"; import { useActiveGoal } from "@/components/workspace/use-active-goal"; import { useAgent } from "@/core/agents"; +import { useBrowserControlEnabled } from "@/core/features"; import { useI18n } from "@/core/i18n/hooks"; import { buildHumanInputResponseText, @@ -74,6 +76,7 @@ export default function AgentChatPage() { const [isWelcomeMode, setIsWelcomeMode] = useState(isNewThread); const [settings, setSettings] = useThreadSettings(threadId); const [localSettings, setLocalSettings] = useLocalSettings(); + const { enabled: browserControlEnabled } = useBrowserControlEnabled(); const { tokenUsageEnabled } = useModels(); const threadTokenUsage = useThreadTokenUsage( isNewThread || isMock ? undefined : threadId, @@ -225,6 +228,11 @@ export default function AgentChatPage() { ? localSettings.tokenUsage.inlineMode : "off"; const hasTodos = (thread.values.todos?.length ?? 0) > 0; + const agentBrowserEnabled = + agent !== null && + (agent.tool_groups == null || agent.tool_groups.includes("browser")); + const browserEnabled = + !isNewThread && !isMock && browserControlEnabled && agentBrowserEnabled; const { activeGoal, hasGoal, setLocalGoal } = useActiveGoal( threadId, thread.values.goal, @@ -245,7 +253,7 @@ export default function AgentChatPage() { context={{ ...settings.context, agent_name }} isMock={isMock} > - +
)} + {browserEnabled && }
diff --git a/frontend/tests/e2e/agent-chat.spec.ts b/frontend/tests/e2e/agent-chat.spec.ts index 4b3ffc811c2..97ce8df44d3 100644 --- a/frontend/tests/e2e/agent-chat.spec.ts +++ b/frontend/tests/e2e/agent-chat.spec.ts @@ -78,6 +78,98 @@ test.describe("Agent chat", () => { ).toBeVisible({ timeout: 15_000 }); }); + for (const { name, toolGroups, browserControlEnabled, expectedVisible } of [ + { + name: "shows Browser Live for an explicit browser tool group", + toolGroups: ["browser"], + browserControlEnabled: true, + expectedVisible: true, + }, + { + name: "shows Browser Live when tool groups are unrestricted", + toolGroups: null, + browserControlEnabled: true, + expectedVisible: true, + }, + { + name: "hides Browser Live without the browser tool group", + toolGroups: ["web"], + browserControlEnabled: true, + expectedVisible: false, + }, + { + name: "hides Browser Live when browser control is unavailable", + toolGroups: ["browser"], + browserControlEnabled: false, + expectedVisible: false, + }, + ]) { + test(name, async ({ page }) => { + const agent = { + name: "browser-agent", + description: "A custom agent for Browser Live tests", + tool_groups: toolGroups, + }; + mockLangGraphAPI(page, { + agents: [agent], + features: { browserControlEnabled }, + threads: [ + { + thread_id: MOCK_THREAD_ID, + title: "Browser agent conversation", + agent_name: agent.name, + messages: [ + { + type: "ai", + id: "msg-ai-browser-agent", + content: "Ready to browse", + }, + ], + }, + ], + }); + + await page.goto( + `/workspace/agents/${agent.name}/chats/${MOCK_THREAD_ID}`, + ); + await expect(page.getByText("Ready to browse")).toBeVisible({ + timeout: 15_000, + }); + + const browserTrigger = page.getByTestId("browser-trigger"); + if (expectedVisible) { + await expect(browserTrigger).toBeVisible(); + await browserTrigger.click(); + await expect( + page.getByPlaceholder("Enter a URL and press Enter"), + ).toBeVisible(); + } else { + await expect(browserTrigger).toHaveCount(0); + } + }); + } + + test("hides Browser Live before a custom-agent thread is created", async ({ + page, + }) => { + mockLangGraphAPI(page, { + agents: [ + { + name: "browser-agent", + description: "A custom agent for Browser Live tests", + tool_groups: ["browser"], + }, + ], + features: { browserControlEnabled: true }, + }); + + await page.goto("/workspace/agents/browser-agent/chats/new"); + await expect(page.getByPlaceholder(/how can i assist you/i)).toBeVisible({ + timeout: 15_000, + }); + await expect(page.getByTestId("browser-trigger")).toHaveCount(0); + }); + test("agent chat can regenerate its latest response", async ({ page }) => { const humanMessage = { type: "human", diff --git a/frontend/tests/e2e/utils/mock-api.ts b/frontend/tests/e2e/utils/mock-api.ts index a2966b191c5..ad636487b63 100644 --- a/frontend/tests/e2e/utils/mock-api.ts +++ b/frontend/tests/e2e/utils/mock-api.ts @@ -46,6 +46,7 @@ export type MockAgent = { name: string; description?: string; system_prompt?: string; + tool_groups?: string[] | null; }; export type MockSkill = { From 36b483a74fe156b9260fbdd74ab07979ced085ce Mon Sep 17 00:00:00 2001 From: Aari Date: Fri, 7 Aug 2026 10:23:59 +0800 Subject: [PATCH 2/2] test(frontend): cover mock Custom Agent Browser Live --- README.md | 2 +- frontend/tests/e2e/agent-chat.spec.ts | 35 +++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a29e517a500..17c14a039e0 100644 --- a/README.md +++ b/README.md @@ -1009,7 +1009,7 @@ uv run playwright install chromium Then uncomment the `group: browser` tool entries in `config.yaml` (`browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_get_text`, `browser_back`, `browser_screenshot`, `browser_close`). `make dev` / Docker startup detects an enabled `browser_navigate` tool and preserves the `browser` extra on dependency syncs. The Gateway fails startup if browser control is configured but Playwright is missing, and `/api/features` hides the Browser UI unless the backend can actually serve it. Keep `headless: true` and `allow_private_addresses: false` for anything but local, trusted debugging. Attaching to an existing Chrome with `cdp_url` cannot enforce DeerFlow's subresource/redirect SSRF guard and therefore fails closed unless `allow_unguarded_cdp: true` explicitly acknowledges that risk; use it only with a trusted local browser. Browser sessions are process-local; keep `GATEWAY_WORKERS=1` while this tool group is enabled because ordinary uvicorn worker dispatch does not provide thread affinity. -Custom Agent chats expose the same Browser Live controls when browser control is available and the agent either leaves `tool_groups` unrestricted or includes the `browser` group. An explicit tool-group allowlist without `browser` keeps those controls hidden. +Existing, non-mock Custom Agent chats expose the same Browser Live controls when browser control is available and the agent either leaves `tool_groups` unrestricted or includes the `browser` group. An explicit tool-group allowlist without `browser` keeps those controls hidden. The workspace Browser Live client negotiates binary JPEG WebSocket frames, keeps only the newest pending frame per display refresh, and revokes replaced diff --git a/frontend/tests/e2e/agent-chat.spec.ts b/frontend/tests/e2e/agent-chat.spec.ts index 97ce8df44d3..5f680e05a0b 100644 --- a/frontend/tests/e2e/agent-chat.spec.ts +++ b/frontend/tests/e2e/agent-chat.spec.ts @@ -78,7 +78,13 @@ test.describe("Agent chat", () => { ).toBeVisible({ timeout: 15_000 }); }); - for (const { name, toolGroups, browserControlEnabled, expectedVisible } of [ + for (const { + name, + toolGroups, + browserControlEnabled, + expectedVisible, + mock, + } of [ { name: "shows Browser Live for an explicit browser tool group", toolGroups: ["browser"], @@ -103,6 +109,13 @@ test.describe("Agent chat", () => { browserControlEnabled: false, expectedVisible: false, }, + { + name: "hides Browser Live in mock custom-agent chats", + toolGroups: ["browser"], + browserControlEnabled: true, + expectedVisible: false, + mock: true, + }, ]) { test(name, async ({ page }) => { const agent = { @@ -129,12 +142,24 @@ test.describe("Agent chat", () => { ], }); + const featuresLoaded = page.waitForResponse( + (response) => + new URL(response.url()).pathname === "/api/features" && + response.status() === 200, + ); await page.goto( - `/workspace/agents/${agent.name}/chats/${MOCK_THREAD_ID}`, + `/workspace/agents/${agent.name}/chats/${MOCK_THREAD_ID}${mock ? "?mock=true" : ""}`, ); - await expect(page.getByText("Ready to browse")).toBeVisible({ - timeout: 15_000, - }); + await featuresLoaded; + if (mock) { + await expect( + page.locator("header span", { hasText: agent.name }), + ).toBeVisible({ timeout: 15_000 }); + } else { + await expect(page.getByText("Ready to browse")).toBeVisible({ + timeout: 15_000, + }); + } const browserTrigger = page.getByTestId("browser-trigger"); if (expectedVisible) {