diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index dad6c8804..5635dedc5 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -255,6 +255,15 @@ func WithParallelToolCalls(enabled bool) Option { } } +func WithThinking(budgetTokens int) Option { + return func(a *Agent) { + a.modelSettings.Thinking = &llm.ThinkingConfig{ + Enabled: true, + BudgetTokens: budgetTokens, + } + } +} + func WithLogger(l *log.Logger) Option { return func(a *Agent) { a.logger = l diff --git a/pkg/agent/model_settings.go b/pkg/agent/model_settings.go index 95980361e..49712fc3c 100644 --- a/pkg/agent/model_settings.go +++ b/pkg/agent/model_settings.go @@ -24,4 +24,5 @@ type ModelSettings struct { MaxTokens *int ToolChoice *llm.ToolChoice ParallelToolCalls *bool + Thinking *llm.ThinkingConfig } diff --git a/pkg/agent/progress.go b/pkg/agent/progress.go new file mode 100644 index 000000000..5ad3a4345 --- /dev/null +++ b/pkg/agent/progress.go @@ -0,0 +1,36 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package agent + +import "context" + +type ( + ProgressEventType string + + ProgressEvent struct { + Type ProgressEventType `json:"type"` + Step string `json:"step"` + ParentStep string `json:"parent_step,omitempty"` + Message string `json:"message"` + } + + ProgressReporter func(ctx context.Context, event ProgressEvent) +) + +const ( + ProgressEventStepStarted ProgressEventType = "step_started" + ProgressEventStepCompleted ProgressEventType = "step_completed" + ProgressEventStepFailed ProgressEventType = "step_failed" +) diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 88e294308..b8a524d27 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -306,6 +306,7 @@ func coreLoop(ctx context.Context, startAgent *Agent, inputMessages []llm.Messag ToolChoice: toolChoice, ParallelToolCalls: s.agent.modelSettings.ParallelToolCalls, ResponseFormat: responseFormat, + Thinking: s.agent.modelSettings.Thinking, } s.logger.InfoCtx( @@ -852,12 +853,20 @@ func executeSingleTool( emitHook(agent, func(h RunHooks) { h.OnToolEnd(ctx, agent, tool, result, nil) }) emitAgentHook(agent, func(h AgentHooks) { h.OnToolEnd(ctx, agent, tool, result) }) - logger.InfoCtx( - ctx, - "tool execution completed", - log.String("tool", tool.Name()), - log.Bool("is_error", result.IsError), - ) + if result.IsError { + logger.WarnCtx( + ctx, + "tool returned error", + log.String("tool", tool.Name()), + log.String("content", result.Content), + ) + } else { + logger.InfoCtx( + ctx, + "tool execution completed", + log.String("tool", tool.Name()), + ) + } return result, nil } diff --git a/pkg/agent/tools/browser/browser.go b/pkg/agent/tools/browser/browser.go new file mode 100644 index 000000000..9c4551d8b --- /dev/null +++ b/pkg/agent/tools/browser/browser.go @@ -0,0 +1,349 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +const defaultToolTimeout = 60 * time.Second + +// waitForPage returns chromedp actions that wait for the page to fully load, +// including SPA content rendered by JavaScript. It first waits for the body to +// be ready, then polls until the page content stabilizes (innerText stops +// changing) with a short debounce. After stabilization, it attempts to dismiss +// common cookie consent banners so they don't interfere with content +// extraction. +func waitForPage() chromedp.Action { + return chromedp.ActionFunc(func(ctx context.Context) error { + if err := chromedp.WaitReady("body").Do(ctx); err != nil { + return err + } + + // Wait for SPA content to stabilize by checking if innerText + // length stops changing over a 500ms window. Gives up after 5s. + // EvaluateAsDevTools is required to await the Promise. + if err := chromedp.EvaluateAsDevTools(` + new Promise((resolve) => { + let lastLen = -1; + let stableCount = 0; + const interval = setInterval(() => { + const curLen = document.body.innerText.length; + if (curLen === lastLen && curLen > 0) { + stableCount++; + } else { + stableCount = 0; + } + lastLen = curLen; + if (stableCount >= 2) { + clearInterval(interval); + resolve(true); + } + }, 250); + setTimeout(() => { + clearInterval(interval); + resolve(true); + }, 5000); + }) + `, nil).Do(ctx); err != nil { + return err + } + + // Dismiss common cookie consent banners. This is best-effort; + // failures are silently ignored because not every page has a + // banner and the selectors may not match. + return chromedp.Evaluate(` + (() => { + const selectors = [ + "#onetrust-accept-btn-handler", + "#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll", + "#CybotCookiebotDialogBodyButtonAccept", + ".cky-btn-accept", + "[data-testid='cookie-policy-dialog-accept-button']", + "button.accept-cookies", + "#cookie-accept", + "#accept-cookies", + ".cc-accept", + ".cc-btn.cc-dismiss", + ]; + for (const sel of selectors) { + const btn = document.querySelector(sel); + if (btn) { btn.click(); return; } + } + const buttons = document.querySelectorAll( + "button, a[role='button'], [role='button']" + ); + const patterns = /^(accept all|accept|agree|i agree|allow all|allow|got it|ok|okay|consent)$/i; + for (const btn of buttons) { + if (patterns.test(btn.innerText.trim())) { + btn.click(); + return; + } + } + })() + `, nil).Do(ctx) + }) +} + +type Browser struct { + addr string + allocCtx context.Context + cancel context.CancelFunc + allowedDomains []string +} + +func NewBrowser(ctx context.Context, addr string) *Browser { + if !strings.HasPrefix(addr, "ws://") && !strings.HasPrefix(addr, "wss://") { + addr = "ws://" + addr + } + + allocCtx, cancel := chromedp.NewRemoteAllocator(ctx, addr) + + return &Browser{ + addr: addr, + allocCtx: allocCtx, + cancel: cancel, + } +} + +// SetAllowedDomain restricts navigation to URLs under the given domain and +// its subdomains. For example, setting "getprobo.com" allows navigation to +// getprobo.com, www.getprobo.com, and compliance.getprobo.com. +// This replaces any previously set domains. +func (b *Browser) SetAllowedDomain(domain string) { + domain = strings.ToLower(strings.TrimSpace(domain)) + + // Strip "www." prefix so that setting either "www.example.com" or + // "example.com" allows navigation to *.example.com. + domain = strings.TrimPrefix(domain, "www.") + + b.allowedDomains = []string{domain} +} + +// checkPDF returns an error tool result if the URL points to a PDF file, +// which cannot be rendered by the headless browser. +func checkPDF(rawURL string) *agent.ToolResult { + if strings.HasSuffix(strings.ToLower(rawURL), ".pdf") { + return &agent.ToolResult{ + Content: fmt.Sprintf("cannot load %s: PDF files are not supported by the browser", rawURL), + IsError: true, + } + } + + return nil +} + +// checkURL validates that the URL is allowed. It returns an error tool result +// if the URL is outside the allowed domains. +func (b *Browser) checkURL(rawURL string) *agent.ToolResult { + if len(b.allowedDomains) == 0 { + return nil + } + + u, err := url.Parse(rawURL) + if err != nil { + return &agent.ToolResult{ + Content: fmt.Sprintf("invalid URL: %s", err), + IsError: true, + } + } + + host := strings.ToLower(u.Hostname()) + for _, allowed := range b.allowedDomains { + if host == allowed || strings.HasSuffix(host, "."+allowed) { + return nil + } + } + + return &agent.ToolResult{ + Content: fmt.Sprintf("navigation blocked: %s is outside the allowed domains", host), + IsError: true, + } +} + +// checkAlive returns a tool error result if the browser connection has been +// lost. Call this at the start of every tool to fail fast with a clear +// message instead of waiting for the tool timeout. +func (b *Browser) checkAlive() *agent.ToolResult { + if err := b.allocCtx.Err(); err != nil { + return &agent.ToolResult{ + Content: "browser connection lost: the remote Chrome instance is no longer reachable", + IsError: true, + } + } + return nil +} + +// classifyError inspects the caller's timeout context and the browser's +// allocator context to produce a human-readable error message. Without this, +// both a tool timeout and a dropped Chrome connection appear as the opaque +// "context canceled". +func (b *Browser) classifyError(timeoutCtx context.Context, rawURL string, err error) string { + if b.allocCtx.Err() != nil { + return fmt.Sprintf( + "browser connection lost while loading %s: the remote Chrome instance is no longer reachable", + rawURL, + ) + } + + if errors.Is(timeoutCtx.Err(), context.DeadlineExceeded) { + return fmt.Sprintf( + "page load timed out after %s for %s: the page may be too slow or unresponsive", + defaultToolTimeout, + rawURL, + ) + } + + return fmt.Sprintf("cannot load %s: %s", rawURL, err) +} + +func (b *Browser) NewTab(ctx context.Context) (context.Context, context.CancelFunc) { + tabCtx, tabCancel := chromedp.NewContext(b.allocCtx) + + // Propagate the caller's cancellation to the Chrome tab so that + // tool-level timeouts and context deadlines actually stop the browser. + go func() { + select { + case <-ctx.Done(): + tabCancel() + case <-tabCtx.Done(): + } + }() + + return tabCtx, tabCancel +} + +func (b *Browser) Close() { + b.cancel() +} + +// BuildReadOnlyTools returns browser tools that only read page content: +// navigate, extract text, extract links, and find links. It excludes +// interactive tools (click, select) that modify page state. Also includes +// standalone HTTP tools (robots.txt, sitemap, PDF download) that do not +// require the browser. +func BuildReadOnlyTools(b *Browser) ([]agent.Tool, error) { + navigateTool, err := NavigateToURLTool(b) + if err != nil { + return nil, err + } + + extractTextTool, err := ExtractPageTextTool(b) + if err != nil { + return nil, err + } + + extractLinksTool, err := ExtractLinksTool(b) + if err != nil { + return nil, err + } + + findLinksTool, err := FindLinksMatchingTool(b) + if err != nil { + return nil, err + } + + robotsTool, err := FetchRobotsTxtTool() + if err != nil { + return nil, err + } + + sitemapTool, err := FetchSitemapTool() + if err != nil { + return nil, err + } + + pdfTool, err := DownloadPDFTool() + if err != nil { + return nil, err + } + + return []agent.Tool{ + navigateTool, + extractTextTool, + extractLinksTool, + findLinksTool, + robotsTool, + sitemapTool, + pdfTool, + }, nil +} + +func BuildTools(b *Browser) ([]agent.Tool, error) { + navigateTool, err := NavigateToURLTool(b) + if err != nil { + return nil, err + } + + extractTextTool, err := ExtractPageTextTool(b) + if err != nil { + return nil, err + } + + extractLinksTool, err := ExtractLinksTool(b) + if err != nil { + return nil, err + } + + findLinksTool, err := FindLinksMatchingTool(b) + if err != nil { + return nil, err + } + + clickTool, err := ClickElementTool(b) + if err != nil { + return nil, err + } + + selectTool, err := SelectOptionTool(b) + if err != nil { + return nil, err + } + + robotsTool, err := FetchRobotsTxtTool() + if err != nil { + return nil, err + } + + sitemapTool, err := FetchSitemapTool() + if err != nil { + return nil, err + } + + pdfTool, err := DownloadPDFTool() + if err != nil { + return nil, err + } + + return []agent.Tool{ + navigateTool, + extractTextTool, + extractLinksTool, + findLinksTool, + clickTool, + selectTool, + robotsTool, + sitemapTool, + pdfTool, + }, nil +} diff --git a/pkg/agent/tools/browser/click.go b/pkg/agent/tools/browser/click.go new file mode 100644 index 000000000..1aee6eecc --- /dev/null +++ b/pkg/agent/tools/browser/click.go @@ -0,0 +1,74 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +type clickParams struct { + URL string `json:"url" jsonschema:"The URL to navigate to before clicking"` + Selector string `json:"selector" jsonschema:"CSS selector of the element to click (e.g. button.next, a[href*=page])"` +} + +func ClickElementTool(b *Browser) (agent.Tool, error) { + return agent.FunctionTool[clickParams]( + "click_element", + "Navigate to a URL, click an element matching a CSS selector, and return the page text after the click. Useful for pagination buttons, 'show all' links, tabs, and other interactive elements.", + func(ctx context.Context, p clickParams) (agent.ToolResult, error) { + if r := b.checkAlive(); r != nil { + return *r, nil + } + + if r := b.checkURL(p.URL); r != nil { + return *r, nil + } + + ctx, timeoutCancel := withToolTimeout(ctx) + defer timeoutCancel() + + tabCtx, cancel := b.NewTab(ctx) + defer cancel() + + var text string + + err := chromedp.Run( + tabCtx, + chromedp.Navigate(p.URL), + waitForPage(), + chromedp.WaitVisible(p.Selector), + chromedp.Click(p.Selector), + waitForPage(), + chromedp.Evaluate(`document.body.innerText`, &text), + ) + if err != nil { + return agent.ToolResult{ + Content: b.classifyError(ctx, p.URL, err), + IsError: true, + }, nil + } + + runes := []rune(text) + if len(runes) > maxTextLength { + text = string(runes[:maxTextLength]) + } + + return agent.ToolResult{Content: text}, nil + }, + ) +} diff --git a/pkg/agent/tools/browser/download_pdf.go b/pkg/agent/tools/browser/download_pdf.go new file mode 100644 index 000000000..ac32be962 --- /dev/null +++ b/pkg/agent/tools/browser/download_pdf.go @@ -0,0 +1,165 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/pdfcpu/pdfcpu/pkg/api" + "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model" + "go.probo.inc/probo/pkg/agent" +) + +type downloadPDFParams struct { + URL string `json:"url" jsonschema:"The URL of the PDF document to download and extract text from"` +} + +type downloadPDFResult struct { + Text string `json:"text"` + PageCount int `json:"page_count"` + ErrorDetail string `json:"error_detail,omitempty"` +} + +func DownloadPDFTool() (agent.Tool, error) { + client := &http.Client{Timeout: 30 * time.Second} + + return agent.FunctionTool[downloadPDFParams]( + "download_pdf", + "Download a PDF document from a URL and extract its text content. Use this for DPAs, SOC 2 reports, privacy policies, and other documents hosted as PDFs.", + func(ctx context.Context, p downloadPDFParams) (agent.ToolResult, error) { + if err := validatePublicURL(p.URL); err != nil { + data, _ := json.Marshal(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("URL not allowed: %s", err), + }) + return agent.ToolResult{Content: string(data), IsError: true}, nil + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.URL, nil) + if err != nil { + data, _ := json.Marshal(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot create request: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + + resp, err := client.Do(req) + if err != nil { + data, _ := json.Marshal(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot download PDF: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + data, _ := json.Marshal(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("PDF download returned status %d", resp.StatusCode), + }) + return agent.ToolResult{Content: string(data)}, nil + } + + // Read PDF into memory (max 20MB). + body, err := io.ReadAll(io.LimitReader(resp.Body, 20*1024*1024)) + if err != nil { + data, _ := json.Marshal(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot read PDF body: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + + // Write to temp file for pdfcpu. + tmpDir, err := os.MkdirTemp("", "pdf-extract-*") + if err != nil { + data, _ := json.Marshal(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot create temp dir: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + defer os.RemoveAll(tmpDir) + + tmpFile := filepath.Join(tmpDir, "input.pdf") + if err := os.WriteFile(tmpFile, body, 0o600); err != nil { + data, _ := json.Marshal(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot write temp file: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + + // Get page count. + conf := model.NewDefaultConfiguration() + pageCount, err := api.PageCountFile(tmpFile) + if err != nil { + data, _ := json.Marshal(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot read PDF: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + + // Extract content to output dir. + outDir := filepath.Join(tmpDir, "out") + if err := os.MkdirAll(outDir, 0o700); err != nil { + data, _ := json.Marshal(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot create output dir: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + + reader := bytes.NewReader(body) + if err := api.ExtractContent(reader, outDir, "content", nil, conf); err != nil { + data, _ := json.Marshal(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot extract PDF content: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + + // Read all extracted content files. + var sb strings.Builder + entries, _ := os.ReadDir(outDir) + for _, entry := range entries { + if entry.IsDir() { + continue + } + content, err := os.ReadFile(filepath.Join(outDir, entry.Name())) + if err != nil { + continue + } + sb.Write(content) + sb.WriteString("\n") + } + + text := sb.String() + if len(text) > maxTextLength { + text = text[:maxTextLength] + "\n[... truncated]" + } + + result := downloadPDFResult{ + Text: text, + PageCount: pageCount, + } + + data, _ := json.Marshal(result) + return agent.ToolResult{Content: string(data)}, nil + }, + ) +} diff --git a/pkg/agent/tools/browser/extract_links.go b/pkg/agent/tools/browser/extract_links.go new file mode 100644 index 000000000..3291fb830 --- /dev/null +++ b/pkg/agent/tools/browser/extract_links.go @@ -0,0 +1,88 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + "encoding/json" + "net/url" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +type extractLinksParams struct { + URL string `json:"url" jsonschema:"The URL to extract links from"` +} + +type link struct { + Href string `json:"href"` + Text string `json:"text"` +} + +func ExtractLinksTool(b *Browser) (agent.Tool, error) { + return agent.FunctionTool[extractLinksParams]( + "extract_links", + "Navigate to a URL and extract all links ( elements) with their href and text.", + func(ctx context.Context, p extractLinksParams) (agent.ToolResult, error) { + if r := b.checkAlive(); r != nil { + return *r, nil + } + + u, err := url.Parse(p.URL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") { + return agent.ToolResult{ + Content: "invalid URL scheme: only http and https are allowed", + IsError: true, + }, nil + } + + if r := b.checkURL(p.URL); r != nil { + return *r, nil + } + + ctx, timeoutCancel := withToolTimeout(ctx) + defer timeoutCancel() + + tabCtx, cancel := b.NewTab(ctx) + defer cancel() + + var links []link + + err = chromedp.Run( + tabCtx, + chromedp.Navigate(p.URL), + waitForPage(), + chromedp.Evaluate( + `Array.from(document.querySelectorAll("a[href]")).map(a => ({ + href: a.href, + text: a.innerText.trim().substring(0, 200) + }))`, + &links, + ), + ) + if err != nil { + return agent.ToolResult{ + Content: b.classifyError(ctx, p.URL, err), + IsError: true, + }, nil + } + + data, _ := json.Marshal(links) + + return agent.ToolResult{Content: string(data)}, nil + }, + ) +} diff --git a/pkg/agent/tools/browser/extract_text.go b/pkg/agent/tools/browser/extract_text.go new file mode 100644 index 000000000..e94bed95b --- /dev/null +++ b/pkg/agent/tools/browser/extract_text.go @@ -0,0 +1,83 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + "time" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +const maxTextLength = 32000 + +type extractTextParams struct { + URL string `json:"url" jsonschema:"The URL to extract text from"` +} + +func ExtractPageTextTool(b *Browser) (agent.Tool, error) { + return agent.FunctionTool[extractTextParams]( + "extract_page_text", + "Navigate to a URL and extract the visible text content of the page, truncated to 32000 characters.", + func(ctx context.Context, p extractTextParams) (agent.ToolResult, error) { + if r := b.checkAlive(); r != nil { + return *r, nil + } + + if r := b.checkURL(p.URL); r != nil { + return *r, nil + } + + if r := checkPDF(p.URL); r != nil { + return *r, nil + } + + ctx, timeoutCancel := withToolTimeout(ctx) + defer timeoutCancel() + + tabCtx, cancel := b.NewTab(ctx) + defer cancel() + + var text string + + err := chromedp.Run( + tabCtx, + chromedp.Navigate(p.URL), + waitForPage(), + // Scroll to bottom to trigger lazy-loaded content, + // then back to top and wait briefly for rendering. + chromedp.Evaluate(`window.scrollTo(0, document.body.scrollHeight)`, nil), + chromedp.Sleep(500*time.Millisecond), + chromedp.Evaluate(`window.scrollTo(0, 0)`, nil), + chromedp.Sleep(200*time.Millisecond), + chromedp.Evaluate(`String(document.body?.innerText ?? '')`, &text), + ) + if err != nil { + return agent.ToolResult{ + Content: b.classifyError(ctx, p.URL, err), + IsError: true, + }, nil + } + + runes := []rune(text) + if len(runes) > maxTextLength { + text = string(runes[:maxTextLength]) + } + + return agent.ToolResult{Content: text}, nil + }, + ) +} diff --git a/pkg/agent/tools/browser/fetch_robots.go b/pkg/agent/tools/browser/fetch_robots.go new file mode 100644 index 000000000..051e62acf --- /dev/null +++ b/pkg/agent/tools/browser/fetch_robots.go @@ -0,0 +1,107 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "go.probo.inc/probo/pkg/agent" +) + +type robotsParams struct { + Domain string `json:"domain" jsonschema:"The domain to fetch robots.txt from (e.g. example.com)"` +} + +type robotsResult struct { + Found bool `json:"found"` + Sitemaps []string `json:"sitemaps,omitempty"` + Disallowed []string `json:"disallowed_paths,omitempty"` + ErrorDetail string `json:"error_detail,omitempty"` +} + +func FetchRobotsTxtTool() (agent.Tool, error) { + client := &http.Client{Timeout: 10 * time.Second} + + return agent.FunctionTool[robotsParams]( + "fetch_robots_txt", + "Fetch and parse the robots.txt file for a domain. Returns sitemap URLs and disallowed paths, which can reveal hidden pages the crawler might miss.", + func(ctx context.Context, p robotsParams) (agent.ToolResult, error) { + if err := validatePublicDomain(p.Domain); err != nil { + data, _ := json.Marshal(robotsResult{ + Found: false, + ErrorDetail: fmt.Sprintf("domain not allowed: %s", err), + }) + return agent.ToolResult{Content: string(data), IsError: true}, nil + } + + u := "https://" + p.Domain + "/robots.txt" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + data, _ := json.Marshal(robotsResult{ + Found: false, + ErrorDetail: fmt.Sprintf("cannot create request: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + + resp, err := client.Do(req) + if err != nil { + data, _ := json.Marshal(robotsResult{ + Found: false, + ErrorDetail: fmt.Sprintf("cannot fetch robots.txt: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + data, _ := json.Marshal(robotsResult{ + Found: false, + ErrorDetail: fmt.Sprintf("robots.txt returned status %d", resp.StatusCode), + }) + return agent.ToolResult{Content: string(data)}, nil + } + + var result robotsResult + result.Found = true + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + if after, ok := strings.CutPrefix(strings.ToLower(line), "sitemap:"); ok { + result.Sitemaps = append(result.Sitemaps, strings.TrimSpace(line[len(line)-len(after):])) + } + + if after, ok := strings.CutPrefix(strings.ToLower(line), "disallow:"); ok { + path := strings.TrimSpace(after) + if path != "" && len(result.Disallowed) < 50 { + result.Disallowed = append(result.Disallowed, path) + } + } + } + + data, _ := json.Marshal(result) + return agent.ToolResult{Content: string(data)}, nil + }, + ) +} diff --git a/pkg/agent/tools/browser/fetch_sitemap.go b/pkg/agent/tools/browser/fetch_sitemap.go new file mode 100644 index 000000000..9c976df57 --- /dev/null +++ b/pkg/agent/tools/browser/fetch_sitemap.go @@ -0,0 +1,155 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "compress/gzip" + "context" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "net/http" + "strings" + "time" + + "go.probo.inc/probo/pkg/agent" +) + +type sitemapParams struct { + URL string `json:"url" jsonschema:"The full URL of the sitemap to fetch (e.g. https://example.com/sitemap.xml)"` +} + +type sitemapResult struct { + Found bool `json:"found"` + URLs []string `json:"urls,omitempty"` + URLCount int `json:"url_count"` + ErrorDetail string `json:"error_detail,omitempty"` +} + +const maxSitemapURLs = 200 + +func FetchSitemapTool() (agent.Tool, error) { + client := &http.Client{Timeout: 15 * time.Second} + + return agent.FunctionTool[sitemapParams]( + "fetch_sitemap", + "Fetch and parse a sitemap XML file. Returns discovered URLs which can reveal pages not linked from the main navigation (trust centers, legal docs, status pages).", + func(ctx context.Context, p sitemapParams) (agent.ToolResult, error) { + if err := validatePublicURL(p.URL); err != nil { + data, _ := json.Marshal(sitemapResult{ + Found: false, + ErrorDetail: fmt.Sprintf("URL not allowed: %s", err), + }) + return agent.ToolResult{Content: string(data), IsError: true}, nil + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.URL, nil) + if err != nil { + data, _ := json.Marshal(sitemapResult{ + Found: false, + ErrorDetail: fmt.Sprintf("cannot create request: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + + resp, err := client.Do(req) + if err != nil { + data, _ := json.Marshal(sitemapResult{ + Found: false, + ErrorDetail: fmt.Sprintf("cannot fetch sitemap: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + data, _ := json.Marshal(sitemapResult{ + Found: false, + ErrorDetail: fmt.Sprintf("sitemap returned status %d", resp.StatusCode), + }) + return agent.ToolResult{Content: string(data)}, nil + } + + var reader io.Reader = resp.Body + if strings.HasSuffix(strings.ToLower(p.URL), ".gz") || + resp.Header.Get("Content-Encoding") == "gzip" { + gz, err := gzip.NewReader(resp.Body) + if err != nil { + data, _ := json.Marshal(sitemapResult{ + Found: false, + ErrorDetail: fmt.Sprintf("cannot decompress gzipped sitemap: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + defer gz.Close() + reader = gz + } + + // Limit read to 5MB. + reader = io.LimitReader(reader, 5*1024*1024) + + urls, err := parseSitemapXML(reader) + if err != nil { + data, _ := json.Marshal(sitemapResult{ + Found: false, + ErrorDetail: fmt.Sprintf("cannot parse sitemap XML: %s", err), + }) + return agent.ToolResult{Content: string(data)}, nil + } + + result := sitemapResult{ + Found: true, + URLCount: len(urls), + } + + if len(urls) > maxSitemapURLs { + result.URLs = urls[:maxSitemapURLs] + } else { + result.URLs = urls + } + + data, _ := json.Marshal(result) + return agent.ToolResult{Content: string(data)}, nil + }, + ) +} + +func parseSitemapXML(r io.Reader) ([]string, error) { + var urls []string + decoder := xml.NewDecoder(r) + + for { + tok, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return urls, err + } + + if se, ok := tok.(xml.StartElement); ok && se.Name.Local == "loc" { + var loc string + if err := decoder.DecodeElement(&loc, &se); err == nil { + loc = strings.TrimSpace(loc) + if loc != "" { + urls = append(urls, loc) + } + } + } + } + + return urls, nil +} diff --git a/pkg/agent/tools/browser/find_links.go b/pkg/agent/tools/browser/find_links.go new file mode 100644 index 000000000..6f7c21b08 --- /dev/null +++ b/pkg/agent/tools/browser/find_links.go @@ -0,0 +1,98 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +type findLinksParams struct { + URL string `json:"url" jsonschema:"The URL to search for links"` + Pattern string `json:"pattern" jsonschema:"Keyword to filter links by (case-insensitive match on href or text)"` +} + +func FindLinksMatchingTool(b *Browser) (agent.Tool, error) { + return agent.FunctionTool[findLinksParams]( + "find_links_matching", + "Navigate to a URL and extract links whose href or text matches a keyword (case-insensitive).", + func(ctx context.Context, p findLinksParams) (agent.ToolResult, error) { + if r := b.checkAlive(); r != nil { + return *r, nil + } + + if r := b.checkURL(p.URL); r != nil { + return *r, nil + } + + if p.Pattern == "" { + return agent.ToolResult{ + Content: "pattern must not be empty", + IsError: true, + }, nil + } + + ctx, timeoutCancel := withToolTimeout(ctx) + defer timeoutCancel() + + tabCtx, cancel := b.NewTab(ctx) + defer cancel() + + var links []link + + js := fmt.Sprintf( + `(() => { + const pattern = %q.toLowerCase(); + const normalize = s => s.replace(/[-_\s]+/g, ""); + const normalizedPattern = normalize(pattern); + return Array.from(document.querySelectorAll("a[href]")) + .filter(a => { + const href = a.href.toLowerCase(); + const text = a.innerText.toLowerCase(); + return href.includes(pattern) || text.includes(pattern) + || normalize(href).includes(normalizedPattern) + || normalize(text).includes(normalizedPattern); + }) + .map(a => ({ + href: a.href, + text: a.innerText.trim().substring(0, 200) + })); + })()`, + p.Pattern, + ) + + err := chromedp.Run( + tabCtx, + chromedp.Navigate(p.URL), + waitForPage(), + chromedp.Evaluate(js, &links), + ) + if err != nil { + return agent.ToolResult{ + Content: b.classifyError(ctx, p.URL, err), + IsError: true, + }, nil + } + + data, _ := json.Marshal(links) + + return agent.ToolResult{Content: string(data)}, nil + }, + ) +} diff --git a/pkg/agent/tools/browser/navigate.go b/pkg/agent/tools/browser/navigate.go new file mode 100644 index 000000000..ca068d85d --- /dev/null +++ b/pkg/agent/tools/browser/navigate.go @@ -0,0 +1,98 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + "encoding/json" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +func withToolTimeout(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(ctx, defaultToolTimeout) +} + +type navigateParams struct { + URL string `json:"url" jsonschema:"The URL to navigate to"` +} + +type navigateResult struct { + Title string `json:"title"` + Description string `json:"description"` + FinalURL string `json:"final_url"` +} + +func NavigateToURLTool(b *Browser) (agent.Tool, error) { + return agent.FunctionTool[navigateParams]( + "navigate_to_url", + "Navigate to a URL and return the page title, meta description, and final URL after redirects.", + func(ctx context.Context, p navigateParams) (agent.ToolResult, error) { + if r := b.checkAlive(); r != nil { + return *r, nil + } + + if r := b.checkURL(p.URL); r != nil { + return *r, nil + } + + if r := checkPDF(p.URL); r != nil { + return *r, nil + } + + ctx, timeoutCancel := withToolTimeout(ctx) + defer timeoutCancel() + + tabCtx, cancel := b.NewTab(ctx) + defer cancel() + + var ( + title string + description string + finalURL string + ) + + err := chromedp.Run( + tabCtx, + chromedp.Navigate(p.URL), + waitForPage(), + chromedp.Title(&title), + chromedp.Evaluate( + `(() => { + const meta = document.querySelector('meta[name="description"]'); + return meta ? meta.getAttribute("content") : ""; + })()`, + &description, + ), + chromedp.Location(&finalURL), + ) + if err != nil { + return agent.ToolResult{ + Content: b.classifyError(ctx, p.URL, err), + IsError: true, + }, nil + } + + data, _ := json.Marshal(navigateResult{ + Title: title, + Description: description, + FinalURL: finalURL, + }) + + return agent.ToolResult{Content: string(data)}, nil + }, + ) +} diff --git a/pkg/agent/tools/browser/select.go b/pkg/agent/tools/browser/select.go new file mode 100644 index 000000000..9782983f6 --- /dev/null +++ b/pkg/agent/tools/browser/select.go @@ -0,0 +1,83 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + "fmt" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +type selectParams struct { + URL string `json:"url" jsonschema:"The URL to navigate to before selecting"` + Selector string `json:"selector" jsonschema:"CSS selector of the select element"` + Value string `json:"value" jsonschema:"The option value to select"` +} + +func SelectOptionTool(b *Browser) (agent.Tool, error) { + return agent.FunctionTool[selectParams]( + "select_option", + "Navigate to a URL, select an option from a