diff --git a/src/commands/webscraper/apify/connect.test.ts b/src/commands/webscraper/apify/connect.test.ts new file mode 100644 index 00000000..fcd956fa --- /dev/null +++ b/src/commands/webscraper/apify/connect.test.ts @@ -0,0 +1,269 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { Command } from 'commander'; + +const apiMock = vi.hoisted(() => ({ + startApifyCliFlow: vi.fn(), + pollApifyConnection: vi.fn(), + fetchApifyConnection: vi.fn(), +})); +const apifyConfigMock = vi.hoisted(() => ({ + storeApifyToken: vi.fn(), +})); +vi.mock('../../../lib/api/webscraper.js', () => ({ ...apiMock, ...apifyConfigMock })); + +const configMock = vi.hoisted(() => ({ + getProjectConfig: vi.fn(() => ({ project_id: 'p1', project_name: 'Test Project' })), + getAccessToken: vi.fn((): string | null => 'tok'), +})); +vi.mock('../../../lib/config.js', () => configMock); + +const analyticsMock = vi.hoisted(() => ({ + trackGroupCommand: vi.fn(), + shutdownAnalytics: vi.fn(async () => {}), +})); +vi.mock('../../../lib/analytics.js', () => analyticsMock); + +const bridgeMock = vi.hoisted(() => ({ + runApifyAuthBridge: vi.fn(async () => ({ skillsInstalled: true })), +})); +vi.mock('../../../lib/apify-bridge.js', () => bridgeMock); + +vi.mock('../../../lib/prompts.js', () => ({ isInteractive: false })); + +// `open` is loaded dynamically inside runConnectFlow (OAuth path); mock so the +// real browser launch doesn't fire during tests. +vi.mock('open', () => ({ default: vi.fn() })); + +// Silence interactive UI noise from clack — tests assert on mocks, not stdout. +vi.mock('@clack/prompts', async (orig) => { + const actual = (await orig()) as Record; + return { + ...actual, + intro: vi.fn(), + outro: vi.fn(), + log: { info: vi.fn(), success: vi.fn(), warn: vi.fn(), error: vi.fn() }, + spinner: vi.fn(() => ({ start: vi.fn(), stop: vi.fn(), message: vi.fn() })), + }; +}); + +const outputMock = vi.hoisted(() => ({ + outputJson: vi.fn(), + outputSuccess: vi.fn(), +})); +vi.mock('../../../lib/output.js', () => outputMock); + +// Imports must come AFTER the vi.mock calls because Vitest hoists the mocks +// but ESM module evaluation order still matters. +import { registerApifyConnectCommand } from './connect.js'; +import { CLIError } from '../../../lib/errors.js'; + +interface RunResult { + exitCode?: number; +} + +// Override process.exit so handleError doesn't kill the test process; capture +// the first exit code. Mirrors src/commands/posthog/setup.test.ts. +async function runConnect(argv: string[]): Promise { + const program = new Command(); + program.option('--json').option('--api-url ').option('-y, --yes'); + const webscraper = program.command('webscraper'); + const apify = webscraper.command('apify'); + registerApifyConnectCommand(apify); + + const origExit = process.exit; + const result: RunResult = {}; + (process.exit as unknown) = (code?: number) => { + if (result.exitCode === undefined) result.exitCode = code; + throw new Error('__exit__'); + }; + try { + await program + .parseAsync(['node', 'test', 'webscraper', 'apify', 'connect', ...argv]) + .catch((err) => { + if (err instanceof Error && err.message === '__exit__') return; + throw err; + }); + } finally { + process.exit = origExit; + } + return result; +} + +beforeEach(() => { + apiMock.startApifyCliFlow.mockReset(); + apiMock.pollApifyConnection.mockReset(); + apiMock.fetchApifyConnection.mockReset(); + apifyConfigMock.storeApifyToken.mockReset(); + bridgeMock.runApifyAuthBridge.mockReset().mockResolvedValue({ skillsInstalled: true }); + outputMock.outputJson.mockReset(); + outputMock.outputSuccess.mockReset(); + analyticsMock.trackGroupCommand.mockReset(); + analyticsMock.shutdownAnalytics.mockClear(); + configMock.getProjectConfig.mockReturnValue({ project_id: 'p1', project_name: 'Test Project' }); + configMock.getAccessToken.mockReturnValue('tok'); +}); + +describe('apify connect', () => { + describe('--token path (self-hosted)', () => { + it('stores the token and skips OAuth entirely', async () => { + apifyConfigMock.storeApifyToken.mockResolvedValue({ + configured: true, + maskedKey: 'apify_ap••••••••mnop', + }); + + await runConnect(['--token', 'apify_api_tok1234567890']); + + expect(apifyConfigMock.storeApifyToken).toHaveBeenCalledWith('apify_api_tok1234567890'); + expect(apiMock.startApifyCliFlow).not.toHaveBeenCalled(); + expect(apiMock.pollApifyConnection).not.toHaveBeenCalled(); + expect(apiMock.fetchApifyConnection).not.toHaveBeenCalled(); + expect(bridgeMock.runApifyAuthBridge).not.toHaveBeenCalled(); + }); + + it('does not require a login token', async () => { + configMock.getAccessToken.mockReturnValue(null); + apifyConfigMock.storeApifyToken.mockResolvedValue({ + configured: true, + maskedKey: 'apify_ap••••••••mnop', + }); + + const r = await runConnect(['--token', 'apify_api_tok1234567890']); + + expect(r.exitCode).toBeUndefined(); + expect(apifyConfigMock.storeApifyToken).toHaveBeenCalledOnce(); + }); + + it('prints the masked key, never the raw token, in non-json mode', async () => { + apifyConfigMock.storeApifyToken.mockResolvedValue({ + configured: true, + maskedKey: 'apify_ap••••••••mnop', + }); + + await runConnect(['--token', 'apify_api_tok1234567890']); + + expect(outputMock.outputSuccess).toHaveBeenCalledWith( + expect.stringContaining('apify_ap••••••••mnop'), + ); + const allCalls = [...outputMock.outputSuccess.mock.calls, ...outputMock.outputJson.mock.calls]; + const serialized = JSON.stringify(allCalls); + expect(serialized).not.toContain('apify_api_tok1234567890'); + }); + + it('emits the masked token status in --json mode', async () => { + apifyConfigMock.storeApifyToken.mockResolvedValue({ + configured: true, + maskedKey: 'apify_ap••••••••mnop', + }); + + await runConnect(['--json', '--token', 'apify_api_tok1234567890']); + + expect(outputMock.outputJson).toHaveBeenCalledOnce(); + const payload = outputMock.outputJson.mock.calls[0][0] as { + success: boolean; + connectionState: string; + token: { configured: boolean; maskedKey: string | null }; + }; + expect(payload.success).toBe(true); + expect(payload.connectionState).toBe('newly-connected'); + expect(payload.token).toEqual({ configured: true, maskedKey: 'apify_ap••••••••mnop' }); + }); + + it('propagates a rejected token as a CLI error and exits non-zero', async () => { + apifyConfigMock.storeApifyToken.mockRejectedValue( + new CLIError('Apify rejected this API token.', 1, 'INVALID_INPUT', 400), + ); + + const r = await runConnect(['--token', 'bogus']); + + expect(r.exitCode).toBe(1); + }); + + it('rejects an empty --token locally instead of falling through to OAuth', async () => { + const r = await runConnect(['--token', '']); + + expect(r.exitCode).toBe(1); + expect(apifyConfigMock.storeApifyToken).not.toHaveBeenCalled(); + expect(apiMock.startApifyCliFlow).not.toHaveBeenCalled(); + expect(apiMock.pollApifyConnection).not.toHaveBeenCalled(); + expect(apiMock.fetchApifyConnection).not.toHaveBeenCalled(); + expect(bridgeMock.runApifyAuthBridge).not.toHaveBeenCalled(); + }); + + it('rejects a whitespace-only --token locally instead of falling through to OAuth', async () => { + const r = await runConnect(['--token', ' ']); + + expect(r.exitCode).toBe(1); + expect(apifyConfigMock.storeApifyToken).not.toHaveBeenCalled(); + expect(apiMock.startApifyCliFlow).not.toHaveBeenCalled(); + expect(bridgeMock.runApifyAuthBridge).not.toHaveBeenCalled(); + }); + }); + + describe('OAuth path (--token absent) is unchanged', () => { + it('fast path: cli-start says connected → verifies via /connection, skips polling, runs the auth bridge', async () => { + apiMock.startApifyCliFlow.mockResolvedValue({ type: 'connected' }); + apiMock.fetchApifyConnection.mockResolvedValue({ + kind: 'connected', + connection: { apifyUsername: 'someone', plan: 'free', status: 'active' }, + }); + + await runConnect(['--skip-browser']); + + expect(apiMock.startApifyCliFlow).toHaveBeenCalledOnce(); + expect(apiMock.fetchApifyConnection).toHaveBeenCalledOnce(); + expect(apiMock.pollApifyConnection).not.toHaveBeenCalled(); + expect(bridgeMock.runApifyAuthBridge).toHaveBeenCalledOnce(); + expect(apifyConfigMock.storeApifyToken).not.toHaveBeenCalled(); + }); + + it('slow path: cli-start returns authorizeUrl → polls until connected', async () => { + apiMock.startApifyCliFlow.mockResolvedValue({ + type: 'authorize', + authorizeUrl: 'https://example.com/auth', + }); + apiMock.pollApifyConnection.mockResolvedValue({ + apifyUsername: 'someone', + plan: 'free', + status: 'active', + }); + + await runConnect(['--skip-browser']); + + expect(apiMock.pollApifyConnection).toHaveBeenCalledOnce(); + expect(apiMock.fetchApifyConnection).not.toHaveBeenCalled(); + expect(apifyConfigMock.storeApifyToken).not.toHaveBeenCalled(); + }); + + it('still requires a login token, exactly as before', async () => { + configMock.getAccessToken.mockReturnValue(null); + apiMock.startApifyCliFlow.mockResolvedValue({ type: 'connected' }); + + const r = await runConnect(['--skip-browser']); + + expect(r.exitCode).toBeGreaterThan(0); + expect(apiMock.startApifyCliFlow).not.toHaveBeenCalled(); + }); + + it('emits JSON with connectionState and connection, no token field', async () => { + apiMock.startApifyCliFlow.mockResolvedValue({ type: 'connected' }); + apiMock.fetchApifyConnection.mockResolvedValue({ + kind: 'connected', + connection: { apifyUsername: 'someone', plan: 'free', status: 'active' }, + }); + + await runConnect(['--json', '--skip-browser']); + + expect(outputMock.outputJson).toHaveBeenCalledOnce(); + const payload = outputMock.outputJson.mock.calls[0][0] as { + success: boolean; + connectionState: string; + connection: unknown; + token?: unknown; + }; + expect(payload.success).toBe(true); + expect(payload.connectionState).toBe('already-connected'); + expect(payload.connection).toMatchObject({ apifyUsername: 'someone', plan: 'free' }); + expect(payload.token).toBeUndefined(); + }); + }); +}); diff --git a/src/commands/webscraper/apify/connect.ts b/src/commands/webscraper/apify/connect.ts index 18de2e3a..be933535 100644 --- a/src/commands/webscraper/apify/connect.ts +++ b/src/commands/webscraper/apify/connect.ts @@ -14,8 +14,10 @@ import { fetchApifyConnection, pollApifyConnection, startApifyCliFlow, + storeApifyToken, type ApifyConnectionResponse, -} from '../../../lib/api/apify.js'; + type ApifyTokenStatus, +} from '../../../lib/api/webscraper.js'; import { outputJson, outputSuccess } from '../../../lib/output.js'; import { trackGroupCommand, shutdownAnalytics } from '../../../lib/analytics.js'; import { runApifyAuthBridge } from '../../../lib/apify-bridge.js'; @@ -33,6 +35,8 @@ interface ConnectResult { plan?: string | null; status?: string; }; + /** Present only on the self-hosted `--token` path. */ + token?: ApifyTokenStatus; } export function registerApifyConnectCommand(program: Command): void { @@ -40,6 +44,7 @@ export function registerApifyConnectCommand(program: Command): void { .command('connect') .description('Connect your Apify account to your InsForge project') .option('--skip-browser', 'Do not auto-open the browser for OAuth; only print the URL') + .option('--token ', 'Apify API token (self-hosted; skips the OAuth flow)') .action(async (opts, cmd) => { const { json, apiUrl } = getRootOpts(cmd); try { @@ -47,6 +52,7 @@ export function registerApifyConnectCommand(program: Command): void { json, apiUrl, skipBrowser: Boolean(opts.skipBrowser), + token: opts.token, }); if (json) { outputJson({ success: true, ...result }); @@ -69,6 +75,8 @@ interface RunConnectOpts { json: boolean; apiUrl?: string; skipBrowser: boolean; + /** Self-hosted path: an Apify API token to store directly, bypassing OAuth. */ + token?: string; } // Ensures the InsForge project has an Apify connection (cli-start / OAuth). @@ -82,6 +90,33 @@ async function runConnect(opts: RunConnectOpts): Promise { throw new ProjectNotLinkedError(); } + // Self-hosted token path: the token is validated and stored server-side via + // the existing admin-authenticated ossFetch, so it needs neither the login + // token below nor any of the OAuth / auth-bridge flow further down. Short- + // circuits before any OAuth work; everything below is unreachable and thus + // unchanged when --token is absent. + // + // Branch on the option being *supplied*, not on its truthiness: `--token ""` + // must not silently fall through to the OAuth flow below (which would print + // a confusing "not logged in" error in CI when an env var expands to empty). + // Reject it locally instead — the backend's zod message for this case + // ("String must contain at least 1 character(s)") is far less clear. + if (opts.token !== undefined) { + if (opts.token.trim().length === 0) { + throw new CLIError('--token requires a non-empty Apify API token.'); + } + trackGroupCommand('apify', 'connect', proj); + const status = await storeApifyToken(opts.token); + if (!opts.json) { + outputSuccess(`Apify connected with token ${status.maskedKey ?? '(hidden)'}`); + } + return { + connectionState: 'newly-connected', + connection: { apifyUsername: null, plan: null, status: 'active' }, + token: status, + }; + } + // 2. Login token const token = getAccessToken(); if (!token) { diff --git a/src/lib/api/oss.test.ts b/src/lib/api/oss.test.ts index 6893316a..9563136b 100644 --- a/src/lib/api/oss.test.ts +++ b/src/lib/api/oss.test.ts @@ -76,4 +76,29 @@ describe('ossFetch', () => { /Upgrade your InsForge project to a version with Model Gateway support/, ); }); + + it('shows an upgrade+token message for a route-level 404 under /api/webscraper (not a cloud-only warning)', async () => { + // A 404 here means the backend predates the web scraper feature, not that + // self-hosting is unsupported — self-hosted backends do serve this route + // once they're upgraded. Guards against regressing to the old wording. + vi.spyOn(config, 'getProjectConfig').mockReturnValue({ + project_id: 'p1', + project_name: 'demo', + org_id: 'o1', + appkey: 'app', + region: 'us-east', + api_key: 'ik_test', + oss_host: 'https://app.us-east.insforge.app', + } satisfies ProjectConfig); + vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'NOT_FOUND' }), { + status: 404, + headers: { 'Content-Type': 'application/json' }, + }), + ); + + await expect(ossFetch('/api/webscraper/apify/config')).rejects.toThrow( + /Upgrade your InsForge instance.*insforge webscraper apify connect --token/s, + ); + }); }); diff --git a/src/lib/api/oss.ts b/src/lib/api/oss.ts index 09231427..f786e2a0 100644 --- a/src/lib/api/oss.ts +++ b/src/lib/api/oss.ts @@ -210,7 +210,7 @@ export async function ossFetch( } if (res.status === 404 && isRouteLevel404 && path.startsWith('/api/webscraper')) { - message = 'The web scraper is not available on this backend.\nThe Apify web scraper is cloud-only. Self-hosted: this feature is not supported. Cloud: contact your InsForge admin to enable it.'; + message = 'The web scraper is not available on this backend.\nUpgrade your InsForge instance to a version with web scraper support, then run `insforge webscraper apify connect --token ` to connect your Apify account.'; } // Safe to treat any 404 on /api/advisor/* as a route-level miss: the OSS diff --git a/src/lib/api/apify.test.ts b/src/lib/api/webscraper.test.ts similarity index 55% rename from src/lib/api/apify.test.ts rename to src/lib/api/webscraper.test.ts index 5c7d0421..7f92a290 100644 --- a/src/lib/api/apify.test.ts +++ b/src/lib/api/webscraper.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; +import * as oss from './oss.js'; import { CLIError } from '../errors.js'; -import { fetchApifyConnection, pollApifyConnection } from './apify.js'; +import { fetchApifyConnection, pollApifyConnection } from './webscraper.js'; const API = 'https://platform.test'; @@ -153,3 +154,121 @@ describe('pollApifyConnection', () => { ).rejects.toThrow(/cancelled/i); }); }); + +describe('storeApifyToken', () => { + it('PUTs the token to the OSS host and returns the masked status', async () => { + const spy = vi.spyOn(oss, 'ossFetch').mockResolvedValue( + new Response( + JSON.stringify({ token: { configured: true, maskedKey: 'apify_ap••••••••mnop' } }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ), + ); + + const { storeApifyToken } = await import('./webscraper.js'); + + await expect(storeApifyToken('apify_api_tok1234567890')).resolves.toEqual({ + configured: true, + maskedKey: 'apify_ap••••••••mnop', + }); + expect(spy).toHaveBeenCalledWith('/api/webscraper/apify/config', { + method: 'PUT', + body: JSON.stringify({ apiToken: 'apify_api_tok1234567890' }), + }); + }); + + it('propagates the backend message when Apify rejects the token', async () => { + vi.spyOn(oss, 'ossFetch').mockRejectedValue( + new CLIError('Apify rejected this API token.', 1, 'INVALID_INPUT', 400), + ); + + const { storeApifyToken } = await import('./webscraper.js'); + + await expect(storeApifyToken('bogus')).rejects.toThrow(/Apify rejected this API token/); + }); + + it('propagates the cloud-managed message on a cloud project', async () => { + vi.spyOn(oss, 'ossFetch').mockRejectedValue( + new CLIError('The Apify connection is managed by InsForge Cloud.', 1, 'INVALID_INPUT', 400), + ); + + const { storeApifyToken } = await import('./webscraper.js'); + + await expect(storeApifyToken('apify_api_tok1234567890')).rejects.toThrow(/InsForge Cloud/); + }); + + it('fails loudly when the response carries no token status', async () => { + vi.spyOn(oss, 'ossFetch').mockResolvedValue( + new Response(JSON.stringify({}), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + + const { storeApifyToken } = await import('./webscraper.js'); + + await expect(storeApifyToken('apify_api_tok1234567890')).rejects.toThrow( + /no token status/i, + ); + }); + + it('throws rather than resolving when the backend reports configured: false', async () => { + vi.spyOn(oss, 'ossFetch').mockResolvedValue( + new Response(JSON.stringify({ token: { configured: false, maskedKey: null } }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + + const { storeApifyToken } = await import('./webscraper.js'); + + const err = await storeApifyToken('apify_api_tok1234567890').catch((e) => e as CLIError); + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).message).toMatch(/did not report the token as stored/i); + expect((err as CLIError).code).toBe('APIFY_TOKEN_NOT_STORED'); + }); + + it('raises APIFY_CONFIG_MALFORMED instead of a raw parser error on a non-JSON 2xx body', async () => { + vi.spyOn(oss, 'ossFetch').mockResolvedValue( + new Response('not json', { + status: 200, + headers: { 'Content-Type': 'text/plain' }, + }), + ); + + const { storeApifyToken } = await import('./webscraper.js'); + + const err = await storeApifyToken('apify_api_tok1234567890').catch((e) => e as CLIError); + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).code).toBe('APIFY_CONFIG_MALFORMED'); + }); + + it('raises APIFY_CONFIG_MALFORMED on an empty 2xx body', async () => { + vi.spyOn(oss, 'ossFetch').mockResolvedValue( + new Response('', { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + + const { storeApifyToken } = await import('./webscraper.js'); + + const err = await storeApifyToken('apify_api_tok1234567890').catch((e) => e as CLIError); + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).code).toBe('APIFY_CONFIG_MALFORMED'); + }); + + it('raises APIFY_CONFIG_MALFORMED when the 2xx body parses to null', async () => { + vi.spyOn(oss, 'ossFetch').mockResolvedValue( + new Response('null', { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + + const { storeApifyToken } = await import('./webscraper.js'); + + const err = await storeApifyToken('apify_api_tok1234567890').catch((e) => e as CLIError); + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).code).toBe('APIFY_CONFIG_MALFORMED'); + }); +}); diff --git a/src/lib/api/apify.ts b/src/lib/api/webscraper.ts similarity index 81% rename from src/lib/api/apify.ts rename to src/lib/api/webscraper.ts index 67b20bcd..1d34373d 100644 --- a/src/lib/api/apify.ts +++ b/src/lib/api/webscraper.ts @@ -1,5 +1,6 @@ import { getPlatformApiUrl } from '../config.js'; import { CLIError, formatFetchError } from '../errors.js'; +import { ossFetch } from './oss.js'; const REQUEST_TIMEOUT_MS = 30_000; @@ -136,6 +137,69 @@ export async function fetchApifyConnection( return { kind: 'connected', connection: conn }; } +export interface ApifyTokenStatus { + configured: boolean; + maskedKey: string | null; +} + +/** + * Store a developer-supplied Apify API token on a self-hosted InsForge backend. + * + * Calls PUT /api/webscraper/apify/config on the project's OSS host. The backend + * validates the token against Apify before saving, so a 400 here means the token + * is bad — or that this is a cloud project, where the connection is made by OAuth + * instead. Both arrive as a CLIError from ossFetch carrying the backend's message, + * so they are left to propagate unchanged. + * + * A 2xx response is not itself proof the token was stored: an empty/non-JSON + * body is treated the same as a missing `token` object (APIFY_CONFIG_MALFORMED + * — the endpoint's response shape can't be trusted), while a well-formed body + * with `configured: false` is a distinct condition — the write may have + * succeeded but the read-back didn't find the secret — surfaced under its own + * APIFY_TOKEN_NOT_STORED code so callers can tell "can't parse the response" + * apart from "parsed fine, but nothing was actually stored". + */ +export async function storeApifyToken(apiToken: string): Promise { + const res = await ossFetch('/api/webscraper/apify/config', { + method: 'PUT', + body: JSON.stringify({ apiToken }), + }); + + let data: unknown; + try { + data = await res.json(); + } catch { + data = null; + } + + if (typeof data !== 'object' || data === null) { + throw new CLIError( + 'Apify config endpoint returned no token status; try again.', + 1, + 'APIFY_CONFIG_MALFORMED', + ); + } + + const token = (data as { token?: ApifyTokenStatus }).token; + if (!token) { + throw new CLIError( + 'Apify config endpoint returned no token status; try again.', + 1, + 'APIFY_CONFIG_MALFORMED', + ); + } + + if (token.configured !== true) { + throw new CLIError( + 'Apify did not report the token as stored; try again.', + 1, + 'APIFY_TOKEN_NOT_STORED', + ); + } + + return token; +} + export interface PollOptions { /** Total deadline before giving up. */ timeoutMs: number;