diff --git a/.changeset/deferred-no-bootstrap-discovery.md b/.changeset/deferred-no-bootstrap-discovery.md new file mode 100644 index 0000000000..603a8329bb --- /dev/null +++ b/.changeset/deferred-no-bootstrap-discovery.md @@ -0,0 +1,5 @@ +--- +"@workflow/next": patch +--- + +Skip upfront workflow discovery scans in deferred (`lazyDiscovery`) mode and rely on deferred/loader-driven discovery only. diff --git a/.changeset/fix-deferred-discovery-with-explicit-dirs.md b/.changeset/fix-deferred-discovery-with-explicit-dirs.md new file mode 100644 index 0000000000..8c01500acb --- /dev/null +++ b/.changeset/fix-deferred-discovery-with-explicit-dirs.md @@ -0,0 +1,6 @@ +--- +"@workflow/ai": patch +"@workflow/next": patch +--- + +Skip redundant workflow discovery on dev server restart when cache exists. Add workflow export condition and transpile configuration for @workflow/ai. diff --git a/packages/ai/package.json b/packages/ai/package.json index e09948ab20..e95f07a65e 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -32,6 +32,7 @@ }, "./agent": { "types": "./dist/agent/durable-agent.d.ts", + "workflow": "./dist/agent/durable-agent.js", "default": "./dist/agent/durable-agent.js" }, "./anthropic": { diff --git a/packages/next/src/builder-deferred.test.ts b/packages/next/src/builder-deferred.test.ts new file mode 100644 index 0000000000..8eb330a906 --- /dev/null +++ b/packages/next/src/builder-deferred.test.ts @@ -0,0 +1,186 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +type BuilderWithInit = { + initializeDiscoveryState: () => Promise; +}; + +type DiscoverEntriesOwner = { + discoverEntries: (...args: unknown[]) => Promise; +}; + +type BuilderWithTransitiveSteps = { + collectTransitiveStepFiles: (args: { + stepFiles: string[]; + seedFiles?: string[]; + }) => Promise; +}; + +describe('NextDeferredBuilder discovery behavior', () => { + let testDir: string; + let discoverEntriesSpy: ReturnType; + let evalSpy: ReturnType; + + const createBuilder = async (watch: boolean) => { + const { getNextBuilderDeferred } = await import('./builder-deferred.js'); + const NextDeferredBuilder = await getNextBuilderDeferred(); + + return new NextDeferredBuilder({ + dirs: ['app'], + workingDir: testDir, + buildTarget: 'next', + stepsBundlePath: '', + workflowsBundlePath: '', + webhookBundlePath: '', + distDir: '.next', + watch, + }); + }; + + const createAppEntrypoint = async () => { + const appDir = join(testDir, 'app'); + await mkdir(appDir, { recursive: true }); + const workflowFilePath = join(appDir, 'page.ts'); + await writeFile( + workflowFilePath, + '"use workflow";\nexport async function test() {}', + 'utf-8' + ); + return workflowFilePath; + }; + + beforeEach(async () => { + vi.resetModules(); + testDir = await mkdtemp(join(tmpdir(), 'workflow-deferred-builder-')); + + const { BaseBuilder } = await import('@workflow/builders'); + discoverEntriesSpy = vi.spyOn( + BaseBuilder.prototype as unknown as DiscoverEntriesOwner, + 'discoverEntries' + ); + + // Vitest executes modules in a VM context where eval('import("...")') + // requires a dynamic import callback. Forward that specific eval call to + // native dynamic import so getNextBuilderDeferred can run in tests. + evalSpy = vi.spyOn(globalThis, 'eval').mockImplementation((source) => { + if (source === 'import("@workflow/builders")') { + return import('@workflow/builders'); + } + throw new Error(`Unexpected eval source in test: ${source}`); + }); + }); + + afterEach(async () => { + discoverEntriesSpy.mockRestore(); + evalSpy.mockRestore(); + await rm(testDir, { recursive: true, force: true }); + }); + + it('should skip discovery in dev mode when cache exists', async () => { + const cachedWorkflowFilePath = await createAppEntrypoint(); + + // Create cache directory and cache file + const cacheDir = join(testDir, '.next', 'cache'); + await mkdir(cacheDir, { recursive: true }); + await writeFile( + join(cacheDir, 'workflows.json'), + JSON.stringify({ + workflowFiles: [cachedWorkflowFilePath], + stepFiles: [], + }), + 'utf-8' + ); + + const builder = await createBuilder(true); + await (builder as unknown as BuilderWithInit).initializeDiscoveryState(); + + // In dev mode with cache, discoverEntries should NOT be called + expect(discoverEntriesSpy).not.toHaveBeenCalled(); + }); + + it('should skip discovery in production builds', async () => { + await createAppEntrypoint(); + const builder = await createBuilder(false); + await (builder as unknown as BuilderWithInit).initializeDiscoveryState(); + + expect(discoverEntriesSpy).not.toHaveBeenCalled(); + }); + + it('should skip discovery on first dev build when no cache', async () => { + await createAppEntrypoint(); + const builder = await createBuilder(true); + await (builder as unknown as BuilderWithInit).initializeDiscoveryState(); + + expect(discoverEntriesSpy).not.toHaveBeenCalled(); + }); + + it('should collect transitive @workflow package step files from workflow imports', async () => { + const appDir = join(testDir, 'app'); + const workflowFilePath = join(appDir, 'page.ts'); + await mkdir(appDir, { recursive: true }); + await writeFile( + workflowFilePath, + [ + '"use workflow";', + "import { closeStream } from '@workflow/ai/agent';", + 'export async function workflowEntry() {', + ' return closeStream();', + '}', + ].join('\n'), + 'utf-8' + ); + + const workflowAiPackageDir = join(testDir, 'node_modules/@workflow/ai'); + await mkdir(workflowAiPackageDir, { recursive: true }); + await writeFile( + join(workflowAiPackageDir, 'package.json'), + JSON.stringify({ name: '@workflow/ai', version: '0.0.0' }), + 'utf-8' + ); + await writeFile( + join(workflowAiPackageDir, 'agent.js'), + [ + "import { nestedStep } from './nested.js';", + 'export async function closeStream() {', + " 'use step';", + ' return nestedStep();', + '}', + ].join('\n'), + 'utf-8' + ); + await writeFile( + join(workflowAiPackageDir, 'nested.js'), + [ + 'export async function nestedStep() {', + " 'use step';", + " return 'ok';", + '}', + ].join('\n'), + 'utf-8' + ); + + const builder = await createBuilder(true); + const transitiveStepFiles = await ( + builder as unknown as BuilderWithTransitiveSteps + ).collectTransitiveStepFiles({ + stepFiles: [], + seedFiles: [workflowFilePath], + }); + + const normalizedStepFiles = transitiveStepFiles.map((filePath) => + filePath.replace(/\\/g, '/') + ); + expect( + normalizedStepFiles.some((filePath) => + filePath.includes('/node_modules/@workflow/ai/agent') + ) + ).toBe(true); + expect( + normalizedStepFiles.some((filePath) => + filePath.includes('/node_modules/@workflow/ai/nested.js') + ) + ).toBe(true); + }); +}); diff --git a/packages/next/src/builder-deferred.ts b/packages/next/src/builder-deferred.ts index 86e1e58c86..af6f351fc9 100644 --- a/packages/next/src/builder-deferred.ts +++ b/packages/next/src/builder-deferred.ts @@ -10,6 +10,7 @@ import { writeFile, } from 'node:fs/promises'; import os from 'node:os'; +import { createRequire } from 'node:module'; import { basename, dirname, @@ -671,7 +672,7 @@ export async function getNextBuilderDeferred() { } await this.loadWorkflowsCache(); - await this.loadDiscoveredEntriesFromInputGraph(); + this.cacheInitialized = true; } @@ -873,10 +874,18 @@ export async function getNextBuilderDeferred() { /\\/g, '/' ); + + // Skip generated files and most node_modules, but allow @workflow scoped packages + // to be discovered (they may contain step functions that need to be registered) + const isWorkflowPackage = + normalizedSourcePathForCheck.includes('/node_modules/@workflow/') || + normalizedSourcePathForCheck.includes('/.pnpm/@workflow+'); if ( normalizedSourcePathForCheck.includes('/.well-known/workflow/') || - normalizedSourcePathForCheck.includes('/node_modules/') || - normalizedSourcePathForCheck.includes('/.pnpm/') || + (normalizedSourcePathForCheck.includes('/node_modules/') && + !isWorkflowPackage) || + (normalizedSourcePathForCheck.includes('/.pnpm/') && + !isWorkflowPackage) || normalizedSourcePathForCheck.includes('/.next/') || normalizedSourcePathForCheck.endsWith('/virtual-entry.js') ) { @@ -980,47 +989,6 @@ export async function getNextBuilderDeferred() { } } - private async loadDiscoveredEntriesFromInputGraph(): Promise { - const inputFiles = await this.getInputFiles(); - if (inputFiles.length === 0) { - return; - } - - const { discoveredWorkflows, discoveredSteps, discoveredSerdeFiles } = - await this.discoverEntries(inputFiles, this.config.workingDir); - const { workflowFiles, stepFiles, serdeFiles } = - await this.reconcileDiscoveredEntries({ - workflowCandidates: discoveredWorkflows, - stepCandidates: discoveredSteps, - serdeCandidates: discoveredSerdeFiles, - validatePatterns: true, - }); - - let hasChanges = false; - for (const filePath of workflowFiles) { - if (!this.discoveredWorkflowFiles.has(filePath)) { - this.discoveredWorkflowFiles.add(filePath); - hasChanges = true; - } - } - for (const filePath of stepFiles) { - if (!this.discoveredStepFiles.has(filePath)) { - this.discoveredStepFiles.add(filePath); - hasChanges = true; - } - } - for (const filePath of serdeFiles) { - if (!this.discoveredSerdeFiles.has(filePath)) { - this.discoveredSerdeFiles.add(filePath); - hasChanges = true; - } - } - - if (hasChanges) { - this.scheduleWorkflowsCacheWrite(); - } - } - private async writeWorkflowsCache(): Promise { const cacheFilePath = this.getWorkflowsCacheFilePath(); const cacheDir = join(this.config.workingDir, this.getDistDir(), 'cache'); @@ -1380,8 +1348,12 @@ export async function getNextBuilderDeferred() { return rewrittenSource; } - private extractRelativeImportSpecifiers(source: string): string[] { - const relativeSpecifiers = new Set(); + private isTransitivePackageSpecifier(specifier: string): boolean { + return specifier.startsWith('@workflow/'); + } + + private extractTransitiveImportSpecifiers(source: string): string[] { + const matchingSpecifiers = new Set(); const importPatterns = [ /from\s+['"]([^'"]+)['"]/g, /import\s+['"]([^'"]+)['"]/g, @@ -1392,22 +1364,37 @@ export async function getNextBuilderDeferred() { for (const importPattern of importPatterns) { for (const match of source.matchAll(importPattern)) { const specifier = match[1]; - if (specifier?.startsWith('.')) { - relativeSpecifiers.add(specifier); + if (!specifier) { + continue; + } + const specifierMatch = specifier.match(/^([^?#]+)(.*)$/); + const importPath = specifierMatch?.[1] ?? specifier; + + if ( + importPath.startsWith('.') || + this.isTransitivePackageSpecifier(importPath) + ) { + matchingSpecifiers.add(specifier); } } } - return Array.from(relativeSpecifiers); + return Array.from(matchingSpecifiers); } private shouldSkipTransitiveStepFile(filePath: string): boolean { const normalizedPath = filePath.replace(/\\/g, '/'); + + // Allow @workflow scoped packages to be followed transitively + const isWorkflowPackage = + normalizedPath.includes('/node_modules/@workflow/') || + normalizedPath.includes('/.pnpm/@workflow+'); + return ( normalizedPath.includes('/.well-known/workflow/') || normalizedPath.includes('/.next/') || - normalizedPath.includes('/node_modules/') || - normalizedPath.includes('/.pnpm/') + (normalizedPath.includes('/node_modules/') && !isWorkflowPackage) || + (normalizedPath.includes('/.pnpm/') && !isWorkflowPackage) ); } @@ -1417,6 +1404,40 @@ export async function getNextBuilderDeferred() { ): Promise { const specifierMatch = specifier.match(/^([^?#]+)(.*)$/); const importPath = specifierMatch?.[1] ?? specifier; + + if (!importPath.startsWith('.')) { + if (!this.isTransitivePackageSpecifier(importPath)) { + return null; + } + + let resolvedPackagePath: string; + try { + const sourceRequire = createRequire(sourceFilePath); + resolvedPackagePath = sourceRequire.resolve(importPath); + } catch { + try { + resolvedPackagePath = require.resolve(importPath, { + paths: [dirname(sourceFilePath), this.config.workingDir], + }); + } catch { + return null; + } + } + + const normalizedResolvedPath = + this.normalizeDiscoveredFilePath(resolvedPackagePath); + if (this.shouldSkipTransitiveStepFile(normalizedResolvedPath)) { + return null; + } + + try { + const fileStats = await stat(normalizedResolvedPath); + return fileStats.isFile() ? normalizedResolvedPath : null; + } catch { + return null; + } + } + const absoluteTargetPath = resolve(dirname(sourceFilePath), importPath); const candidatePaths = new Set([ @@ -1538,9 +1559,9 @@ export async function getNextBuilderDeferred() { discoveredStepFiles.add(currentFile); } - const relativeImportSpecifiers = - this.extractRelativeImportSpecifiers(currentSource); - for (const specifier of relativeImportSpecifiers) { + const transitiveImportSpecifiers = + this.extractTransitiveImportSpecifiers(currentSource); + for (const specifier of transitiveImportSpecifiers) { const resolvedImportPath = await this.resolveTransitiveStepImportTargetPath( currentFile, @@ -1648,9 +1669,9 @@ export async function getNextBuilderDeferred() { continue; } - const relativeImportSpecifiers = - this.extractRelativeImportSpecifiers(currentSource); - for (const specifier of relativeImportSpecifiers) { + const transitiveImportSpecifiers = + this.extractTransitiveImportSpecifiers(currentSource); + for (const specifier of transitiveImportSpecifiers) { const resolvedImportPath = await this.resolveTransitiveStepImportTargetPath( currentFile, diff --git a/packages/next/src/index.test.ts b/packages/next/src/index.test.ts index 08d22b78df..421d1764b2 100644 --- a/packages/next/src/index.test.ts +++ b/packages/next/src/index.test.ts @@ -109,4 +109,23 @@ describe('withWorkflow outputFileTracingRoot', () => { workingDir: process.cwd(), }); }); + + it('keeps turbopack content conditions enabled in deferred mode', async () => { + shouldUseDeferredBuilderMock.mockReturnValue(true); + + const config = withWorkflow({}); + const nextConfig = await config('phase-development-server', { + defaultConfig: {}, + }); + + const tsRule = (nextConfig.turbopack?.rules as Record)?.[ + '*.ts' + ] as { + condition?: { all?: unknown[] }; + }; + + expect(tsRule?.condition?.all).toBeDefined(); + expect(Array.isArray(tsRule?.condition?.all)).toBe(true); + expect((tsRule?.condition?.all?.length ?? 0) > 0).toBe(true); + }); }); diff --git a/packages/next/src/index.ts b/packages/next/src/index.ts index fce1beb34f..4892aad574 100644 --- a/packages/next/src/index.ts +++ b/packages/next/src/index.ts @@ -90,10 +90,10 @@ export function withWorkflow( const supportsTurboCondition = semver.gte(nextVersion, 'v16.0.0'); const useDeferredBuilder = shouldUseDeferredBuilder(nextVersion); - // Deferred builder discovers files via loader socket notifications, so - // turbopack content conditions are only needed with the eager builder. - const shouldApplyTurboCondition = - supportsTurboCondition && !useDeferredBuilder; + // Apply content conditions in both eager and deferred modes. + // Running the workflow loader on every TS/JS module in large apps can + // severely delay first-route compilation under Turbopack. + const shouldApplyTurboCondition = supportsTurboCondition; const shouldWatch = process.env.NODE_ENV === 'development'; let workflowBuilderPromise: Promise | undefined; @@ -174,6 +174,23 @@ export function withWorkflow( } }, }; + + // Ensure @workflow packages are transpiled so our loader can process them + // and discover step functions in dependencies like @workflow/ai. + // Note: @workflow/ai is optional - Next.js handles missing transpilePackages + // entries gracefully, so this is safe even if the package isn't installed. + const existingTranspilePackages = Array.isArray( + nextConfig.transpilePackages + ) + ? nextConfig.transpilePackages + : []; + const workflowPackagesToTranspile = ['@workflow/ai']; + nextConfig.transpilePackages = [ + ...new Set([ + ...existingTranspilePackages, + ...workflowPackagesToTranspile, + ]), + ]; } for (const key of [