diff --git a/src/build/functions/edge.ts b/src/build/functions/edge.ts index aacefd5427..1da35a98a5 100644 --- a/src/build/functions/edge.ts +++ b/src/build/functions/edge.ts @@ -1,5 +1,5 @@ import { cp, lstat, mkdir, readdir, readFile, readlink, rm, writeFile } from 'node:fs/promises' -import { dirname, join, relative } from 'node:path/posix' +import { basename, dirname, join, normalize, relative } from 'node:path/posix' import type { Manifest, ManifestFunction } from '@netlify/edge-functions' import { glob } from 'fast-glob' @@ -37,7 +37,10 @@ type EdgeOrNodeMiddlewareDefinition = { } ) -const writeEdgeManifest = async (ctx: PluginContext, manifest: Manifest) => { +const writeEdgeManifest = async ( + ctx: PluginContext, + manifest: Manifest & { import_map?: string }, +) => { await mkdir(ctx.edgeFunctionsDir, { recursive: true }) await writeFile(join(ctx.edgeFunctionsDir, 'manifest.json'), JSON.stringify(manifest, null, 2)) } @@ -202,7 +205,123 @@ const copyHandlerDependenciesForEdgeMiddleware = async ( } const NODE_MIDDLEWARE_NAME = 'node-middleware' -const copyHandlerDependenciesForNodeMiddleware = async (ctx: PluginContext) => { + +type NodeMiddlewareImportMap = { imports: Record } + +const packageNameFromNodeModulesPath = (posixPath: string): string | null => { + const marker = 'node_modules/' + const idx = posixPath.lastIndexOf(marker) + if (idx === -1) { + return null + } + const rest = posixPath.slice(idx + marker.length) + if (!rest) { + return null + } + if (rest.startsWith('@')) { + const [scope, name] = rest.split('/') + if (!scope || !name) { + return null + } + return `${scope}/${name}` + } + return rest.split('/')[0] ?? null +} + +const packageDirFromNodeModulesPath = (posixPath: string): string | null => { + const name = packageNameFromNodeModulesPath(posixPath) + if (!name) { + return null + } + const marker = 'node_modules/' + const idx = posixPath.lastIndexOf(marker) + return posixPath.slice(0, idx + marker.length) + name +} + +const matchEsmExportTarget = (target: unknown): string | null => { + if (typeof target === 'string') { + return target + } + if (Array.isArray(target)) { + for (const item of target) { + const matched = matchEsmExportTarget(item) + if (matched) { + return matched + } + } + return null + } + if (target && typeof target === 'object') { + const record = target as Record + for (const condition of ['import', 'default', 'module', 'node']) { + if (condition in record) { + const matched = matchEsmExportTarget(record[condition]) + if (matched) { + return matched + } + } + } + } + return null +} + +const getEsmEntryRelPath = (pkgJson: Record): string | null => { + if (pkgJson.exports) { + const exportsField = pkgJson.exports + const main = + typeof exportsField === 'object' && + exportsField !== null && + !Array.isArray(exportsField) && + '.' in (exportsField as Record) + ? (exportsField as Record)['.'] + : exportsField + const matched = matchEsmExportTarget(main) + if (matched) { + return matched.replace(/^\.\//, '') + } + } + if (typeof pkgJson.module === 'string') { + return pkgJson.module.replace(/^\.\//, '') + } + if (pkgJson.type === 'module' && typeof pkgJson.main === 'string') { + return pkgJson.main.replace(/^\.\//, '') + } + return null +} + +const isEsmOnlyPackage = (pkgJson: Record): boolean => { + if (pkgJson.type === 'module') { + const exportsField = pkgJson.exports + if (exportsField && typeof exportsField === 'object' && !Array.isArray(exportsField)) { + const main = + '.' in (exportsField as Record) + ? (exportsField as Record)['.'] + : exportsField + if (main && typeof main === 'object' && !Array.isArray(main) && 'require' in main) { + return false + } + } + return true + } + return typeof pkgJson.main === 'string' && pkgJson.main.endsWith('.mjs') +} + +const isHashedTurbopackSpecifier = (name: string): boolean => /-[a-f\d]{8,}$/i.test(name) + +const patchExternalImport = (source: string) => + source.replaceAll( + 'await import(id)', + '(globalThis.__netlifyEsmExternals?.[id] ?? await import(id))', + ) + +const isNextAliasTreePath = (posixPath: string): boolean => + posixPath === '.next/node_modules' || + posixPath.startsWith('.next/node_modules/') || + posixPath.includes('/.next/node_modules/') + +const copyHandlerDependenciesForNodeMiddleware = async ( + ctx: PluginContext, +): Promise => { const name = NODE_MIDDLEWARE_NAME const srcDir = join(ctx.standaloneDir, ctx.nextDistDir) @@ -266,6 +385,9 @@ const copyHandlerDependenciesForNodeMiddleware = async (ctx: PluginContext) => { parts.push(`const virtualModules = new Map();`, `const virtualSymlinks = new Map();`) + const writtenSources = new Map() + const hashedSpecifiers = new Map() + const handleFileOrDirectory = async (fileOrDir: string) => { const srcPath = join(srcDir, fileOrDir) @@ -277,22 +399,163 @@ const copyHandlerDependenciesForNodeMiddleware = async (ctx: PluginContext) => { } } else if (stats.isSymbolicLink()) { const symlinkTarget = await readlink(srcPath) + const registeredPath = join(commonPrefix, fileOrDir) parts.push( - `virtualSymlinks.set(${JSON.stringify(join(commonPrefix, fileOrDir))}, ${JSON.stringify(symlinkTarget)});`, + `virtualSymlinks.set(${JSON.stringify(registeredPath)}, ${JSON.stringify(symlinkTarget)});`, ) + // Turbopack emits hashed bare specifiers as `.next/node_modules/-` + // symlinks. Those names are what `import()` looks up at runtime. + if (basename(dirname(registeredPath)) === 'node_modules') { + hashedSpecifiers.set( + basename(registeredPath), + normalize(join(dirname(registeredPath), symlinkTarget)), + ) + } } else { const content = await readFile(srcPath, 'utf8') - - parts.push( - `virtualModules.set(${JSON.stringify(join(commonPrefix, fileOrDir))}, ${JSON.stringify(content)});`, - ) + const registeredPath = join(commonPrefix, fileOrDir) + writtenSources.set(registeredPath, content) } } for (const file of files) { await handleFileOrDirectory(file) } - parts.push(`registerCJSModules(import.meta.url, virtualModules, virtualSymlinks); + + // ESM externals are loaded with Deno `import()`, which never hits the virtual + // CJS registry. Materialize those packages as real files and map both the + // package name (webpack) and the hashed turbopack specifier onto the ESM entry. + // The internal edge-functions manifest `import_map` field is how Netlify's + // edge bundler picks this up: + // https://github.com/netlify/edge-bundler/blob/main/node/deploy_config.ts + const esmEntries = new Map() + const configuredExternals = new Set( + (ctx.buildConfig as { serverExternalPackages?: string[] }).serverExternalPackages ?? [], + ) + + const tryAddEsmEntry = (specifier: string, pkgDir: string) => { + const pkgJsonRaw = + writtenSources.get(join(pkgDir, 'package.json')) ?? + writtenSources.get(join('.next/node_modules', specifier, 'package.json')) + if (!pkgJsonRaw) { + return + } + let pkgJson: Record + try { + pkgJson = JSON.parse(pkgJsonRaw) as Record + } catch { + return + } + if (!isEsmOnlyPackage(pkgJson)) { + return + } + const entryRel = getEsmEntryRelPath(pkgJson) + if (!entryRel) { + return + } + esmEntries.set(specifier, join(pkgDir, entryRel)) + } + + for (const [path] of writtenSources) { + if (!path.endsWith('package.json') || !path.includes('node_modules/')) { + continue + } + if (isNextAliasTreePath(path)) { + continue + } + const pkgDir = packageDirFromNodeModulesPath(path) + const pkgName = packageNameFromNodeModulesPath(path) + if (!pkgDir || !pkgName || isHashedTurbopackSpecifier(pkgName)) { + continue + } + if (!configuredExternals.has(pkgName)) { + continue + } + tryAddEsmEntry(pkgName, pkgDir) + } + + // Test copies and some filesystems dereference the Turbopack alias symlink, + // so the hashed name shows up as a real directory of files instead of a link. + for (const [path, content] of writtenSources) { + const match = path.match(/(?:^|\/)\.next\/node_modules\/([^/]+)\/package\.json$/) + if (!match || !isHashedTurbopackSpecifier(match[1])) { + continue + } + let pkgJson: Record + try { + pkgJson = JSON.parse(content) as Record + } catch { + continue + } + if (typeof pkgJson.name === 'string') { + hashedSpecifiers.set(match[1], `node_modules/${pkgJson.name}`) + } + } + + for (const [specifier, pkgDir] of hashedSpecifiers) { + tryAddEsmEntry(specifier, pkgDir) + const pkgName = packageNameFromNodeModulesPath(join(pkgDir, 'package.json')) + if (pkgName) { + tryAddEsmEntry(pkgName, pkgDir) + } + } + + const packagesToMaterialize = new Set() + for (const entryPath of esmEntries.values()) { + const pkgDir = packageDirFromNodeModulesPath(entryPath) + if (pkgDir) { + packagesToMaterialize.add(pkgDir) + } + } + + const canonicalPathFor = (path: string): string => { + for (const [specifier, pkgDir] of hashedSpecifiers) { + const nextAliasRoot = `.next/node_modules/${specifier}` + if (path === nextAliasRoot || path.startsWith(`${nextAliasRoot}/`)) { + return pkgDir + path.slice(nextAliasRoot.length) + } + } + return path + } + + for (const [path, content] of writtenSources) { + const canonical = canonicalPathFor(path) + const pkgDir = packageDirFromNodeModulesPath(canonical) + if (!pkgDir || !packagesToMaterialize.has(pkgDir)) { + continue + } + const outPath = join(destDir, 'server', canonical) + await mkdir(dirname(outPath), { recursive: true }) + await writeFile(outPath, content) + } + + const entryPathToNs = new Map() + const staticImportLines: string[] = [] + for (const [index, entryPath] of [...new Set(esmEntries.values())].entries()) { + const ns = `__netlifyEsm${index}` + entryPathToNs.set(entryPath, ns) + staticImportLines.push(`import * as ${ns} from ${JSON.stringify(`./${entryPath}`)};`) + } + + // Deno's eszip build cannot evaluate `import(id)` when `id` is a runtime value + // ("A dynamic import callback was not specified"). Point Turbopack's + // externalImport at the statically imported namespace instead. + + for (const [path, content] of writtenSources) { + parts.push( + `virtualModules.set(${JSON.stringify(path)}, ${JSON.stringify(patchExternalImport(content))});`, + ) + } + + parts.push(`${staticImportLines.join('\n')} +globalThis.__netlifyEsmExternals = { +${[...esmEntries] + .map( + ([specifier, entryPath]) => ` ${JSON.stringify(specifier)}: ${entryPathToNs.get(entryPath)},`, + ) + .join('\n')} +}; +registerCJSModules(import.meta.url, virtualModules, virtualSymlinks); const require = createRequire(import.meta.url); const middlewareEntrypoint = "${join(commonPrefix, entry)}" @@ -307,16 +570,32 @@ const copyHandlerDependenciesForNodeMiddleware = async (ctx: PluginContext) => { await mkdir(dirname(outputFile), { recursive: true }) await writeFile(outputFile, parts.join('\n')) + + if (esmEntries.size === 0) { + return undefined + } + + const handlerDirName = getHandlerName({ name }) + const imports: Record = {} + for (const [specifier, entryPath] of esmEntries) { + imports[specifier] = `./${handlerDirName}/server/${entryPath}` + } + + return { imports } } const createEdgeHandler = async ( ctx: PluginContext, definition: EdgeOrNodeMiddlewareDefinition, -): Promise => { - await (definition.runtime === 'edge' - ? copyHandlerDependenciesForEdgeMiddleware(ctx, definition.functionDefinition) - : copyHandlerDependenciesForNodeMiddleware(ctx)) +): Promise => { + let importMap: NodeMiddlewareImportMap | undefined + if (definition.runtime === 'edge') { + await copyHandlerDependenciesForEdgeMiddleware(ctx, definition.functionDefinition) + } else { + importMap = await copyHandlerDependenciesForNodeMiddleware(ctx) + } await writeHandlerFile(ctx, definition) + return importMap } const getHandlerName = ({ name }: Pick): string => @@ -366,15 +645,28 @@ export const createEdgeHandlers = async (ctx: PluginContext) => { }) } - await Promise.all(middlewareDefinitions.map((def) => createEdgeHandler(ctx, def))) + const importMaps = await Promise.all( + middlewareDefinitions.map((def) => createEdgeHandler(ctx, def)), + ) const netlifyDefinitions = middlewareDefinitions.flatMap((def) => buildHandlerDefinition(ctx, def), ) - const netlifyManifest: Manifest = { + const imports = Object.assign({}, ...importMaps.map((map) => map?.imports ?? {})) as Record< + string, + string + > + const netlifyManifest: Manifest & { import_map?: string } = { version: 1, functions: netlifyDefinitions, } + if (Object.keys(imports).length !== 0) { + await writeFile( + join(ctx.edgeFunctionsDir, 'import_map.json'), + JSON.stringify({ imports }, null, 2), + ) + netlifyManifest.import_map = './import_map.json' + } await writeEdgeManifest(ctx, netlifyManifest) } diff --git a/tests/fixtures/middleware-node-esm-externals/app/layout.js b/tests/fixtures/middleware-node-esm-externals/app/layout.js new file mode 100644 index 0000000000..d44041a68b --- /dev/null +++ b/tests/fixtures/middleware-node-esm-externals/app/layout.js @@ -0,0 +1,11 @@ +export const metadata = { + title: 'Node middleware ESM externals', +} + +export default function RootLayout({ children }) { + return ( + + {children} + + ) +} diff --git a/tests/fixtures/middleware-node-esm-externals/app/page.js b/tests/fixtures/middleware-node-esm-externals/app/page.js new file mode 100644 index 0000000000..1a9fe06903 --- /dev/null +++ b/tests/fixtures/middleware-node-esm-externals/app/page.js @@ -0,0 +1,7 @@ +export default function Home() { + return ( +
+

Home

+
+ ) +} diff --git a/tests/fixtures/middleware-node-esm-externals/app/protected/page.js b/tests/fixtures/middleware-node-esm-externals/app/protected/page.js new file mode 100644 index 0000000000..740d9174b2 --- /dev/null +++ b/tests/fixtures/middleware-node-esm-externals/app/protected/page.js @@ -0,0 +1,7 @@ +export default function Protected() { + return ( +
+

Protected

+
+ ) +} diff --git a/tests/fixtures/middleware-node-esm-externals/middleware.ts b/tests/fixtures/middleware-node-esm-externals/middleware.ts new file mode 100644 index 0000000000..f20c3ac324 --- /dev/null +++ b/tests/fixtures/middleware-node-esm-externals/middleware.ts @@ -0,0 +1,15 @@ +import { nanoid } from 'nanoid' +import { NextResponse } from 'next/server' +import type { NextRequest } from 'next/server' + +export function middleware(request: NextRequest) { + const response = NextResponse.next() + response.headers.set('x-request-id', nanoid()) + response.headers.set('x-pathname', request.nextUrl.pathname) + return response +} + +export const config = { + matcher: '/protected/:path*', + runtime: 'nodejs', +} diff --git a/tests/fixtures/middleware-node-esm-externals/next.config.js b/tests/fixtures/middleware-node-esm-externals/next.config.js new file mode 100644 index 0000000000..f301759c57 --- /dev/null +++ b/tests/fixtures/middleware-node-esm-externals/next.config.js @@ -0,0 +1,14 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = { + output: 'standalone', + eslint: { + ignoreDuringBuilds: true, + }, + experimental: { + nodeMiddleware: true, + }, + serverExternalPackages: ['nanoid'], + outputFileTracingRoot: __dirname, +} + +module.exports = nextConfig diff --git a/tests/fixtures/middleware-node-esm-externals/package.json b/tests/fixtures/middleware-node-esm-externals/package.json new file mode 100644 index 0000000000..406b03af53 --- /dev/null +++ b/tests/fixtures/middleware-node-esm-externals/package.json @@ -0,0 +1,21 @@ +{ + "name": "middleware-node-esm-externals", + "version": "0.1.0", + "private": true, + "scripts": { + "postinstall": "next build", + "dev": "next dev", + "build": "next build" + }, + "dependencies": { + "nanoid": "^5.1.6", + "next": "latest", + "react": "18.2.0", + "react-dom": "18.2.0" + }, + "test": { + "dependencies": { + "next": ">=15.5.0" + } + } +} diff --git a/tests/integration/middleware.test.ts b/tests/integration/middleware.test.ts index c093ed248b..78f27226c3 100644 --- a/tests/integration/middleware.test.ts +++ b/tests/integration/middleware.test.ts @@ -1,3 +1,6 @@ +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' + import { v4 } from 'uuid' import { beforeEach, describe, expect, test, vi } from 'vitest' import { type FixtureTestContext } from '../utils/contexts.js' @@ -790,6 +793,39 @@ for (const { 'https://docs.netlify.com/build/frameworks/framework-setup-guides/nextjs/overview/#limitations', ) }) + + test('should run Node.js middleware that imports an ESM-only serverExternalPackages package', async (ctx) => { + await createFixture('middleware-node-esm-externals', ctx) + await runPlugin(ctx) + + const importMap = JSON.parse( + await readFile(join(ctx.cwd, '.netlify/edge-functions/import_map.json'), 'utf8'), + ) as { imports: Record } + const importMapTargets = Object.values(importMap.imports) + expect(importMapTargets.some((target) => target.includes('nanoid'))).toBe(true) + expect( + Object.keys(importMap.imports).some( + (specifier) => specifier === 'nanoid' || specifier.startsWith('nanoid-'), + ), + ).toBe(true) + + const origin = await LocalServer.run(async (_req, res) => { + res.write('Hello from origin!') + res.end() + }) + ctx.cleanup?.push(() => origin.stop()) + + const response = await invokeEdgeFunction(ctx, { + functions: [edgeFunctionNameRoot], + origin, + url: '/protected', + }) + + expect(response.status).toBe(200) + expect(await response.text()).toBe('Hello from origin!') + expect(response.headers.get('x-request-id')).toMatch(/^[A-Za-z0-9_-]+$/) + expect(response.headers.get('x-pathname')).toBe('/protected') + }) }) describe('Proxy specific', () => {