diff --git a/native-lib/node/README.md b/native-lib/node/README.md index 50cedea2..3f135c41 100644 --- a/native-lib/node/README.md +++ b/native-lib/node/README.md @@ -224,6 +224,36 @@ try { - `runStreaming(script, inputs?)`: Same as module-level `runStreaming()` - `runTransform(script, input, opts?)`: Same as module-level `runTransform()` +### External Modules + +DataWeave scripts can import external modules using the `resolveModule` option. The module-level convenience functions (`run()`, `runStreaming()`, `runTransform()`) operate on a lazily-initialized singleton that cannot be configured with a resolver — you must construct your own `DataWeave` instance: + +```typescript +import { DataWeave, composeResolvers, modulesFromDirectory, modulesFromJars } from '@dataweave/native'; + +const dw = new DataWeave({ + resolveModule: composeResolvers( + modulesFromDirectory('./my-modules'), + await modulesFromJars(['./libs/dw-utils.jar']) + ) +}); +dw.initialize(); + +const result = dw.run(` + %dw 2.0 + import org::company::utils + output application/json + --- + utils::doSomething() +`); + +if (result.success) { + console.log(result.getString()); +} +``` + +See [docs/external-modules.md](docs/external-modules.md) for complete documentation, resolver factories, error handling, and dependency management. Note: a resolver runs with full process permissions (no sandboxing) — see the "Security / Trust Model" section there before pointing one at untrusted sources. + ### Input Formats Inputs can be provided in multiple formats: @@ -409,12 +439,22 @@ try { The Node.js binding uses **N-API** (Node-API) for C addon integration: - **Thread-safe**: N-API calls are serialized on the Node.js event loop -- **Worker threads**: Safe to use from Worker threads (each thread needs its own `DataWeave` instance) - **Async operations**: Streaming operations yield control to the event loop between chunks - **No blocking**: Long-running scripts execute on the native side without blocking the event loop **Important:** Do not share a single `DataWeave` instance across Worker threads. Use the module-level functions (which use a global singleton) or create separate instances per thread. +**Custom module resolvers and Worker threads:** the native layer installs at +most one resolver callback for the whole process lifetime, and it is bound to +the Worker (main thread or a `worker_threads` Worker) that registered it +first — see [External Modules: Multiple Resolvers](docs/external-modules.md#multiple-resolvers-in-one-process). +Custom-module resolution attempted from any *other* thread is not routed to +that thread's own `resolveModule` callback; it silently falls back to +built-in modules only (custom module paths resolve as "not found" rather than +crashing or hanging). If you need per-Worker custom modules, resolve them on +the thread that first constructs a resolver-backed `DataWeave` instance, or +avoid resolver-backed instances in worker pools altogether. + ## Platform Support Supported platforms: diff --git a/native-lib/node/docs/external-modules.md b/native-lib/node/docs/external-modules.md new file mode 100644 index 00000000..6b67ae91 --- /dev/null +++ b/native-lib/node/docs/external-modules.md @@ -0,0 +1,327 @@ +# External Module Support + +DataWeave scripts can import external modules using the `resolveModule` option. This allows you to organize code into reusable modules and import them into your scripts. + +## Quick Start + +```typescript +import { DataWeave, modulesFromMap } from '@dataweave/native'; + +const dw = new DataWeave({ + resolveModule: modulesFromMap({ + 'org/company/lib.dwl': '%dw 2.0\nfun greet(n) = "Hello " ++ n', + }), +}); +dw.initialize(); + +const result = dw.run(` + %dw 2.0 + import org::company::lib + output application/json + --- + lib::greet("World") +`); +console.log(result.getString()); // "Hello World" +``` + +**Important:** The module-level convenience functions (`run()`, `runStreaming()`, `runTransform()` exported directly from `@dataweave/native`) operate on a lazily-initialized singleton that takes no constructor options and therefore cannot be configured with `resolveModule` — you **must** construct your own `DataWeave` instance to use external modules, as shown above. + +Additionally, external module resolution is currently supported only through `.run()` (the synchronous API). `.runStreaming()` and `.runTransform()` do not yet support external modules and will only have access to built-in modules. + +## Resolver Factories + +### modulesFromMap + +In-memory map of module paths to source code: + +```typescript +import { DataWeave, modulesFromMap } from '@dataweave/native'; + +const dw = new DataWeave({ + resolveModule: modulesFromMap({ + 'org/test/lib.dwl': '%dw 2.0\nfun foo() = 42', + }), +}); +dw.initialize(); +const result = dw.run('import org::test::lib\n%dw 2.0\n---\nlib::foo()'); +``` + +Best for: Small, in-memory module sets; testing and development. + +### modulesFromDirectory + +Read modules from a directory tree on disk: + +```typescript +import { DataWeave, modulesFromDirectory } from '@dataweave/native'; + +const dw = new DataWeave({ + resolveModule: modulesFromDirectory('./my-modules'), +}); +dw.initialize(); + +// Resolves "org/test/lib.dwl" → reads "./my-modules/org/test/lib.dwl" +const result = dw.run('import org::test::lib\n%dw 2.0\n---\nlib::foo()'); +``` + +Best for: Development and file-based module repositories. + +### modulesFromJars + +Extract modules from JAR files (asynchronous): + +```typescript +import { DataWeave, modulesFromJars } from '@dataweave/native'; + +// modulesFromJars is async and returns a Promise +const resolver = await modulesFromJars([ + './libs/dw-strings-1.0.jar', + './libs/dw-dates-2.1.jar', +]); + +const dw = new DataWeave({ + resolveModule: resolver, +}); +dw.initialize(); +const result = dw.run('import org::mule::weave::core::Strings\n%dw 2.0\n---\nStrings::capitalize("hello")'); +``` + +**Note:** `modulesFromJars()` returns a `Promise` because JAR extraction must complete first. The returned resolver itself is synchronous and can be used repeatedly. + +Best for: Packaged dependencies and distributed libraries. + +### composeResolvers + +Combine multiple resolvers with fallback chain (tries each in order, returns first match): + +```typescript +import { DataWeave, composeResolvers, modulesFromMap, modulesFromDirectory, modulesFromJars } from '@dataweave/native'; + +const resolver = composeResolvers( + modulesFromMap({ 'override.dwl': '...' }), // Try first + modulesFromDirectory('./shared'), // Then directory + await modulesFromJars(['./vendor/lib.jar']) // Finally JAR +); + +const dw = new DataWeave({ + resolveModule: resolver, +}); +dw.initialize(); +``` + +Best for: Layered resolution with fallbacks (overrides, shared libraries, vendor code). + +## How It Works + +- **One resolver per process**: The native engine maintains a single resolver per process lifetime. Only the first resolver registered is used; subsequent `DataWeave` instances with different resolvers will silently reuse the first one. +- **Resolution at compile time**: The resolver is invoked during script compilation, not per execution. +- **Synchronous resolution**: The resolver callback must be synchronous (no `async`/`await`, no Promise return). +- **Built-in modules**: Built-in modules (CompositeResolver) are always available and work alongside custom resolvers. + +## Error Handling + +### Module Not Found + +When a module cannot be resolved: + +```typescript +const dw = new DataWeave({ + resolveModule: modulesFromMap({ + // Only 'org/test/lib.dwl' is available + }), +}); +dw.initialize(); + +const result = dw.run(` + %dw 2.0 + import org::missing::module // Not found + --- + missing::something() +`); + +if (!result.success) { + console.error(result.error); // "Unable to resolve module with identifier ..." +} +``` + +The resolver returns `null`, and the engine reports a compile-time error. + +### File I/O Errors + +When the resolver encounters file system errors (unreadable files, permission denied, etc.), the resolver throws an error. This error is caught internally by the native layer and the callback returns `null` — indistinguishable from "module not found" to the DataWeave compiler: + +```typescript +const dw = new DataWeave({ + resolveModule: modulesFromDirectory('./my-modules'), +}); +dw.initialize(); + +const result = dw.run(` + %dw 2.0 + import org::test::lib + --- + lib::foo() +`); + +if (!result.success) { + // result.error is the same generic message as "module not found": + console.error(result.error); // "Unable to resolve module with identifier ..." + // The actual error details (permissions, encoding, etc.) are not available + // in the result object; see "Debugging" below for how to surface them. +} +``` + +**Debugging:** By default, a resolver failure logs only a fixed, content-free diagnostic line to stderr — the actual exception message and stack are suppressed, since they can carry resolver-controlled data (module source, credentials, filesystem paths). To see the detailed message and stack for diagnosing a failing resolver (e.g., directory does not exist, file unreadable due to permissions), set `DATAWEAVE_RESOLVER_DEBUG=1` in the process environment before running. Only enable this in a trusted debugging context, since the detailed output may expose sensitive resolver-controlled data. + +### Multiple Resolvers in One Process + +If you construct multiple `DataWeave` instances with different resolvers in the same process: + +```typescript +const dw1 = new DataWeave({ + resolveModule: modulesFromMap({ 'a.dwl': '...' }), +}); +dw1.initialize(); + +const dw2 = new DataWeave({ + resolveModule: modulesFromMap({ 'b.dwl': '...' }), +}); +dw2.initialize(); // Only loads/ref-counts the native library — does NOT register a resolver + +dw1.run('...'); // First resolver-backed run() in the process: installs dw1's resolver +dw2.run('...'); // Logs warning, silently reuses dw1's resolver instead of dw2's + +// Both dw1 and dw2 use dw1's resolver (only 'a.dwl' is available) +``` + +**The rule is "first resolver-backed `run()` wins," not "first `initialize()` wins."** +`initialize()` only loads and ref-counts the native library; the resolver +itself is registered lazily, on whichever instance's `run()` executes first +with a resolver configured. If `dw2.run()` happens to execute before +`dw1.run()` — even though `dw1.initialize()` ran first — `dw2`'s resolver +wins instead. + +**Workaround:** Use `composeResolvers()` to combine all modules into a single resolver: + +```typescript +const resolver = composeResolvers( + modulesFromMap({ 'a.dwl': '...' }), + modulesFromMap({ 'b.dwl': '...' }) +); + +const dw1 = new DataWeave({ resolveModule: resolver }); +dw1.initialize(); + +const dw2 = new DataWeave({ resolveModule: resolver }); +dw2.initialize(); // Both use the same resolver +``` + +**Worker threads:** the same one-resolver-per-process rule applies across +`worker_threads` Workers, not just across instances on one thread. The +resolver callback is additionally bound to the specific thread that first +registered it. A resolver-backed `DataWeave` constructed and initialized on a +Worker other than the one that registered the process's resolver will not +have its `resolveModule` invoked at all — custom module paths resolve as "not +found" (falling back to built-ins only) rather than crashing. There is +currently no supported way to run distinct custom-module resolvers on +different Workers in the same process; either resolve modules on the thread +that owns the process's resolver, or avoid resolver-backed instances in +worker pools. + +**Concurrent resolver-backed runs across Workers are unsupported and +memory-unsafe.** Beyond the "not found" fallback described above, calling a +resolver-backed `run()` concurrently from more than one Worker is not just +unsupported behavior — it is a memory-safety hazard. The native layer tracks +in-flight resolver results in unsynchronized, process-global state, and one +Worker's cleanup can free memory another Worker's concurrent call is still +using. Restrict resolver-backed execution to a single thread (or fully +serialize resolver-backed calls across Workers) until a future release +isolates per-instance engine state. + +## Security / Trust Model + +A `resolveModule` callback executes with **full process permissions** — the same trust model as the `dw` CLI resolving `.dwl` files from disk. There is no sandboxing: the callback can read/write the filesystem, make network calls, or run arbitrary Node.js code, and its return value (module source) is compiled and executed by the DataWeave engine with no additional isolation. Only configure a resolver that points at trusted sources (your own modules, vetted directories, or JARs from a trusted registry) — treat it with the same care you would give any code that runs with the permissions of your process. + +**`modulesFromDirectory` and symlinks:** each lookup canonicalizes the candidate path and rejects it if the canonical path falls outside the configured base directory, which blocks a *stable* symlink pointing out of the module tree. This cannot portably close a time-of-check/time-of-use race, though: an actor able to write into the module tree could swap a validated file (or an ancestor directory) for an out-of-base symlink between the validation check and the subsequent read — Node has no portable equivalent of Linux's `openat2(RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS)` to close this atomically. As with the trust model above, only point `modulesFromDirectory` at a directory tree that is not writable by principals less trusted than the process itself. + +## JAR Dependency Management + +This release does NOT include Maven/coursier dependency resolution. You must provide JAR paths manually. + +### Option 1: Download Using curl + +```bash +curl -o dw-lib.jar https://repository.mulesoft.org/.../dw-lib-1.0.jar +``` + +### Option 2: Use Maven CLI + +```bash +mvn dependency:copy \ + -Dartifact=org.mule.weave:dw-lib:1.0 \ + -DoutputDirectory=./libs +``` + +### Option 3: Use npm scripts (if configured) + +Future releases may add `npm run dw-deps` for automatic resolution. Check your project's `package.json`. + +Then pass JAR paths to `modulesFromJars()`: + +```typescript +const resolver = await modulesFromJars([ + './libs/dw-lib-1.0.jar', + './libs/dw-utils-2.1.jar' +]); + +const dw = new DataWeave({ resolveModule: resolver }); +dw.initialize(); +``` + +## Complete Example + +```typescript +import { DataWeave, composeResolvers, modulesFromMap, modulesFromDirectory, modulesFromJars } from '@dataweave/native'; + +async function main() { + // Combine in-memory, file-based, and JAR-based modules + const resolver = composeResolvers( + // Override specific modules in-memory + modulesFromMap({ + 'org/company/constants.dwl': '%dw 2.0\nfun version() = "1.0.0"' + }), + // Share modules from local directory + modulesFromDirectory('./shared-modules'), + // Load dependencies from JARs + await modulesFromJars(['./vendor/dw-strings.jar']) + ); + + const dw = new DataWeave({ resolveModule: resolver }); + dw.initialize(); + + try { + const result = dw.run(` + %dw 2.0 + import org::company::constants + import org::mule::weave::core::Strings + output application/json + --- + { + version: constants::version(), + greeting: Strings::capitalize("hello world") + } + `); + + if (result.success) { + console.log(result.getString()); + // {"version":"1.0.0","greeting":"Hello World"} + } else { + console.error('Error:', result.error); + } + } finally { + dw.cleanup(); + } +} + +main(); +``` diff --git a/native-lib/node/package-lock.json b/native-lib/node/package-lock.json index a25496b1..b6bef9fe 100644 --- a/native-lib/node/package-lock.json +++ b/native-lib/node/package-lock.json @@ -12,6 +12,9 @@ "linux", "win32" ], + "dependencies": { + "adm-zip": "^0.6.0" + }, "devDependencies": { "@types/node": "^20", "@vitest/coverage-v8": "^3.0", @@ -1218,6 +1221,15 @@ "node": "^18.17.0 || >=20.5.0" } }, + "node_modules/adm-zip": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.0.tgz", + "integrity": "sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg==", + "license": "MIT", + "engines": { + "node": ">=14.0" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", diff --git a/native-lib/node/package.json b/native-lib/node/package.json index b1a3e863..b2ff6012 100644 --- a/native-lib/node/package.json +++ b/native-lib/node/package.json @@ -22,7 +22,8 @@ "native/", "build/Release/dwlib_addon.node", "src/addon.c", - "binding.gyp" + "binding.gyp", + "docs/" ], "os": [ "darwin", @@ -33,6 +34,9 @@ "node": ">=18" }, "gypfile": true, + "dependencies": { + "adm-zip": "^0.6.0" + }, "devDependencies": { "@types/node": "^20", "@vitest/coverage-v8": "^3.0", diff --git a/native-lib/node/src/addon.c b/native-lib/node/src/addon.c index ed61f455..5aa31535 100644 --- a/native-lib/node/src/addon.c +++ b/native-lib/node/src/addon.c @@ -13,9 +13,22 @@ typedef void* (*run_script_fn)(void*, const char*, const char*); typedef void (*free_cstring_fn)(void*, void*); typedef int (*write_callback_t)(void* ctx, const char* buf, int len); typedef int (*read_callback_t)(void* ctx, char* buf, int buf_size); +typedef char* (*resolve_module_callback_t)(void* thread, const char* module_path); typedef void* (*run_script_callback_fn)(void*, const char*, const char*, write_callback_t, void*); typedef void* (*run_script_input_output_callback_fn)(void*, const char*, const char*, const char*, const char*, const char*, read_callback_t, write_callback_t, void*); +// Resolver-aware entrypoint types +// NOTE: run_script_with_resolver has no mimeType parameter on the native side +// (NativeLib.runScriptWithResolver(thread, script, inputsJson, resolverCallback) +// delegates to ScriptRuntime.run(script, inputsJson), which infers/hardcodes +// output mime type internally). The JS-facing mimeType argument is accepted +// for API symmetry with other entrypoints but is NOT forwarded across the FFI +// boundary — passing it here would misalign the native call's argument +// registers and corrupt the callback function pointer. +typedef char* (*run_script_with_resolver_fn)(void*, const char*, const char*, resolve_module_callback_t); +typedef void* (*run_script_callback_with_resolver_fn)(void*, const char*, const char*, const char*, write_callback_t, void*, resolve_module_callback_t); +typedef void* (*run_script_input_output_callback_with_resolver_fn)(void*, const char*, const char*, const char*, const char*, const char*, read_callback_t, write_callback_t, void*, resolve_module_callback_t); + // Global state static uv_lib_t g_lib; static int g_lib_loaded = 0; @@ -24,6 +37,13 @@ static void* g_thread = NULL; static int g_initialized = 0; static int g_ref_count = 0; static uv_mutex_t g_mutex; +// Guards initialization of the process-global g_mutex. Init() runs once per +// Worker environment that loads this addon, but g_mutex is process-global — +// re-running uv_mutex_init() on an already-initialized mutex from a second +// Worker's Init() call is undefined behavior (and can corrupt the mutex for +// every other thread already relying on it). uv_once ensures the real init +// body runs exactly once per process regardless of how many Workers load us. +static uv_once_t g_mutex_once = UV_ONCE_INIT; static graal_create_isolate_fn fn_create_isolate = NULL; static graal_attach_thread_fn fn_attach_thread = NULL; @@ -34,6 +54,69 @@ static free_cstring_fn fn_free_cstring = NULL; static run_script_callback_fn fn_run_script_callback = NULL; static run_script_input_output_callback_fn fn_run_script_input_output_callback = NULL; +// Resolver-aware entrypoints +static run_script_with_resolver_fn fn_run_script_with_resolver = NULL; +static run_script_callback_with_resolver_fn fn_run_script_callback_with_resolver = NULL; +static run_script_input_output_callback_with_resolver_fn fn_run_script_input_output_callback_with_resolver = NULL; + +// Resolver bridge state (one resolver per process). +// +// Unlike the streaming/transform entrypoints, runWithResolver's native call +// executes synchronously on the very thread that invoked it from JS — no +// background uv_thread is spawned. So when native code calls back into +// resolve_module_callback(), we are already on the correct (JS) thread and +// can call directly into V8/napi. Do NOT use napi_threadsafe_function here: +// that pattern queues work for "the" JS thread to pick up and blocks the +// caller on a condition variable until it's serviced — but if the caller +// *is* the JS thread, it can never service its own queued item, causing a +// deadlock (a real bug fixed in this codebase — see Task 11 report). +static napi_env g_resolver_env = NULL; +static napi_ref g_resolver_ref = NULL; + +// The OS thread that first installed the resolver (see napi_run_with_resolver +// below). ScriptRuntime's engine is a process-wide singleton, so once a +// resolver is installed, resolve_module_callback() can be reached from ANY +// entrypoint that later compiles a script against that shared engine — +// including runScriptStreaming/runScriptTransform, whose native calls run on +// a background uv_thread (see streaming_thread_fn/transform_thread_fn), not +// the JS thread. napi_env/napi_ref are thread-affine; calling into them from +// a thread other than the one that created them is undefined behavior. We +// record the owning thread here so resolve_module_callback can detect the +// mismatch and fail closed (return "not found") instead of crashing. +static uv_thread_t g_resolver_thread; + +// A single runWithResolver call may trigger resolve_module_callback multiple +// times (one script can import several modules). Native copies each +// returned buffer immediately, but the copy is made *after* our callback +// returns — we don't get a per-call "done freeing" signal, only "the whole +// run finished". So track every buffer allocated during one call and free +// them all once fn_run_script_with_resolver returns. +typedef struct resolver_result_node { + char* buf; + struct resolver_result_node* next; +} resolver_result_node_t; +static resolver_result_node_t* g_resolver_results = NULL; + +static void resolver_results_track(char* buf) { + if (buf == NULL) return; + resolver_result_node_t* node = (resolver_result_node_t*)malloc(sizeof(resolver_result_node_t)); + if (node == NULL) return; // Leak the buffer rather than crash; best-effort tracking. + node->buf = buf; + node->next = g_resolver_results; + g_resolver_results = node; +} + +static void resolver_results_free_all(void) { + resolver_result_node_t* node = g_resolver_results; + while (node != NULL) { + resolver_result_node_t* next = node->next; + free(node->buf); + free(node); + node = next; + } + g_resolver_results = NULL; +} + // --- Initialization --- struct init_args { @@ -62,6 +145,16 @@ static void init_thread_fn(void* arg) { uv_dlsym(&g_lib, "run_script_callback", (void**)&fn_run_script_callback); uv_dlsym(&g_lib, "run_script_input_output_callback", (void**)&fn_run_script_input_output_callback); + // Load resolver-aware entrypoints (optional - newer symbols) + uv_dlsym(&g_lib, "run_script_with_resolver", (void**)&fn_run_script_with_resolver); + // fn_run_script_callback_with_resolver / fn_run_script_input_output_callback_with_resolver + // are resolved here but intentionally never called from this file. Wiring them into + // runScriptStreaming/runScriptTransform would put the resolver callback on a background + // uv_thread, which is unsafe for the same reason resolve_module_callback() above guards + // against cross-thread napi calls — do not wire these up without solving that hazard first. + uv_dlsym(&g_lib, "run_script_callback_with_resolver", (void**)&fn_run_script_callback_with_resolver); + uv_dlsym(&g_lib, "run_script_input_output_callback_with_resolver", (void**)&fn_run_script_input_output_callback_with_resolver); + if (!fn_create_isolate || !fn_run_script || !fn_free_cstring) { snprintf(args->error, sizeof(args->error), "Missing required symbols in library"); args->result = -2; @@ -628,6 +721,260 @@ static napi_value napi_run_script_transform(napi_env env, napi_callback_info inf return promise; } +// --- Resolver callback bridge --- + +// Called by native code, synchronously, on the same JS thread that invoked +// runWithResolver (see the comment on g_resolver_env above for why this must +// NOT hop through napi_threadsafe_function). Calls the JS resolver directly +// and returns its result copied onto the heap; the caller (napi_run_with_resolver) +// frees it via g_resolver_last_result after the native side has copied it. +static char* resolve_module_callback(void* thread, const char* module_path) { + (void)thread; + + if (g_resolver_env == NULL || g_resolver_ref == NULL) { + return NULL; // No resolver set + } + + // Guard against cross-thread napi calls. The engine that triggers this + // callback is a process-wide singleton shared by run()/runStreaming()/ + // runTransform(); streaming and transform execute their native call on a + // background uv_thread (streaming_thread_fn/transform_thread_fn), not the + // JS thread that registered g_resolver_env/g_resolver_ref. If we're not + // on the thread that owns this napi_env, calling napi_get_reference_value + // or napi_call_function here is undefined behavior (typically a crash). + // Fail closed instead: report "not found", which matches the documented + // built-ins-only fallback for streaming/transform. + uv_thread_t current = uv_thread_self(); + if (!uv_thread_equal(¤t, &g_resolver_thread)) { + return NULL; + } + + napi_env env = g_resolver_env; + + napi_value js_callback; + if (napi_get_reference_value(env, g_resolver_ref, &js_callback) != napi_ok) { + return NULL; + } + + // NameIdentifierHelper.toWeaveFilePath (Java side, via CallbackWeaveResourceResolver) + // always renders paths with a leading separator, e.g. "/org/test/lib.dwl". Every + // resolver factory in resolver.ts (modulesFromMap, modulesFromDirectory, ...) and + // their documented examples key/join on the separator-less form ("org/test/lib.dwl"), + // so strip exactly one leading '/' here before handing the path to JS. + const char* js_module_path = module_path; + if (js_module_path[0] == '/') { + js_module_path++; + } + + napi_value module_path_str; + if (napi_create_string_utf8(env, js_module_path, NAPI_AUTO_LENGTH, &module_path_str) != napi_ok) { + return NULL; + } + + napi_value undefined, result; + napi_get_undefined(env, &undefined); + napi_status status = napi_call_function(env, undefined, js_callback, 1, &module_path_str, &result); + if (status != napi_ok) { + // JS resolver threw — clear the pending exception so it doesn't leak + // into the next napi call, extract and log its message/stack for + // diagnostics (same pattern as the read-callback bridge above), and + // report "not found". + if (status == napi_pending_exception) { + napi_value exception; + napi_get_and_clear_last_exception(env, &exception); + + // The resolver is user-provided code; its exception message/stack + // can carry module source, file paths, credentials, or other + // tenant data. Logging that to stderr by default risks leaking it + // into aggregated log systems. Only log a fixed, content-free + // diagnostic unless the caller has opted in via + // DATAWEAVE_RESOLVER_DEBUG=1 (checked once and cached, since + // getenv() is not safe to call from arbitrary threads on all + // platforms and this callback can run off the JS thread). + static int debug_checked = 0; + static int debug_enabled = 0; + if (!debug_checked) { + const char* debug_env = getenv("DATAWEAVE_RESOLVER_DEBUG"); + debug_enabled = (debug_env != NULL && strcmp(debug_env, "1") == 0); + debug_checked = 1; + } + + if (!debug_enabled) { + fprintf(stderr, + "[DataWeave Node addon] Resolver callback threw an exception " + "(details suppressed; set DATAWEAVE_RESOLVER_DEBUG=1 to log " + "message/stack — may expose resolver-controlled data).\n"); + } else { + napi_value message_prop, stack_prop; + char message_buf[512] = {0}; + char stack_buf[2048] = {0}; + size_t message_len = 0, stack_len = 0; + + if (napi_get_named_property(env, exception, "message", &message_prop) == napi_ok) { + napi_get_value_string_utf8(env, message_prop, message_buf, sizeof(message_buf), &message_len); + } + + if (napi_get_named_property(env, exception, "stack", &stack_prop) == napi_ok) { + napi_get_value_string_utf8(env, stack_prop, stack_buf, sizeof(stack_buf), &stack_len); + } + + fprintf(stderr, "[DataWeave Node addon] Resolver callback threw exception:\n"); + if (message_len > 0) { + fprintf(stderr, " Message: %s\n", message_buf); + } + if (stack_len > 0) { + fprintf(stderr, " Stack:\n%s\n", stack_buf); + } + if (message_len == 0 && stack_len == 0) { + fprintf(stderr, " (Unable to extract exception details)\n"); + } + } + } else { + fprintf(stderr, "Resolver callback threw exception\n"); + } + return NULL; + } + + napi_valuetype result_type; + napi_typeof(env, result, &result_type); + + char* result_source = NULL; + if (result_type == napi_string) { + size_t len; + napi_get_value_string_utf8(env, result, NULL, 0, &len); + result_source = (char*)malloc(len + 1); + if (result_source != NULL) { + napi_get_value_string_utf8(env, result, result_source, len + 1, NULL); + } + } + // null/undefined/other → not found (result_source stays NULL) + + resolver_results_track(result_source); + return result_source; // Native copies this immediately; we free the original after the call. +} + +// N-API method: runWithResolver +static napi_value napi_run_with_resolver(napi_env env, napi_callback_info info) { + if (!g_initialized) { + napi_throw_error(env, NULL, "Not initialized. Call initialize() first."); + return NULL; + } + if (!fn_run_script_with_resolver) { + napi_throw_error(env, NULL, "run_script_with_resolver not available in native library"); + return NULL; + } + + size_t argc = 5; + napi_value args[5]; + napi_get_cb_info(env, info, &argc, args, NULL, NULL); + + if (argc < 5) { + napi_throw_error(env, NULL, "Expected 5 arguments: script, inputs, mimeType, resolverCallback, isolate"); + return NULL; + } + + // Extract script, inputs, mimeType + size_t script_len, inputs_len, mime_len; + napi_get_value_string_utf8(env, args[0], NULL, 0, &script_len); + napi_get_value_string_utf8(env, args[1], NULL, 0, &inputs_len); + napi_get_value_string_utf8(env, args[2], NULL, 0, &mime_len); + + char* script = (char*)malloc(script_len + 1); + char* inputs = (char*)malloc(inputs_len + 1); + char* mime_type = (char*)malloc(mime_len + 1); + + if (script == NULL || inputs == NULL || mime_type == NULL) { + free(script); + free(inputs); + free(mime_type); + napi_throw_error(env, NULL, "Failed to allocate memory for arguments"); + return NULL; + } + + napi_get_value_string_utf8(env, args[0], script, script_len + 1, NULL); + napi_get_value_string_utf8(env, args[1], inputs, inputs_len + 1, NULL); + napi_get_value_string_utf8(env, args[2], mime_type, mime_len + 1, NULL); + + // Resolver is installed once per process lifetime. Subsequent calls with + // different resolver callbacks will reuse the first resolver, as enforced by + // ScriptRuntime.setResolver() on the native side (one resolver per engine). + // + // No thread-hop machinery is needed: fn_run_script_with_resolver() below + // runs on this very thread, so resolve_module_callback() (invoked from + // inside that call) can call directly back into JS via the stored + // napi_ref. See the comment on g_resolver_env for why napi_threadsafe_function + // must NOT be used here. + uv_mutex_lock(&g_mutex); + if (g_resolver_ref == NULL) { + napi_status status = napi_create_reference(env, args[3], 1, &g_resolver_ref); + if (status != napi_ok) { + uv_mutex_unlock(&g_mutex); + free(script); + free(inputs); + free(mime_type); + napi_throw_error(env, NULL, "Failed to reference resolver callback"); + return NULL; + } + g_resolver_env = env; + g_resolver_thread = uv_thread_self(); + } + // Note: subsequent calls reuse the first resolver for this process lifetime. + uv_mutex_unlock(&g_mutex); + + // Need to attach thread for this call + void* thread = NULL; + int rc = fn_attach_thread(g_isolate, &thread); + if (rc != 0) { + free(script); + free(inputs); + free(mime_type); + napi_throw_error(env, NULL, "Failed to attach thread"); + return NULL; + } + + // Call native with resolver callback. mime_type is accepted from JS for API + // symmetry but is not part of the native run_script_with_resolver signature + // (see run_script_with_resolver_fn typedef comment) — do not forward it. + char* result = fn_run_script_with_resolver( + thread, + script, + inputs, + resolve_module_callback + ); + + // Native has copied every resolver result returned during this call; free + // our copies now that it's done. + resolver_results_free_all(); + + // result (if non-NULL) is a GraalVM UnmanagedMemory.malloc'd buffer, like + // every other native result pointer in this file; it must be released via + // fn_free_cstring(), not libc free(), and while the isolate thread is + // still attached. Copy it to a libc-owned buffer first so we can build + // the JS string after detaching, matching the strdup + fn_free_cstring + // pattern used by run_script_thread_fn/streaming_thread_fn/transform_thread_fn. + char* result_copy = result ? strdup(result) : NULL; + if (result != NULL) { + fn_free_cstring(thread, result); + } + + fn_detach_thread(thread); + + free(script); + free(inputs); + free(mime_type); + + if (result_copy == NULL) { + napi_throw_error(env, NULL, "Script execution failed"); + return NULL; + } + + napi_value result_str; + napi_create_string_utf8(env, result_copy, NAPI_AUTO_LENGTH, &result_str); + free(result_copy); + + return result_str; +} + // --- Cleanup (must run on a separate thread to avoid V8 signal handler conflict) --- static void cleanup_thread_fn(void* arg) { @@ -653,6 +1000,14 @@ static napi_value napi_cleanup(napi_env env, napi_callback_info info) { if (g_initialized) { g_ref_count--; if (g_ref_count <= 0) { + // Clean up resolver reference + if (g_resolver_ref != NULL && g_resolver_env != NULL) { + napi_delete_reference(g_resolver_env, g_resolver_ref); + } + g_resolver_ref = NULL; + g_resolver_env = NULL; + resolver_results_free_all(); + uv_thread_t tid; uv_thread_options_t opts; opts.flags = UV_THREAD_HAS_STACK_SIZE; @@ -672,8 +1027,12 @@ static napi_value napi_cleanup(napi_env env, napi_callback_info info) { // --- Module init --- -static napi_value Init(napi_env env, napi_value exports) { +static void init_g_mutex(void) { uv_mutex_init(&g_mutex); +} + +static napi_value Init(napi_env env, napi_value exports) { + uv_once(&g_mutex_once, init_g_mutex); napi_value fn; @@ -689,6 +1048,9 @@ static napi_value Init(napi_env env, napi_value exports) { napi_create_function(env, "runScriptTransform", NAPI_AUTO_LENGTH, napi_run_script_transform, NULL, &fn); napi_set_named_property(env, exports, "runScriptTransform", fn); + napi_create_function(env, "runWithResolver", NAPI_AUTO_LENGTH, napi_run_with_resolver, NULL, &fn); + napi_set_named_property(env, exports, "runWithResolver", fn); + napi_create_function(env, "cleanup", NAPI_AUTO_LENGTH, napi_cleanup, NULL, &fn); napi_set_named_property(env, exports, "cleanup", fn); diff --git a/native-lib/node/src/dataweave.ts b/native-lib/node/src/dataweave.ts index 9956862b..dbaa63a3 100644 --- a/native-lib/node/src/dataweave.ts +++ b/native-lib/node/src/dataweave.ts @@ -5,6 +5,48 @@ import { createChunkReader } from "./reader"; import { streamFromNative } from "./stream"; import { DataWeaveError, DataWeaveScriptError } from "./errors"; import type { ExecutionResult, StreamingResult, Inputs, TransformOptions } from "./types"; +import type { ModuleResolver } from "./resolver"; + +/** + * Constructor options for {@link DataWeave}. + */ +export interface DataWeaveOptions { + /** + * Path to dwlib native library. + * If not provided, uses default location. + */ + libPath?: string; + + /** + * Module resolver for external DataWeave modules. + * Optional. If not provided, only built-in modules are available. + * + * MUST be synchronous (cannot return Promise). + * + * Note: the native layer installs at most one resolver per process + * lifetime, bound on the first resolver-backed {@link DataWeave.run} call + * (not on {@link DataWeave.initialize}, which only loads/ref-counts the + * native library) and to the thread (main thread or `worker_threads` + * Worker) that made that first call. If you construct multiple `DataWeave` + * instances with different `resolveModule` callbacks in the same process, + * whichever instance's `run()` executes first wins; later instances + * silently reuse that resolver instead of their own. If a later instance's + * `run()` executes on a *different* thread, its resolver is not invoked at + * all and custom module paths resolve as "not found" (see + * docs/external-modules.md#multiple-resolvers-in-one-process). + * + * Concurrency warning: calling a resolver-backed `run()` concurrently from + * more than one Worker is not just unsupported — it is memory-unsafe (see + * docs/external-modules.md, Worker threads section). Restrict + * resolver-backed execution to a single thread, or serialize calls across + * Workers. + * + * Security: the resolver runs with full process permissions and no + * sandboxing (same trust model as the CLI resolving `.dwl` files from + * disk) — only use resolvers pointed at trusted sources. + */ + resolveModule?: ModuleResolver; +} /** * A handle to the DataWeave native runtime for executing scripts. @@ -18,14 +60,22 @@ import type { ExecutionResult, StreamingResult, Inputs, TransformOptions } from */ export class DataWeave { private readonly libPath: string; + private readonly resolveModule?: ModuleResolver; private initialized = false; /** - * @param libPath - Absolute path to the `dwlib` shared library. When omitted, - * it is discovered via {@link findLibrary} (env var, packaged, or dev-build). + * @param options - Configuration options or a legacy libPath string. + * When a string is provided, it is treated as {@link DataWeaveOptions.libPath}. */ - constructor(libPath?: string) { - this.libPath = libPath ?? findLibrary(); + constructor(options?: DataWeaveOptions | string) { + if (typeof options === "string") { + // Legacy constructor signature: DataWeave(libPath) + this.libPath = options; + this.resolveModule = undefined; + } else { + this.libPath = options?.libPath ?? findLibrary(); + this.resolveModule = options?.resolveModule; + } } /** @@ -68,7 +118,16 @@ export class DataWeave { run(script: string, inputs?: Inputs, opts?: { raiseOnError?: boolean }): ExecutionResult { this.ensureInitialized(); const inputsJson = buildInputsJson(inputs ?? {}); - const raw = ffi.runScript(script, inputsJson); + + let raw: string; + if (this.resolveModule) { + // Use resolver-aware entrypoint + raw = ffi.runWithResolver(script, inputsJson, "application/json", this.resolveModule); + } else { + // Use standard entrypoint (backward compatible) + raw = ffi.runScript(script, inputsJson); + } + const result = parseNativeResponse(raw); if (opts?.raiseOnError && !result.success) { diff --git a/native-lib/node/src/ffi.ts b/native-lib/node/src/ffi.ts index 75bb65b8..924de436 100644 --- a/native-lib/node/src/ffi.ts +++ b/native-lib/node/src/ffi.ts @@ -1,4 +1,5 @@ import { join } from "node:path"; +import type { ModuleResolver } from "./resolver"; interface NativeAddon { initialize(libPath: string): void; @@ -13,6 +14,13 @@ interface NativeAddon { readCb: (bufSize: number) => Buffer | null, writeCb: (chunk: Buffer) => void ): Promise; + runWithResolver( + script: string, + inputsJson: string, + mimeType: string, + resolverCallback: ModuleResolver, + isolate: null + ): string; cleanup(): void; } @@ -54,6 +62,15 @@ export function runScriptTransform( return getAddon().runScriptTransform(script, inputsJson, inputName, inputMimeType, inputCharset, readCb, writeCb); } +export function runWithResolver( + script: string, + inputsJson: string, + mimeType: string, + resolverCallback: ModuleResolver +): string { + return getAddon().runWithResolver(script, inputsJson, mimeType, resolverCallback, null); +} + export function cleanup(): void { getAddon().cleanup(); } diff --git a/native-lib/node/src/index.ts b/native-lib/node/src/index.ts index 05711e29..073d0349 100644 --- a/native-lib/node/src/index.ts +++ b/native-lib/node/src/index.ts @@ -1,5 +1,12 @@ export { DataWeave, run, runStreaming, runTransform, cleanup } from "./dataweave"; +export type { DataWeaveOptions } from "./dataweave"; export { DataWeaveError, DataWeaveScriptError } from "./errors"; +export { + modulesFromMap, + modulesFromDirectory, + modulesFromJars, + composeResolvers, +} from "./resolver"; export type { ExecutionResult, @@ -8,4 +15,6 @@ export type { InputValue, InputEntry, TransformOptions, -} from "./types"; \ No newline at end of file +} from "./types"; + +export type { ModuleResolver } from "./resolver"; \ No newline at end of file diff --git a/native-lib/node/src/resolver.ts b/native-lib/node/src/resolver.ts new file mode 100644 index 00000000..d72e251a --- /dev/null +++ b/native-lib/node/src/resolver.ts @@ -0,0 +1,195 @@ +import * as fs from "fs"; +import * as path from "path"; +import AdmZip from "adm-zip"; + +/** + * Module resolver function type. + * Takes a module path (e.g., "org/mule/weave/v2/libs/lib.dwl") and returns + * the .dwl source as a string, or null if not found. + * + * MUST be synchronous (no async/await, no Promise return). + */ +export type ModuleResolver = (modulePath: string) => string | null; + +/** + * Creates a resolver backed by an in-memory map of path -> source. + * + * @param modules Map of module paths to .dwl source + * @returns Resolver function + * + * @example + * const resolver = modulesFromMap({ + * 'org/test/lib.dwl': '%dw 2.0\nfun greet(n) = "Hello " ++ n' + * }); + */ +export function modulesFromMap(modules: Record): ModuleResolver { + return (modulePath: string): string | null => { + // Object.hasOwn (not `in`) avoids matching inherited properties like + // "toString" or "constructor", which would violate the string | null + // contract above. + if (Object.hasOwn(modules, modulePath)) { + return modules[modulePath]; + } + return null; + }; +} + +/** + * True iff `candidate` is `base` itself or lies strictly beneath it. + * + * Deliberately not a `startsWith(base + path.sep)` string check: that + * degrades at a root base (`/` -> `startsWith("//")`, and the analogous + * duplicated separator on Windows drive/UNC roots), which would reject every + * legitimate child of a root-level baseDir. path.relative() has no such edge + * case: a candidate outside base always relates back via a leading "..", or + * — for a different drive/root on Windows — comes back absolute. + */ +function isContained(base: string, candidate: string): boolean { + if (candidate === base) return true; + const rel = path.relative(base, candidate); + // An escape climbs out via a ".." segment (rel === ".." or rel starts with + // "../"). A literal filename that merely starts with two dots, e.g. + // "..foo.dwl", is a single segment and is NOT an escape — checking for the + // separator (or an exact ".." match) avoids misclassifying it. + const isUpwardEscape = rel === ".." || rel.startsWith(".." + path.sep); + return rel !== "" && !isUpwardEscape && !path.isAbsolute(rel); +} + +/** + * Creates a resolver that reads .dwl files from a directory tree. + * Scans recursively for nested namespace structures. + * Reads from disk on every resolution (no caching). + * + * @param baseDir Base directory to scan for .dwl files + * @returns Resolver function + * + * @example + * const resolver = modulesFromDirectory('./my-modules'); + * // Resolves "org/test/lib.dwl" → reads "./my-modules/org/test/lib.dwl" + */ +export function modulesFromDirectory(baseDir: string): ModuleResolver { + // Unresolved base, for the cheap lexical check below (must be compared + // against an equally-unresolved candidate path — see baseDirResolved). + const baseDirLexical = path.resolve(baseDir); + // Canonicalized base, for the filesystem-truth check below. This also + // fails fast if baseDir doesn't exist, rather than silently resolving + // nothing. Kept separate from baseDirLexical: on macOS, os.tmpdir() (and + // other paths) can live under a symlink (e.g. /var -> /private/var), so + // comparing an unresolved candidate against a canonicalized base would + // reject every legitimate path. + const baseDirResolved = fs.realpathSync(baseDir); + + return (modulePath: string): string | null => { + // Join against the already-absolute baseDirLexical, not the original + // (possibly relative) baseDir: a relative baseDir re-resolves against the + // *current* cwd on every call, so a later process.chdir() would silently + // relocate every lookup. baseDirLexical was captured once, above, so it + // is stable regardless of later chdir() calls. + const fullPath = path.resolve(path.join(baseDirLexical, modulePath)); + + // Lexical containment check first (cheap, catches plain ".." traversal + // before touching the filesystem). Compared against the unresolved base + // so a symlinked baseDir itself doesn't cause a false rejection. + if (!isContained(baseDirLexical, fullPath)) { + return null; // Path escapes baseDir + } + + let realPath: string; + try { + // Canonicalize the candidate too: a symlink inside baseDir can point + // outside it and would otherwise pass the lexical check above, since + // path.resolve() never touches the filesystem. + realPath = fs.realpathSync(fullPath); + } catch (error) { + if (error instanceof Error && "code" in error && (error as NodeJS.ErrnoException).code === "ENOENT") { + return null; + } + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to read module ${modulePath} from ${fullPath}: ${message}`); + } + + // Re-check containment against the canonical path, rejecting symlinks + // (or symlinked ancestor directories) that resolve outside baseDir. + if (!isContained(baseDirResolved, realPath)) { + return null; + } + + try { + return fs.readFileSync(realPath, "utf-8"); + } catch (error) { + // File not found is expected, return null + if (error instanceof Error && "code" in error && (error as NodeJS.ErrnoException).code === "ENOENT") { + return null; + } + // Other errors (permissions, invalid UTF-8, etc.) should throw + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to read module ${modulePath} from ${fullPath}: ${message}`); + } + }; +} + +/** + * Creates a resolver that extracts .dwl files from JAR archives. + * Returns a Promise because JAR extraction must complete before resolver is used. + * The returned resolver itself is synchronous (backed by in-memory map). + * + * @param jarPaths Array of paths to JAR files + * @returns Promise resolving to resolver function + * + * @example + * const resolver = await modulesFromJars(['./libs/dw-strings.jar']); + * // Now use synchronously: resolver('dw/core/Strings.dwl') + */ +export async function modulesFromJars(jarPaths: string[]): Promise { + const modules: Record = {}; + + for (const jarPath of jarPaths) { + try { + const zip = new AdmZip(jarPath); + const entries = zip.getEntries(); + + for (const entry of entries) { + // Extract only .dwl files, skip directories + if (!entry.isDirectory && entry.entryName.endsWith(".dwl")) { + const source = entry.getData().toString("utf-8"); + modules[entry.entryName] = source; + } + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to read JAR ${jarPath}: ${message}`); + } + } + + // Return synchronous resolver backed by extracted modules + return modulesFromMap(modules); +} + +/** + * Composes multiple resolvers into one with fallback chain. + * Tries each resolver in order, returns first non-null result. + * + * @param resolvers Resolvers to try in order + * @returns Composite resolver function + * + * @example + * const resolver = composeResolvers( + * modulesFromMap({ 'override.dwl': '...' }), // Try first + * modulesFromDirectory('./shared'), // Then directory + * await modulesFromJars(['./vendor/lib.jar']) // Finally JAR + * ); + */ +export function composeResolvers(...resolvers: ModuleResolver[]): ModuleResolver { + return (modulePath: string): string | null => { + for (const resolver of resolvers) { + const result = resolver(modulePath); + if (result !== null) { + return result; // First match wins + } + } + + // None matched + console.debug(`Module not found in any resolver: ${modulePath}`); + return null; + }; +} diff --git a/native-lib/node/tests/fixtures/test-lib.jar b/native-lib/node/tests/fixtures/test-lib.jar new file mode 100644 index 00000000..7cbd1dae Binary files /dev/null and b/native-lib/node/tests/fixtures/test-lib.jar differ diff --git a/native-lib/node/tests/integration/dataweave-resolver.test.ts b/native-lib/node/tests/integration/dataweave-resolver.test.ts new file mode 100644 index 00000000..6578bb6b --- /dev/null +++ b/native-lib/node/tests/integration/dataweave-resolver.test.ts @@ -0,0 +1,170 @@ +import { describe, it, expect, afterAll } from "vitest"; +import { DataWeave, cleanup } from '../../src/dataweave'; +import { modulesFromMap } from '../../src/resolver'; + +// Every test below constructs its own explicit DataWeave instance (rather +// than the module-level singleton) so each can configure its own resolver. +// `cleanup()` above only releases the *singleton* (`globalInstance`), which +// nothing in this file ever creates -- so without this tracking, every +// explicit instance's native library reference (and the shared addon-level +// ref-count, see addon.c's g_ref_count) would leak for the lifetime of the +// test process. Track every instance created in this file and release them +// all in afterAll. +const instances: DataWeave[] = []; +function trackedDataWeave(...args: ConstructorParameters): DataWeave { + const dw = new DataWeave(...args); + instances.push(dw); + return dw; +} + +afterAll(() => { + for (const dw of instances) { + dw.cleanup(); + } + cleanup(); +}); + +// ScriptRuntime installs at most one resolver for the whole process lifetime +// (see ScriptRuntime.setResolver()): whichever DataWeave instance's resolver +// gets installed first "wins", and every later DataWeave instance in this +// file — regardless of its own resolveModule map — silently reuses it. Since +// vitest runs the `it` blocks in this file sequentially in the same process, +// that's always this first module-map, so it must contain every module path +// any test below needs to resolve for the first time (including the +// cross-thread regression test's two never-before-resolved paths). +const SHARED_RESOLVER_MODULES: Record = { + 'org/test/lib.dwl': '%dw 2.0\nfun greet(n: String) = "Hello " ++ n', + 'org/test/resolverGuardInstall.dwl': '%dw 2.0\nfun greet(n: String) = "Hello " ++ n', + 'org/test/resolverGuardStreamed.dwl': '%dw 2.0\nfun greet(n: String) = "Hello " ++ n', +}; + +describe('DataWeave with resolver', () => { + it('resolves imported module from map', () => { + const dw = trackedDataWeave({ + resolveModule: modulesFromMap(SHARED_RESOLVER_MODULES), + }); + dw.initialize(); + + const result = dw.run(` + %dw 2.0 + import org::test::lib + output application/json + --- + lib::greet("World") + `); + + expect(result.success).toBe(true); + expect(JSON.parse(result.getString()!)).toBe("Hello World"); + }); + + it('works without resolver (backward compatible)', () => { + const dw = trackedDataWeave(); // No resolver + dw.initialize(); + + const result = dw.run(` + %dw 2.0 + output application/json + --- + { message: "no imports" } + `); + + expect(result.success).toBe(true); + expect(JSON.parse(result.getString()!)).toEqual({ message: "no imports" }); + }); + + it('throws when module not found', () => { + const dw = trackedDataWeave({ + resolveModule: modulesFromMap({ 'a.dwl': '...' }), + }); + dw.initialize(); + + expect(() => dw.run(` + %dw 2.0 + import missing::mod + output application/json + --- + {} + `, undefined, { raiseOnError: true })).toThrow(); + }); + + it('built-in modules still resolve with resolver', () => { + const dw = trackedDataWeave({ + resolveModule: modulesFromMap({ 'custom.dwl': '...' }), + }); + dw.initialize(); + + // Built-in modules should still work (CompositeResolver: ClassLoader + Callback) + const result = dw.run(` + %dw 2.0 + import dw::core::Strings + output application/json + --- + Strings::capitalize("hello") + `); + + expect(result.success).toBe(true); + expect(JSON.parse(result.getString()!)).toBe("Hello"); + }); + + // Regression test for the cross-thread resolver hazard: ScriptRuntime's engine + // is a process-wide singleton, so once any .run() call installs a resolver on + // it, that same composite resolver is used by ALL later execution paths -- + // including runStreaming()/runTransform(), whose native call executes on a + // background uv_thread (see addon.c's streaming_thread_fn), not the JS thread + // that registered the resolver. Before the thread-identity guard in addon.c's + // resolve_module_callback, a streamed script importing a non-built-in module + // would trigger a napi call from that background thread -- undefined behavior, + // typically a crash of the whole process. After the guard, the callback fails + // closed (reports "not found" instead of calling back into JS), so the script + // fails cleanly with a compile error and the process survives. + it('runStreaming fails cleanly (does not crash) for a custom module on the shared singleton engine', async () => { + // Once a module name has been resolved anywhere in the process, the + // DataWeave compiler caches it and won't call back into the resolver for + // that same name again — so the install script and the streaming script + // below import two module paths that no earlier test in this file has + // imported yet (both pre-registered in SHARED_RESOLVER_MODULES above, + // since only the first-installed resolver's map is ever consulted). + const dw = trackedDataWeave({ + resolveModule: modulesFromMap(SHARED_RESOLVER_MODULES), + }); + dw.initialize(); + + // Install (or confirm already-installed) resolver on the shared singleton + // engine via a synchronous run() call. Per ScriptRuntime.setResolver(), only + // the first resolver registered for the process is ever used, so this is + // safe to call even if an earlier test in this file already installed one. + const installResult = dw.run(` + %dw 2.0 + import org::test::resolverGuardInstall + output application/json + --- + resolverGuardInstall::greet("Installer") + `); + expect(installResult.success).toBe(true); + + // Now stream a script that imports a DIFFERENT non-built-in module, never + // resolved before in this process. The singleton engine's composite + // resolver (ClassLoader + Callback) will miss in the ClassLoader half (not + // a built-in) and fall through to the Callback half, invoking + // resolve_module_callback from runStreaming's background thread. + const chunks: Buffer[] = []; + const gen = dw.runStreaming(` + %dw 2.0 + import org::test::resolverGuardStreamed + output application/json + --- + resolverGuardStreamed::greet("Streaming") + `); + let result = await gen.next(); + while (!result.done) { + chunks.push(result.value); + result = await gen.next(); + } + const metadata = result.value; + + // Fails cleanly (built-ins-only fallback), rather than crashing the process. + expect(metadata.success).toBe(false); + expect(metadata.error).toBeTruthy(); + expect(chunks.length).toBe(0); + }); +}); diff --git a/native-lib/node/tests/integration/first-resolver-wins.test.ts b/native-lib/node/tests/integration/first-resolver-wins.test.ts new file mode 100644 index 00000000..75e7da59 --- /dev/null +++ b/native-lib/node/tests/integration/first-resolver-wins.test.ts @@ -0,0 +1,34 @@ +// Verifies the process-wide "first resolver wins" behavior documented in +// docs/external-modules.md#multiple-resolvers-in-one-process and +// ScriptRuntime.setResolver(): once a DataWeave instance's resolver is +// installed on the native engine singleton, a second instance constructed +// with a *different* resolver in the same process never has its resolver +// installed. That's only observable when the second instance's resolver is +// the second one ever installed for the whole process, so — like +// init-bad-path.test.ts — this runs in a dedicated child process rather than +// in-lane, making it order- and pool-configuration-independent. +import { describe, it, expect } from "vitest"; +import { execFileSync } from "node:child_process"; +import { join } from "node:path"; +import { existsSync } from "node:fs"; + +const FIXTURE = join(__dirname, "fixtures", "first-resolver-wins.cjs"); +const DIST_ENTRY = join(__dirname, "..", "..", "dist", "index.js"); + +describe("first-resolver-wins (isolated process)", () => { + it("a second DataWeave instance's resolver is silently ignored in favor of the first", () => { + expect(existsSync(DIST_ENTRY), `built entry missing at ${DIST_ENTRY} — run \`npm run build:ts\``).toBe(true); + + // execFileSync throws on a non-zero exit, so a "wrong resolver won" / + // native-crash outcome in the child fails this test. A timeout is also + // required: execFileSync blocks synchronously with no way for Vitest to + // interrupt it, so a native deadlock in the child would otherwise hang + // the whole suite instead of failing this one test. + const stdout = execFileSync(process.execPath, [FIXTURE], { + encoding: "utf-8", + timeout: 30_000, + }); + + expect(stdout).toContain("OK:first-resolver-wins"); + }); +}); diff --git a/native-lib/node/tests/integration/fixtures/first-resolver-wins.cjs b/native-lib/node/tests/integration/fixtures/first-resolver-wins.cjs new file mode 100644 index 00000000..3dc2fd42 --- /dev/null +++ b/native-lib/node/tests/integration/fixtures/first-resolver-wins.cjs @@ -0,0 +1,103 @@ +// Child-process fixture for the first-resolver-wins regression test. +// +// Runs in a FRESH process (spawned by first-resolver-wins.test.ts) so the +// process-wide ScriptRuntime singleton in the native layer starts with no +// resolver installed (see ScriptRuntime.setResolver(): once any DataWeave +// instance's resolver is installed, every later instance's resolver is +// silently ignored — a warning is logged and the first resolver keeps being +// used). That behavior is only observable on the FIRST resolver installation +// of a process, so this fixture -- not an in-lane vitest test -- is the only +// reliable way to exercise it. +// +// Contract with the parent: +// - Requires the built CommonJS entry at ../../../dist/index.js. +// - Constructs dw1 with a resolver for 'first.dwl' and dw2 with a +// *different* resolver for 'second.dwl', then initializes both. +// - Runs a script through dw1 that imports 'first.dwl' to force-install +// dw1's resolver on the singleton engine (must succeed). +// - Runs a script through dw2 that imports 'second.dwl'. Per the singleton +// semantics, dw2's resolver is never installed, so this import must fail. +// - Runs a THIRD script, through dw2, that imports 'first.dwl' again and +// asserts it still returns "Hello World". This is the check that actually +// distinguishes "the first resolver remains active" from "custom +// resolution broke entirely after the first call" — the second script +// alone would fail identically under either explanation. +// - Always calls cleanup() on both instances via try/finally, so teardown +// is exercised even on failure, then exits naturally (no process.exit()). +// - Prints "OK:first-resolver-wins" when all three expectations hold, or +// "FAIL:" (with a non-zero exitCode) otherwise. A native crash +// surfaces as a non-zero signal exit, which the parent also treats as +// failure. +const path = require("node:path"); + +const { DataWeave, modulesFromMap } = require(path.join(__dirname, "..", "..", "..", "dist", "index.js")); + +const dw1 = new DataWeave({ + resolveModule: modulesFromMap({ + "first.dwl": '%dw 2.0\nfun greet(n: String) = "Hello " ++ n', + }), +}); + +const dw2 = new DataWeave({ + resolveModule: modulesFromMap({ + "second.dwl": '%dw 2.0\nfun shout(n: String) = n ++ "!"', + }), +}); + +let failure = null; + +try { + dw1.initialize(); + dw2.initialize(); + + const firstResult = dw1.run(` + %dw 2.0 + import first + output application/json + --- + first::greet("World") + `); + + if (!firstResult.success) { + failure = "first-resolver-did-not-resolve:" + firstResult.error; + } else { + const secondResult = dw2.run(` + %dw 2.0 + import second + output application/json + --- + second::shout("hi") + `); + + if (secondResult.success) { + failure = "second-resolver-unexpectedly-won"; + } else { + // Prove the first resolver is still ACTIVE on dw2 (not merely that + // dw2's own resolver lost). A resolver that died entirely after the + // first call would also make second.dwl fail above -- this second + // check on dw2 is what actually distinguishes "first resolver wins" + // from "custom resolution stopped working after the first run". + const stillFirstResult = dw2.run(` + %dw 2.0 + import first + output application/json + --- + first::greet("World") + `); + + if (!stillFirstResult.success || JSON.parse(stillFirstResult.getString()) !== "Hello World") { + failure = "first-resolver-no-longer-active-on-dw2:" + (stillFirstResult.error || stillFirstResult.getString()); + } + } + } +} finally { + dw1.cleanup(); + dw2.cleanup(); +} + +if (failure) { + console.log("FAIL:" + failure); + process.exitCode = 1; +} else { + console.log("OK:first-resolver-wins"); +} diff --git a/native-lib/node/tests/integration/init-bad-path.test.ts b/native-lib/node/tests/integration/init-bad-path.test.ts index 367a1c5c..fd235fc5 100644 --- a/native-lib/node/tests/integration/init-bad-path.test.ts +++ b/native-lib/node/tests/integration/init-bad-path.test.ts @@ -20,9 +20,13 @@ describe("bad library path initialization (isolated process)", () => { expect(existsSync(DIST_ENTRY), `built entry missing at ${DIST_ENTRY} — run \`npm run build:ts\``).toBe(true); // execFileSync throws on a non-zero exit, so a "no throw" / wrong-error / - // native-crash outcome in the child fails this test. + // native-crash outcome in the child fails this test. A timeout is also + // required: execFileSync blocks synchronously with no way for Vitest to + // interrupt it, so a native deadlock in the child would otherwise hang + // the whole suite instead of failing this one test. const stdout = execFileSync(process.execPath, [FIXTURE, "/no/such/dwlib-xyz.dylib"], { encoding: "utf-8", + timeout: 30_000, }); expect(stdout).toContain("OK:DataWeaveError"); diff --git a/native-lib/node/tests/tck/fixtures/org/mule/weave/v2/libs/lib.dwl b/native-lib/node/tests/tck/fixtures/org/mule/weave/v2/libs/lib.dwl new file mode 100644 index 00000000..7c1215cb --- /dev/null +++ b/native-lib/node/tests/tck/fixtures/org/mule/weave/v2/libs/lib.dwl @@ -0,0 +1,3 @@ +var name="Shoki" +import upper from dw::Core +fun foo(value: String): String = upper(value) diff --git a/native-lib/node/tests/tck/ignore-list.ts b/native-lib/node/tests/tck/ignore-list.ts index e8d22aee..ea3b5c92 100644 --- a/native-lib/node/tests/tck/ignore-list.ts +++ b/native-lib/node/tests/tck/ignore-list.ts @@ -28,12 +28,6 @@ export interface IgnoreEntry { export const IGNORED_CASES: Readonly> = { // unresolved-module — library/resource not present in dwlib "dw-binary-out.dwl": { reason: "unresolved-module: readUrl/classpath resource" }, - "full-qualified-name-ref-out.json": { reason: "unresolved-module: org::mule::weave::v2::libs" }, - "import-component-alias-lib-out.json": { reason: "unresolved-module: import lib not in dwlib" }, - "import-lib-out.json": { reason: "unresolved-module: import lib not in dwlib" }, - "import-lib-with-alias-out.json": { reason: "unresolved-module: import lib not in dwlib" }, - "import-named-lib-out.json": { reason: "unresolved-module: import lib not in dwlib" }, - "import-star-out.json": { reason: "unresolved-module: import lib not in dwlib" }, "is-empty-using-empty-stream-out.json": { reason: "unresolved-module: dw::Client streaming" }, "module-singleton-out.json": { reason: "unresolved-module: lib not in dwlib" }, "private_scope_directives-out.xml": { reason: "unresolved-module: resource not in dwlib" }, diff --git a/native-lib/node/tests/tck/tck.test.ts b/native-lib/node/tests/tck/tck.test.ts index 5ce46159..aecfec2f 100644 --- a/native-lib/node/tests/tck/tck.test.ts +++ b/native-lib/node/tests/tck/tck.test.ts @@ -9,12 +9,13 @@ import { describe, it, expect } from "vitest"; import { readFileSync, readdirSync, existsSync, statSync } from "node:fs"; import { join } from "node:path"; -import { DataWeave } from "../../src/index"; +import { DataWeave, modulesFromDirectory } from "../../src/index"; import { parseCase, MAIN_TRANSFORM, type TckScenario } from "./case-loader"; import { compareOutput } from "./compare"; import { isIgnored, ignoreReason } from "./ignore-list"; const SUITES_DIR = join(__dirname, "suites"); +const FIXTURES_DIR = join(__dirname, "fixtures"); /** A discovered case: its directory and the scenarios parsed from it. */ interface DiscoveredCase { @@ -61,8 +62,11 @@ if (!existsSync(SUITES_DIR)) { } else { const { cases, skipped } = discoverCases(); - // One shared runtime for the whole lane. - const dw = new DataWeave(); + // One shared runtime for the whole lane. Modules imported by a handful of + // TCK cases (org::mule::weave::v2::libs::lib) live only in the private + // data-weave runtime repo's test resources, not in any published + // artifact/TCK zip — resolve them from a committed fixture instead. + const dw = new DataWeave({ resolveModule: modulesFromDirectory(FIXTURES_DIR) }); describe("TCK conformance", () => { // eslint-disable-next-line no-console diff --git a/native-lib/node/tests/unit/resolver.test.ts b/native-lib/node/tests/unit/resolver.test.ts new file mode 100644 index 00000000..711be21a --- /dev/null +++ b/native-lib/node/tests/unit/resolver.test.ts @@ -0,0 +1,312 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { modulesFromMap, modulesFromDirectory, modulesFromJars, composeResolvers } from "../../src/resolver"; +import * as fs from "fs"; +import * as path from "path"; +import * as os from "os"; +import AdmZip from "adm-zip"; + +describe("modulesFromMap", () => { + it("returns source when module exists", () => { + const resolver = modulesFromMap({ + "org/test/lib.dwl": '%dw 2.0\nfun greet(n) = "Hello " ++ n', + }); + + const result = resolver("org/test/lib.dwl"); + + expect(result).toBe('%dw 2.0\nfun greet(n) = "Hello " ++ n'); + }); + + it("returns null when module not found", () => { + const resolver = modulesFromMap({ + "org/test/lib.dwl": "%dw 2.0\n...", + }); + + const result = resolver("missing/mod.dwl"); + + expect(result).toBeNull(); + }); + + it("handles multiple modules", () => { + const resolver = modulesFromMap({ + "a.dwl": "source a", + "b.dwl": "source b", + }); + + expect(resolver("a.dwl")).toBe("source a"); + expect(resolver("b.dwl")).toBe("source b"); + expect(resolver("c.dwl")).toBeNull(); + }); + + it("returns empty string when module source is empty", () => { + const resolver = modulesFromMap({ + "org/test/empty.dwl": "", + }); + + expect(resolver("org/test/empty.dwl")).toBe(""); + }); +}); + +describe("modulesFromDirectory", () => { + let tempDir: string; + + beforeEach(() => { + // Create temp directory with test .dwl files + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "dw-test-")); + + // Create org/test/lib.dwl + const orgTestDir = path.join(tempDir, "org", "test"); + fs.mkdirSync(orgTestDir, { recursive: true }); + fs.writeFileSync( + path.join(orgTestDir, "lib.dwl"), + '%dw 2.0\nfun greet(n) = "Hello " ++ n' + ); + + // Create top-level simple.dwl + fs.writeFileSync(path.join(tempDir, "simple.dwl"), "%dw 2.0\nvar x = 42"); + }); + + afterEach(() => { + // Clean up temp directory + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + it("reads file from nested directory", () => { + const resolver = modulesFromDirectory(tempDir); + + const result = resolver("org/test/lib.dwl"); + + expect(result).toContain("%dw 2.0"); + expect(result).toContain("fun greet"); + }); + + it("reads file from root directory", () => { + const resolver = modulesFromDirectory(tempDir); + + const result = resolver("simple.dwl"); + + expect(result).toContain("var x = 42"); + }); + + it("returns null when file not found", () => { + const resolver = modulesFromDirectory(tempDir); + + const result = resolver("missing/file.dwl"); + + expect(result).toBeNull(); + }); + + it.skipIf(process.platform === "win32")("throws on unreadable file", () => { + const badFile = path.join(tempDir, "bad.dwl"); + fs.writeFileSync(badFile, "content"); + fs.chmodSync(badFile, 0o000); // Make unreadable + + const resolver = modulesFromDirectory(tempDir); + + expect(() => resolver("bad.dwl")).toThrow("Failed to read module"); + + // Cleanup + fs.chmodSync(badFile, 0o644); + }); + + it("returns null for path traversal attempts", () => { + const resolver = modulesFromDirectory(tempDir); + expect(resolver("../../outside.dwl")).toBeNull(); + }); + + it.skipIf(process.platform === "win32")( + "returns null when an in-tree symlink escapes baseDir", + () => { + // outsideDir sits alongside tempDir, outside the configured base. + const outsideDir = fs.mkdtempSync(path.join(os.tmpdir(), "dw-outside-")); + const secretFile = path.join(outsideDir, "secret.dwl"); + fs.writeFileSync(secretFile, "%dw 2.0\n// should never be resolved"); + + // A symlink inside tempDir that resolves outside it. The lexical + // containment check alone would accept this path; only realpath + // canonicalization catches the escape. + const linkPath = path.join(tempDir, "escape.dwl"); + fs.symlinkSync(secretFile, linkPath); + + const resolver = modulesFromDirectory(tempDir); + expect(resolver("escape.dwl")).toBeNull(); + + fs.rmSync(outsideDir, { recursive: true, force: true }); + } + ); + + it("keeps resolving a relative baseDir after process.chdir()", () => { + // modulesFromDirectory captures an absolute baseDirLexical up front. The + // candidate path built on each call must be derived from that captured + // absolute base -- not re-resolved against baseDir (which, if relative, + // silently tracks the *current* cwd) -- or a later chdir() breaks every + // lookup against a resolver that was already constructed and working. + const originalCwd = process.cwd(); + const relativeBase = path.relative(originalCwd, tempDir); + const resolver = modulesFromDirectory(relativeBase); + + const elsewhere = fs.mkdtempSync(path.join(os.tmpdir(), "dw-elsewhere-")); + try { + process.chdir(elsewhere); + const result = resolver("simple.dwl"); + expect(result).toContain("var x = 42"); + } finally { + process.chdir(originalCwd); + fs.rmSync(elsewhere, { recursive: true, force: true }); + } + }); +}); + +describe("modulesFromDirectory with a root-level baseDir", () => { + // A root base (POSIX "/", or a Windows drive root) makes `base + path.sep` + // duplicate the separator (e.g. "//"), which broke the old prefix-based + // containment check for every child path. Exercise the actual filesystem + // root's realpath so this covers whatever isContained() computes for it, + // without assuming write access to "/" itself. + it("does not reject a path solely because baseDir is a filesystem root", () => { + const root = path.parse(process.cwd()).root; // e.g. "/" or "C:\\" + const rootResolved = fs.realpathSync(root); + const resolver = modulesFromDirectory(root); + + // A false containment rejection and a genuine "not found" both surface as + // `null` from the resolver, so this needs a file that definitely exists + // under root to tell them apart -- create one under the OS temp dir, + // which is itself always a descendant of the filesystem root. + const probeDir = fs.mkdtempSync(path.join(os.tmpdir(), "dw-root-probe-")); + try { + const probeResolved = fs.realpathSync(probeDir); + expect(probeResolved.startsWith(rootResolved)).toBe(true); + + fs.writeFileSync(path.join(probeDir, "probe.dwl"), "%dw 2.0\nvar probe = true"); + // Resolve relative to the *actual* root, using the probe dir's path + // relative to root as the "module path" -- this only works if root + // containment doesn't reject valid, deeply-nested children. + const relFromRoot = path.relative(rootResolved, path.join(probeResolved, "probe.dwl")); + const result = resolver(relFromRoot); + expect(result).toContain("var probe = true"); + } finally { + fs.rmSync(probeDir, { recursive: true, force: true }); + } + }); +}); + +describe("modulesFromJars", () => { + it("extracts .dwl files from JAR", async () => { + const jarPath = path.join(__dirname, "..", "fixtures", "test-lib.jar"); + + const resolver = await modulesFromJars([jarPath]); + + const strings = resolver("dw/core/Strings.dwl"); + expect(strings).toContain("fun capitalize"); + + const math = resolver("org/test/math.dwl"); + expect(math).toContain("fun multiply"); + }); + + it("returns null when module not in JAR", async () => { + const jarPath = path.join(__dirname, "..", "fixtures", "test-lib.jar"); + + const resolver = await modulesFromJars([jarPath]); + + const result = resolver("missing/mod.dwl"); + + expect(result).toBeNull(); + }); + + it("handles multiple JARs", async () => { + const jarPath = path.join(__dirname, "..", "fixtures", "test-lib.jar"); + + // Build a second, distinct JAR (different module name) on the fly so + // this test actually exercises merging across archives, rather than + // passing trivially because only the first archive's contents matter. + const secondJarDir = fs.mkdtempSync(path.join(os.tmpdir(), "dw-jar-")); + const secondJarPath = path.join(secondJarDir, "second-lib.jar"); + const zip = new AdmZip(); + zip.addFile("org/test/second.dwl", Buffer.from('%dw 2.0\nfun square(n) = n * n')); + zip.writeZip(secondJarPath); + + const resolver = await modulesFromJars([jarPath, secondJarPath]); + + expect(resolver("dw/core/Strings.dwl")).toContain("fun capitalize"); + expect(resolver("org/test/second.dwl")).toContain("fun square"); + + fs.rmSync(secondJarDir, { recursive: true, force: true }); + }); + + it("throws on invalid JAR", async () => { + const badJar = path.join(__dirname, "..", "fixtures", "not-a-jar.txt"); + fs.writeFileSync(badJar, "not a zip file"); + + await expect(modulesFromJars([badJar])).rejects.toThrow("Failed to read JAR"); + + fs.unlinkSync(badJar); + }); + + it("ignores non-.dwl files in JAR", async () => { + // Test JAR contains only .dwl files, but verify behavior + const jarPath = path.join(__dirname, "..", "fixtures", "test-lib.jar"); + + const resolver = await modulesFromJars([jarPath]); + + // Should not throw, just not find non-.dwl paths + expect(resolver("some-text-file.txt")).toBeNull(); + }); +}); + +describe("composeResolvers", () => { + it("returns first match", () => { + const r1 = modulesFromMap({ "a.dwl": "source1" }); + const r2 = modulesFromMap({ "a.dwl": "source2" }); + + const composed = composeResolvers(r1, r2); + + expect(composed("a.dwl")).toBe("source1"); // First wins + }); + + it("falls through to next resolver on null", () => { + const r1 = modulesFromMap({ "a.dwl": "source1" }); + const r2 = modulesFromMap({ "b.dwl": "source2" }); + + const composed = composeResolvers(r1, r2); + + expect(composed("a.dwl")).toBe("source1"); // r1 matched + expect(composed("b.dwl")).toBe("source2"); // r1 returned null, r2 matched + }); + + it("returns null when all resolvers return null", () => { + const r1 = modulesFromMap({ "a.dwl": "source1" }); + const r2 = modulesFromMap({ "b.dwl": "source2" }); + + const composed = composeResolvers(r1, r2); + + expect(composed("c.dwl")).toBeNull(); + }); + + it("handles three resolvers", () => { + const r1 = modulesFromMap({ "a.dwl": "source1" }); + const r2 = modulesFromMap({ "b.dwl": "source2" }); + const r3 = modulesFromMap({ "c.dwl": "source3" }); + + const composed = composeResolvers(r1, r2, r3); + + expect(composed("a.dwl")).toBe("source1"); + expect(composed("b.dwl")).toBe("source2"); + expect(composed("c.dwl")).toBe("source3"); + expect(composed("d.dwl")).toBeNull(); + }); + + it("combines directory and map resolvers", () => { + // Create temp dir with one file + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "dw-test-")); + fs.writeFileSync(path.join(tempDir, "file.dwl"), "from disk"); + + const composed = composeResolvers( + modulesFromMap({ "override.dwl": "from map" }), + modulesFromDirectory(tempDir) + ); + + expect(composed("override.dwl")).toBe("from map"); // Map first + expect(composed("file.dwl")).toBe("from disk"); // Fallback to disk + + fs.rmSync(tempDir, { recursive: true, force: true }); + }); +}); diff --git a/native-lib/src/main/java/org/mule/weave/lib/CallbackWeaveResourceResolver.java b/native-lib/src/main/java/org/mule/weave/lib/CallbackWeaveResourceResolver.java new file mode 100644 index 00000000..d6b80912 --- /dev/null +++ b/native-lib/src/main/java/org/mule/weave/lib/CallbackWeaveResourceResolver.java @@ -0,0 +1,79 @@ +package org.mule.weave.lib; + +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.c.type.CCharPointer; +import org.graalvm.nativeimage.c.type.CTypeConversion; +import org.mule.weave.v2.parser.ast.variables.NameIdentifier; +import org.mule.weave.v2.sdk.NameIdentifierHelper; +import org.mule.weave.v2.sdk.WeaveResource; +import org.mule.weave.v2.sdk.WeaveResourceResolver; +import scala.Option; +import scala.collection.JavaConverters; +import scala.collection.immutable.Seq; +import scala.collection.immutable.Seq$; + +import java.util.Collections; + +/** + * WeaveResourceResolver implementation backed by a C function pointer callback. + * Delegates module resolution to the host environment (Node.js, Python, etc.). + */ +public class CallbackWeaveResourceResolver implements WeaveResourceResolver { + private final NativeCallbacks.ResolveModuleCallback callback; + + public CallbackWeaveResourceResolver(NativeCallbacks.ResolveModuleCallback callback) { + if (callback.isNull()) { + throw new IllegalArgumentException("Resolver callback cannot be null"); + } + this.callback = callback; + } + + @Override + public Option resolve(NameIdentifier nameIdentifier) { + // Convert NameIdentifier to file path (outside try block for error logging) + String path = NameIdentifierHelper.toWeaveFilePath(nameIdentifier, "/"); + + try { + // Convert path to C string + try (CTypeConversion.CCharPointerHolder pathHolder = + CTypeConversion.toCString(path)) { + CCharPointer pathPtr = pathHolder.get(); + + // Invoke callback (blocks if threadsafe function is in use) + CCharPointer resultPtr = callback.invoke( + CurrentIsolate.getCurrentThread(), + pathPtr + ); + + // Null means "not found" + if (resultPtr.isNull()) { + return Option.empty(); + } + + // Copy result to Java string immediately (host will free pointer) + String source = CTypeConversion.toJavaString(resultPtr); + + // Return as WeaveResource + return Option.apply( + WeaveResource.apply(path, source) + ); + } + } catch (Exception e) { + // Log and return empty on any error + System.err.println("Error resolving module " + path + ": " + e.getMessage()); + return Option.empty(); + } + } + + @Override + public Seq resolveAll(NameIdentifier nameIdentifier) { + // Not used for module resolution, return single result or empty + Option result = resolve(nameIdentifier); + if (result.isDefined()) { + return JavaConverters.asScalaBuffer(Collections.singletonList(result.get())) + .toList(); + } else { + return (Seq) Seq$.MODULE$.empty(); + } + } +} diff --git a/native-lib/src/main/java/org/mule/weave/lib/NativeCallbacks.java b/native-lib/src/main/java/org/mule/weave/lib/NativeCallbacks.java index 38723548..3e993c7e 100644 --- a/native-lib/src/main/java/org/mule/weave/lib/NativeCallbacks.java +++ b/native-lib/src/main/java/org/mule/weave/lib/NativeCallbacks.java @@ -1,5 +1,6 @@ package org.mule.weave.lib; +import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.function.InvokeCFunctionPointer; import org.graalvm.nativeimage.c.type.CCharPointer; @@ -46,4 +47,14 @@ public interface ReadCallback extends CFunctionPointer { @InvokeCFunctionPointer int invoke(PointerBase ctx, CCharPointer buffer, int bufferSize); } + + /** + * Callback invoked by native code to resolve DataWeave modules. + * Takes a module path (e.g., "org/mule/weave/v2/libs/lib.dwl") and returns + * the .dwl source as a C string, or null if the module is not found. + */ + public interface ResolveModuleCallback extends CFunctionPointer { + @InvokeCFunctionPointer + CCharPointer invoke(IsolateThread thread, CCharPointer modulePath); + } } diff --git a/native-lib/src/main/java/org/mule/weave/lib/NativeLib.java b/native-lib/src/main/java/org/mule/weave/lib/NativeLib.java index de776e7f..549ea3ac 100644 --- a/native-lib/src/main/java/org/mule/weave/lib/NativeLib.java +++ b/native-lib/src/main/java/org/mule/weave/lib/NativeLib.java @@ -330,4 +330,239 @@ private static CCharPointer toUnmanagedCString(String value) { return ptr; } + // ── Resolver-aware FFI Entrypoints ─────────────────────────────────── + + /** + * Runs a DataWeave script with module resolver callback. + * + *

This variant accepts a {@link NativeCallbacks.ResolveModuleCallback} to resolve + * external modules during script execution. The resolver is installed before script + * execution and remains active for the lifetime of the process.

+ * + * @param thread GraalVM isolate thread + * @param script DataWeave script source (C string) + * @param inputsJson JSON string of inputs (C string) + * @param resolverCallback Callback for resolving external modules + * @return JSON result or error message (unmanaged C string, must be freed) + */ + @CEntryPoint(name = "run_script_with_resolver") + public static CCharPointer runScriptWithResolver( + IsolateThread thread, + CCharPointer script, + CCharPointer inputsJson, + NativeCallbacks.ResolveModuleCallback resolverCallback) { + + try { + // Install resolver (idempotent if already set) + ScriptRuntime.setResolver(resolverCallback); + + // Delegate to existing run logic + String dwScript = CTypeConversion.toJavaString(script); + String inputs = CTypeConversion.toJavaString(inputsJson); + + ScriptRuntime runtime = ScriptRuntime.getInstance(); + String result = runtime.run(dwScript, inputs); + return toUnmanagedCString(result); + } catch (Exception e) { + return toUnmanagedCString("{\"success\":false,\"error\":\"" + + escapeJsonString(e.getMessage()) + "\"}"); + } + } + + /** + * Runs a DataWeave script with streaming output and module resolver. + * + *

This variant combines streaming output via a write callback with external module + * resolution. The resolver is installed before script execution.

+ * + * @param thread GraalVM isolate thread + * @param script DataWeave script source (C string) + * @param inputsJson JSON-encoded inputs map (C string), may be null + * @param writeCallback function pointer invoked with each output chunk + * @param ctx opaque context pointer forwarded to callback + * @param resolverCallback Callback for resolving external modules + * @return an unmanaged C string with JSON metadata/error (must be freed) + * + *

NOTE: compiled/linked but intentionally NOT invoked from the Node binding's + * TypeScript layer. runStreaming() deliberately uses the resolver-less streaming entrypoint + * instead: streaming runs its native call on a background thread, and wiring a resolver + * callback there would call back into JS from a non-owning OS thread (undefined behavior / + * crash). Do not wire this up without first solving that cross-thread hazard.

+ */ + @CEntryPoint(name = "run_script_callback_with_resolver") + public static CCharPointer runScriptCallbackWithResolver( + IsolateThread thread, + CCharPointer script, + CCharPointer inputsJson, + NativeCallbacks.WriteCallback writeCallback, + PointerBase ctx, + NativeCallbacks.ResolveModuleCallback resolverCallback) { + + try { + // Install resolver + ScriptRuntime.setResolver(resolverCallback); + + // Delegate to existing streaming logic + String dwScript = CTypeConversion.toJavaString(script); + String inputs = inputsJson.isNull() ? null : CTypeConversion.toJavaString(inputsJson); + + ScriptRuntime runtime = ScriptRuntime.getInstance(); + StreamSession session = runtime.runStreaming(dwScript, inputs); + + if (session.isError()) { + return toUnmanagedCString("{\"success\":false,\"error\":\"" + + escapeJsonString(session.getError()) + "\"}"); + } + + try { + byte[] buf = new byte[CALLBACK_BUFFER_SIZE]; + CCharPointer nativeBuf = UnmanagedMemory.malloc(CALLBACK_BUFFER_SIZE); + try { + int n; + while ((n = session.read(buf, buf.length)) > 0) { + for (int i = 0; i < n; i++) { + nativeBuf.write(i, buf[i]); + } + int rc = writeCallback.invoke(ctx, nativeBuf, n); + if (rc != 0) { + return toUnmanagedCString("{\"success\":false,\"error\":\"" + + "Write callback returned error: " + rc + "\"}"); + } + } + } finally { + UnmanagedMemory.free(nativeBuf); + } + } catch (IOException e) { + return toUnmanagedCString("{\"success\":false,\"error\":\"" + + escapeJsonString(e.getMessage()) + "\"}"); + } finally { + session.closeStream(); + } + + return toUnmanagedCString("{\"success\":true" + + ",\"mimeType\":\"" + session.getMimeType() + "\"" + + ",\"charset\":\"" + session.getCharset() + "\"" + + ",\"binary\":" + session.isBinary() + + "}"); + } catch (Exception e) { + return toUnmanagedCString("{\"success\":false,\"error\":\"" + + escapeJsonString(e.getMessage()) + "\"}"); + } + } + + /** + * Runs a DataWeave script with streaming input/output and module resolver. + * + *

This variant combines streaming input via read callback, streaming output via write + * callback, and external module resolution. The resolver is installed before script execution.

+ * + * @param thread GraalVM isolate thread + * @param script DataWeave script source (C string) + * @param inputsJson JSON-encoded inputs map (C string), may be null + * @param inputName the binding name for the callback-supplied input (C string) + * @param inputMimeType the MIME type of the callback-supplied input (C string) + * @param inputCharset the charset of the callback-supplied input (C string), may be null + * @param readCallback function pointer invoked to read input chunks + * @param writeCallback function pointer invoked with output chunks + * @param ctx opaque context pointer forwarded to callbacks + * @param resolverCallback Callback for resolving external modules + * @return an unmanaged C string with JSON metadata/error (must be freed) + * + *

NOTE: compiled/linked but intentionally NOT invoked from the Node binding's + * TypeScript layer. runTransform() deliberately uses the resolver-less transform entrypoint + * instead: transform runs its native call on a background thread, and wiring a resolver + * callback there would call back into JS from a non-owning OS thread (undefined behavior / + * crash). Do not wire this up without first solving that cross-thread hazard.

+ */ + @CEntryPoint(name = "run_script_input_output_callback_with_resolver") + public static CCharPointer runScriptInputOutputCallbackWithResolver( + IsolateThread thread, + CCharPointer script, + CCharPointer inputsJson, + CCharPointer inputName, + CCharPointer inputMimeType, + CCharPointer inputCharset, + NativeCallbacks.ReadCallback readCallback, + NativeCallbacks.WriteCallback writeCallback, + PointerBase ctx, + NativeCallbacks.ResolveModuleCallback resolverCallback) { + + try { + // Install resolver + ScriptRuntime.setResolver(resolverCallback); + + // Delegate to existing streaming I/O logic + String dwScript = CTypeConversion.toJavaString(script); + String inputs = inputsJson.isNull() ? null : CTypeConversion.toJavaString(inputsJson); + String inName = CTypeConversion.toJavaString(inputName); + String inMime = CTypeConversion.toJavaString(inputMimeType); + String inCharset = inputCharset.isNull() ? null : CTypeConversion.toJavaString(inputCharset); + + // Create a piped input stream session for the callback-supplied input + InputStreamSession inputSession = new InputStreamSession(inMime, inCharset); + long inputHandle = inputSession.register(); + + // Merge the stream handle into the inputs JSON + String streamEntry = "{\"streamHandle\":\"" + inputHandle + "\",\"mimeType\":\"" + inMime + "\"" + + (inCharset != null ? ",\"charset\":\"" + inCharset + "\"" : "") + "}"; + String mergedInputs = mergeInputEntry(inputs, inName, streamEntry); + + // Start background thread for reading input + final long readCallbackAddr = readCallback.rawValue(); + final long ctxAddr = ctx.rawValue(); + Thread feeder = new Thread(new InputCallbackFeeder( + readCallbackAddr, ctxAddr, inputSession), "dw-input-callback-feeder"); + feeder.setDaemon(true); + feeder.start(); + + // Execute the script and stream output via the writeCallback + ScriptRuntime runtime = ScriptRuntime.getInstance(); + StreamSession session = runtime.runStreaming(dwScript, mergedInputs); + + if (session.isError()) { + cleanupFeeder(feeder, inputHandle); + return toUnmanagedCString("{\"success\":false,\"error\":\"" + + escapeJsonString(session.getError()) + "\"}"); + } + + try { + byte[] buf = new byte[CALLBACK_BUFFER_SIZE]; + CCharPointer writeBuf = UnmanagedMemory.malloc(CALLBACK_BUFFER_SIZE); + try { + int n; + while ((n = session.read(buf, buf.length)) > 0) { + for (int i = 0; i < n; i++) { + writeBuf.write(i, buf[i]); + } + int rc = writeCallback.invoke(ctx, writeBuf, n); + if (rc != 0) { + cleanupFeeder(feeder, inputHandle); + return toUnmanagedCString("{\"success\":false,\"error\":\"" + + "Write callback returned error: " + rc + "\"}"); + } + } + } finally { + UnmanagedMemory.free(writeBuf); + } + } catch (IOException e) { + cleanupFeeder(feeder, inputHandle); + return toUnmanagedCString("{\"success\":false,\"error\":\"" + + escapeJsonString(e.getMessage()) + "\"}"); + } finally { + session.closeStream(); + } + + cleanupFeeder(feeder, inputHandle); + + return toUnmanagedCString("{\"success\":true" + + ",\"mimeType\":\"" + session.getMimeType() + "\"" + + ",\"charset\":\"" + session.getCharset() + "\"" + + ",\"binary\":" + session.isBinary() + + "}"); + } catch (Exception e) { + return toUnmanagedCString("{\"success\":false,\"error\":\"" + + escapeJsonString(e.getMessage()) + "\"}"); + } + } + } diff --git a/native-lib/src/main/java/org/mule/weave/lib/ScriptRuntime.java b/native-lib/src/main/java/org/mule/weave/lib/ScriptRuntime.java index 9cb4007e..3371127a 100644 --- a/native-lib/src/main/java/org/mule/weave/lib/ScriptRuntime.java +++ b/native-lib/src/main/java/org/mule/weave/lib/ScriptRuntime.java @@ -4,9 +4,13 @@ import org.mule.weave.v2.runtime.BindingValue; import org.mule.weave.v2.runtime.DataWeaveResult; import org.mule.weave.v2.runtime.ScriptingBindings; +import org.mule.weave.v2.runtime.api.DWModuleComponentsFactory; import org.mule.weave.v2.runtime.api.DWResult; import org.mule.weave.v2.runtime.api.DWScript; import org.mule.weave.v2.runtime.api.DWScriptingEngine; +import org.mule.weave.v2.sdk.ClassLoaderWeaveResourceResolver; +import org.mule.weave.v2.sdk.CompositeWeaveResourceResolver; +import org.mule.weave.v2.sdk.WeaveResourceResolver; import scala.Option; import scala.Tuple2; import scala.collection.immutable.Map; @@ -28,6 +32,9 @@ public class ScriptRuntime { private static final ScriptRuntime INSTANCE = new ScriptRuntime(); + // Static field for callback resolver, volatile for thread-safe double-checked locking + private static volatile CallbackWeaveResourceResolver resolver = null; + /** * Returns the singleton instance. * @@ -37,10 +44,71 @@ public static ScriptRuntime getInstance() { return INSTANCE; } - private DWScriptingEngine engine; + /** + * Sets the module resolver callback and rebuilds the engine. + * Can only be called once per process (engine is a singleton). + * Thread-safe but should be called early in application lifecycle before script execution. + * + *

IMPORTANT: The callback function must be thread-safe if using + * GraalVM's threadsafe function pointers, as it may be invoked from multiple threads + * during concurrent module resolution.

+ * + * @param callback Thread-safe function pointer for resolving modules + */ + public static synchronized void setResolver(NativeCallbacks.ResolveModuleCallback callback) { + if (resolver != null) { + System.err.println("WARNING: Module resolver already set for this process. " + + "Only one resolver configuration is supported. Ignoring new resolver."); + return; + } + + if (callback.isNull()) { + System.err.println("WARNING: Attempted to set null resolver, ignoring."); + return; + } + + resolver = new CallbackWeaveResourceResolver(callback); + + // Rebuild engine with composite resolver (built-ins + callback) + synchronized (INSTANCE) { + INSTANCE.engine = DWScriptingEngine.builder() + .withDWModuleComponentsFactory(createModuleComponentsFactory()) + .build(); + } + } + + /** + * Creates composite resolver: ClassLoader (built-ins) + Callback (user modules). + * If no callback resolver is set, returns ClassLoader only. + */ + private static WeaveResourceResolver compositeResolver() { + WeaveResourceResolver classLoaderResolver = ClassLoaderWeaveResourceResolver.apply(); + + CallbackWeaveResourceResolver currentResolver = resolver; + if (currentResolver == null) { + return classLoaderResolver; + } + + return CompositeWeaveResourceResolver.apply( + classLoaderResolver, // Try built-ins first + currentResolver // Then callback for user modules + ); + } + + private static DWModuleComponentsFactory createModuleComponentsFactory() { + return DWModuleComponentsFactory.createSimpleDWModuleComponentsFactoryBuilder() + .withWeaveResourceResolver(compositeResolver()) + .build(); + } + + // Instance field for the scripting engine, access synchronized in setResolver + private volatile DWScriptingEngine engine; private ScriptRuntime() { - engine = DWScriptingEngine.builder().build(); + // Initialize with ClassLoader-only resolver (no callback yet) + engine = DWScriptingEngine.builder() + .withDWModuleComponentsFactory(createModuleComponentsFactory()) + .build(); } /**