esm: add initialize hook, integrate with register

Follows @giltayar's proposed API: > `register` can pass any data it wants to the loader, which will be passed to the exported `initialize` function of the loader. Additionally, if the user of `register` wants to communicate with the loader, it can just create a `MessageChannel` and pass the port to the loader as data. The `register` API is now: ```ts interface Options{parentUrl?: string; data?: any; transferList?: any[]} function register(loader: string, parentUrl?: string): any; function register(loader: string, options?: Options): any; ``` This API is backwards compatible with the old one (new arguments are optional and at the end) and allows for passing data into the new `initialize` hook. If this hook returns data it is passed back to `register`: ```ts function initialize(data: any): Promise<any> ``` **NOTE**: Currently there is no mechanism for a loader to exchange ownership of something back to the caller. Refs: nodejs/loaders#147 PR-URL: #48842 Backport-PR-URL: #50669 Reviewed-By: Antoine du Hamel <[email protected]> Reviewed-By: Matteo Collina <[email protected]> Reviewed-By: Geoffrey Booth <[email protected]>

Author: izaakschroeder

doc/api/esm.md

@@ -684,6 +684,9 @@ of Node.js applications.
 <!-- YAML
 added: v8.8.0
 changes:
+ - version: REPLACEME
+ pr-url: https://github.com/nodejs/node/pull/48842
+ description: Added `initialize` hook to replace `globalPreload`.
  - version:
  - v18.6.0
  pr-url: https://github.com/nodejs/node/pull/42623
@@ -737,6 +740,69 @@ different [realm](https://tc39.es/ecma262/#realm). The hooks thread may be
 terminated by the main thread at any time, so do not depend on asynchronous
 operations to (like `console.log`) complete.
 
+#### `initialize()`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> The loaders API is being redesigned. This hook may disappear or its
+> signature may change. Do not rely on the API described below.
+
+* `data`{any} The data from `register(loader, import.meta.url,{data })`.
+* Returns:{any} The data to be returned to the caller of `register`.
+
+The `initialize` hook provides a way to define a custom function that runs
+in the loader's thread when the loader is initialized. Initialization happens
+when the loader is registered via [`register`][] or registered via the
+`--loader` command line option.
+
+This hook can send and receive data from a [`register`][] invocation, including
+ports and other transferrable objects. The return value of `initialize` must be
+either:
+
+* `undefined`,
+* something that can be posted as a message between threads (e.g. the input to
+ [`port.postMessage`][]),
+* a `Promise` resolving to one of the aforementioned values.
+
+Loader code:
+
+```js
+// In the below example this file is referenced as
+// '/path-to-my-loader.js'
+
+exportasyncfunctioninitialize({number, port }){
+port.postMessage(`increment: ${number +1}`);
+return'ok';
+}
+```
+
+Caller code:
+
+```js
+importassertfrom'node:assert';
+import{register } from'node:module';
+import{MessageChannel } from'node:worker_threads';
+
+// This example showcases how a message channel can be used to
+// communicate between the main (application) thread and the loader
+// running on the loaders thread, by sending `port2` to the loader.
+const{port1, port2 } =newMessageChannel();
+
+port1.on('message', (msg) =>{
+assert.strictEqual(msg, 'increment: 2');
+});
+
+constresult=register('/path-to-my-loader.js',{
+ parentURL:import.meta.url,
+ data:{number:1, port: port2 },
+ transferList: [port2],
+});
+
+assert.strictEqual(result, 'ok');
+```
+
 #### `resolve(specifier, context, nextResolve)`
 
 <!-- YAML
@@ -941,8 +1007,8 @@ changes:
  description: Add support for chaining globalPreload hooks.
 -->
 
-> The loaders API is being redesigned. This hook may disappear or its
-> signature may change. Do not rely on the API described below.
+> This hook will be removed in a future version. Use [`initialize`][] instead.
+> When a loader has an `initialize` export, `globalPreload` will be ignored.
 
 > In a previous version of this API, this hook was named
 > `getGlobalPreloadCode`.
@@ -1642,13 +1708,16 @@ success!
 [`import.meta.resolve`]: #importmetaresolvespecifier-parent
 [`import.meta.url`]: #importmetaurl
 [`import`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import
+[`initialize`]: #initialize
 [`module.createRequire()`]: module.md#modulecreaterequirefilename
 [`module.register()`]: module.md#moduleregister
 [`module.syncBuiltinESMExports()`]: module.md#modulesyncbuiltinesmexports
 [`package.json`]: packages.md#nodejs-packagejson-field-definitions
+[`port.postMessage`]: worker_threads.md#portpostmessagevalue-transferlist
 [`port.ref()`]: https://nodejs.org/dist/latest-v17.x/docs/api/worker_threads.html#portref
 [`port.unref()`]: https://nodejs.org/dist/latest-v17.x/docs/api/worker_threads.html#portunref
 [`process.dlopen`]: process.md#processdlopenmodule-filename-flags
+[`register`]: module.md#moduleregister
 [`string`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String
 [`util.TextDecoder`]: util.md#class-utiltextdecoder
 [cjs-module-lexer]: https://github.com/nodejs/cjs-module-lexer/tree/1.2.2

doc/api/module.md

@@ -173,6 +173,28 @@ globalPreload: http-to-https
 globalPreload: unpkg
 ```
 
+This function can also be used to pass data to the loader's [`initialize`][]
+hook; the data passed to the hook may include transferrable objects like ports.
+
+```mjs
+import{register } from'node:module';
+import{MessageChannel } from'node:worker_threads';
+
+// This example showcases how a message channel can be used to
+// communicate to the loader, by sending `port2` to the loader.
+const{port1, port2 } =newMessageChannel();
+
+port1.on('message', (msg) =>{
+console.log(msg);
+});
+
+register('./my-programmatic-loader.mjs',{
+ parentURL:import.meta.url,
+ data:{number:1, port: port2 },
+ transferList: [port2],
+});
+```
+
 ### `module.syncBuiltinESMExports()`
 
 <!-- YAML
@@ -358,6 +380,7 @@ returned object contains the following keys:
 [`--enable-source-maps`]: cli.md#--enable-source-maps
 [`NODE_V8_COVERAGE=dir`]: cli.md#node_v8_coveragedir
 [`SourceMap`]: #class-modulesourcemap
+[`initialize`]: esm.md#initialize
 [`module`]: modules.md#the-module-object
 [module wrapper]: modules.md#the-module-wrapper
 [source map include directives]: https://sourcemaps.info/spec.html#h.lmz475t4mvbx

lib/internal/modules/esm/hooks.js

@@ -2,6 +2,7 @@
 
 const{
  ArrayPrototypePush,
+ ArrayPrototypePushApply,
  FunctionPrototypeCall,
  Int32Array,
  ObjectAssign,
@@ -46,8 +47,10 @@ const{
  validateObject,
  validateString,
 }=require('internal/validators');
-
-const{ kEmptyObject }=require('internal/util');
+const{
+ emitExperimentalWarning,
+ kEmptyObject,
+}=require('internal/util');
 
 const{
  defaultResolve,
@@ -82,6 +85,7 @@ let importMetaInitializer;
 
 // [2] `validate...()`s throw the wrong error
 
+letglobalPreloadWarned=false;
 classHooks{
  #chains ={
 /**
@@ -126,31 +130,43 @@ class Hooks{
  * Import and register custom/user-defined module loader hook(s).
  * @param{string} urlOrSpecifier
  * @param{string} parentURL
+ * @param{any} [data] Arbitrary data to be passed from the custom
+ * loader (user-land) to the worker.
  */
-asyncregister(urlOrSpecifier,parentURL){
+asyncregister(urlOrSpecifier,parentURL,data){
 constmoduleLoader=require('internal/process/esm_loader').esmLoader;
 constkeyedExports=awaitmoduleLoader.import(
 urlOrSpecifier,
 parentURL,
 kEmptyObject,
 );
-this.addCustomLoader(urlOrSpecifier,keyedExports);
+returnthis.addCustomLoader(urlOrSpecifier,keyedExports,data);
 }
 
 /**
  * Collect custom/user-defined module loader hook(s).
  * After all hooks have been collected, the global preload hook(s) must be initialized.
  * @param{string} url Custom loader specifier
  * @param{Record<string, unknown>} exports
+ * @param{any} [data] Arbitrary data to be passed from the custom loader (user-land)
+ * to the worker.
+ * @returns{any} The result of the loader's `initialize` hook, if provided.
  */
-addCustomLoader(url,exports){
+addCustomLoader(url,exports,data){
 const{
  globalPreload,
+ initialize,
  resolve,
  load,
 }=pluckHooks(exports);
 
-if(globalPreload){
+if(globalPreload&&!initialize){
+if(globalPreloadWarned===false){
+globalPreloadWarned=true;
+emitExperimentalWarning(
+'`globalPreload` will be removed in a future version. Please use `initialize` instead.',
+);
+}
 ArrayPrototypePush(this.#chains.globalPreload,{__proto__: null,fn: globalPreload, url });
 }
 if(resolve){
@@ -161,6 +177,7 @@ class Hooks{
 constnext=this.#chains.load[this.#chains.load.length-1];
 ArrayPrototypePush(this.#chains.load,{__proto__: null,fn: load, url, next });
 }
+returninitialize?.(data);
 }
 
 /**
@@ -552,15 +569,30 @@ class HooksProxy{
 }
 }
 
-asyncmakeAsyncRequest(method, ...args){
+/**
+ * Invoke a remote method asynchronously.
+ * @param{string} method Method to invoke
+ * @param{any[]} [transferList] Objects in `args` to be transferred
+ * @param{any[]} args Arguments to pass to `method`
+ * @returns{Promise<any>}
+ */
+asyncmakeAsyncRequest(method,transferList, ...args){
 this.waitForWorker();
 
 MessageChannel??=require('internal/worker/io').MessageChannel;
 constasyncCommChannel=newMessageChannel();
 
 // Pass work to the worker.
-debug('post async message to worker',{ method, args });
-this.#worker.postMessage({ method, args,port: asyncCommChannel.port2},[asyncCommChannel.port2]);
+debug('post async message to worker',{ method, args, transferList });
+constfinalTransferList=[asyncCommChannel.port2];
+if(transferList){
+ArrayPrototypePushApply(finalTransferList,transferList);
+}
+this.#worker.postMessage({
+__proto__: null,
+ method, args,
+port: asyncCommChannel.port2,
+},finalTransferList);
 
 if(this.#numberOfPendingAsyncResponses++===0){
 // On the next lines, the main thread will await a response from the worker thread that might
@@ -592,12 +624,19 @@ class HooksProxy{
 returnbody;
 }
 
-makeSyncRequest(method, ...args){
+/**
+ * Invoke a remote method synchronously.
+ * @param{string} method Method to invoke
+ * @param{any[]} [transferList] Objects in `args` to be transferred
+ * @param{any[]} args Arguments to pass to `method`
+ * @returns{any}
+ */
+makeSyncRequest(method,transferList, ...args){
 this.waitForWorker();
 
 // Pass work to the worker.
-debug('post sync message to worker',{ method, args });
-this.#worker.postMessage({ method, args });
+debug('post sync message to worker',{ method, args, transferList});
+this.#worker.postMessage({__proto__: null,method, args },transferList);
 
 letresponse;
 do{
@@ -707,6 +746,7 @@ ObjectSetPrototypeOf(HooksProxy.prototype, null);
  */
 functionpluckHooks({
  globalPreload,
+ initialize,
  resolve,
  load,
 }){
@@ -722,6 +762,10 @@ function pluckHooks({
 acceptedHooks.load=load;
 }
 
+if(initialize){
+acceptedHooks.initialize=initialize;
+}
+
 returnacceptedHooks;
 }