vm: support using the default loader to handle dynamic import()

This patch adds support for using `vm.constants.USE_MAIN_CONTEXT_DEFAULT_LOADER` as `importModuleDynamically` in all APIs that take the option except `vm.SourceTextModule`. This allows users to have a shortcut to support dynamic import() in the compiled code without missing the compilation cache if they don't need customization of the loading process. We emit an experimental warning when the `import()` is actually handled by the default loader through this option instead of requiring `--experimental-vm-modules`. In addition this refactors the documentation for `importModuleDynamically` and adds a dedicated section for it with examples. `vm.SourceTextModule` is not supported in this patch because it needs additional refactoring to handle `initializeImportMeta`, which can be done in a follow-up. PR-URL: #51244Fixes: #51154 Reviewed-By: Antoine du Hamel <[email protected]> Reviewed-By: Chengzhong Wu <[email protected]>

Author: joyeecheung

doc/api/vm.md

@@ -294,7 +294,6 @@ require('url'); // eslint-disable-line no-restricted-modules
 internalBinding('module_wrap');
 require('internal/modules/cjs/loader');
 require('internal/modules/esm/utils');
-require('internal/vm/module');
 
 // Needed to refresh the time origin.
 require('internal/perf/utils');

lib/internal/bootstrap/switches/is_main_thread.js

@@ -52,7 +52,6 @@ const{
  SafeMap,
  SafeWeakMap,
  String,
- Symbol,
  StringPrototypeCharAt,
  StringPrototypeCharCodeAt,
  StringPrototypeEndsWith,
@@ -114,7 +113,6 @@ const{
  initializeCjsConditions,
  loadBuiltinModule,
  makeRequireFunction,
- normalizeReferrerURL,
  stripBOM,
  toRealPath,
 }=require('internal/modules/helpers');
@@ -125,12 +123,10 @@ const policy = getLazy(
 );
 constshouldReportRequiredModules=getLazy(()=>process.env.WATCH_REPORT_DEPENDENCIES);
 
-constgetCascadedLoader=getLazy(
-()=>require('internal/process/esm_loader').esmLoader,
-);
-
 constpermission=require('internal/process/permission');
-
+const{
+ vm_dynamic_import_default_internal,
+}=internalBinding('symbols');
 // Whether any user-provided CJS modules had been loaded (executed).
 // Used for internal assertions.
 lethasLoadedAnyUserCJSModule=false;
@@ -1258,12 +1254,8 @@ let hasPausedEntry = false;
  * @param{object} codeCache The SEA code cache
  */
 functionwrapSafe(filename,content,cjsModuleInstance,codeCache){
-consthostDefinedOptionId=Symbol(`cjs:${filename}`);
-asyncfunctionimportModuleDynamically(specifier,_,importAttributes){
-constcascadedLoader=getCascadedLoader();
-returncascadedLoader.import(specifier,normalizeReferrerURL(filename),
-importAttributes);
-}
+consthostDefinedOptionId=vm_dynamic_import_default_internal;
+constimportModuleDynamically=vm_dynamic_import_default_internal;
 if(patched){
 constwrapped=Module.wrap(content);
 constscript=makeContextifyScript(

lib/internal/modules/cjs/loader.js

@@ -15,7 +15,6 @@ const{
  StringPrototypeReplaceAll,
  StringPrototypeSlice,
  StringPrototypeStartsWith,
- Symbol,
  SyntaxErrorPrototype,
 globalThis: { WebAssembly },
 }=primordials;
@@ -59,7 +58,9 @@ const{ModuleWrap } = moduleWrap;
 constasyncESM=require('internal/process/esm_loader');
 const{ emitWarningSync }=require('internal/process/warning');
 const{ internalCompileFunction }=require('internal/vm');
-
+const{
+ vm_dynamic_import_default_internal,
+}=internalBinding('symbols');
 // Lazy-loading to avoid circular dependencies.
 letgetSourceSync;
 /**
@@ -206,9 +207,8 @@ function enrichCJSError(err, content, filename){
  */
 functionloadCJSModule(module,source,url,filename){
 letcompiledWrapper;
-asyncfunctionimportModuleDynamically(specifier,_,importAttributes){
-returnasyncESM.esmLoader.import(specifier,url,importAttributes);
-}
+consthostDefinedOptionId=vm_dynamic_import_default_internal;
+constimportModuleDynamically=vm_dynamic_import_default_internal;
 try{
 compiledWrapper=internalCompileFunction(
 source,// code,
@@ -226,8 +226,8 @@ function loadCJSModule(module, source, url, filename){
 '__filename',
 '__dirname',
 ],
-Symbol(`cjs:${filename}`),// hostDefinedOptionsId
-importModuleDynamically,// importModuleDynamically
+hostDefinedOptionId,// hostDefinedOptionsId
+importModuleDynamically,// importModuleDynamically
 ).function;
 }catch(err){
 enrichCJSError(err,source,filename);

lib/internal/modules/esm/translators.js

@@ -4,7 +4,6 @@ const{
  ArrayIsArray,
  SafeSet,
  SafeWeakMap,
- Symbol,
  ObjectFreeze,
 }=primordials;
 
@@ -14,8 +13,10 @@ const{
 },
 }=internalBinding('util');
 const{
- default_host_defined_options,
+ vm_dynamic_import_default_internal,
+ vm_dynamic_import_main_context_default,
  vm_dynamic_import_missing_flag,
+ vm_dynamic_import_no_callback,
 }=internalBinding('symbols');
 
 const{
@@ -28,12 +29,19 @@ const{
  loadPreloadModules,
  initializeFrozenIntrinsics,
 }=require('internal/process/pre_execution');
-const{ getCWDURL }=require('internal/util');
+const{
+ emitExperimentalWarning,
+ getCWDURL,
+ getLazy,
+}=require('internal/util');
 const{
  setImportModuleDynamicallyCallback,
  setInitializeImportMetaObjectCallback,
 }=internalBinding('module_wrap');
 constassert=require('internal/assert');
+const{
+ normalizeReferrerURL,
+}=require('internal/modules/helpers');
 
 letdefaultConditions;
 /**
@@ -145,8 +153,10 @@ const moduleRegistries = new SafeWeakMap();
  */
 functionregisterModule(referrer,registry){
 constidSymbol=referrer[host_defined_option_symbol];
-if(idSymbol===default_host_defined_options||
-idSymbol===vm_dynamic_import_missing_flag){
+if(idSymbol===vm_dynamic_import_no_callback||
+idSymbol===vm_dynamic_import_missing_flag||
+idSymbol===vm_dynamic_import_main_context_default||
+idSymbol===vm_dynamic_import_default_internal){
 // The referrer is compiled without custom callbacks, so there is
 // no registry to hold on to. We'll throw
 // ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING when a callback is
@@ -158,26 +168,6 @@ function registerModule(referrer, registry){
 moduleRegistries.set(idSymbol,registry);
 }
 
-/**
- * Registers the ModuleRegistry for dynamic import() calls with a realm
- * as the referrer. Similar to{@link registerModule}, but this function
- * generates a new id symbol instead of using the one from the referrer
- * object.
- * @param{globalThis} globalThis The globalThis object of the realm.
- * @param{ModuleRegistry} registry
- */
-functionregisterRealm(globalThis,registry){
-letidSymbol=globalThis[host_defined_option_symbol];
-// If the per-realm host-defined options is already registered, do nothing.
-if(idSymbol){
-return;
-}
-// Otherwise, register the per-realm host-defined options.
-idSymbol=Symbol('Realm globalThis');
-globalThis[host_defined_option_symbol]=idSymbol;
-moduleRegistries.set(idSymbol,registry);
-}
-
 /**
  * Defines the `import.meta` object for a given module.
  * @param{symbol} symbol - Reference to the module.
@@ -191,16 +181,44 @@ function initializeImportMetaObject(symbol, meta){
 }
 }
 }
+constgetCascadedLoader=getLazy(
+()=>require('internal/process/esm_loader').esmLoader,
+);
+
+/**
+ * Proxy the dynamic import to the default loader.
+ * @param{string} specifier - The module specifier string.
+ * @param{Record<string, string>} attributes - The import attributes object.
+ * @param{string|null|undefined} referrerName - name of the referrer.
+ * @returns{Promise<import('internal/modules/esm/loader.js').ModuleExports>} - The imported module object.
+ */
+functiondefaultImportModuleDynamically(specifier,attributes,referrerName){
+constparentURL=normalizeReferrerURL(referrerName);
+returngetCascadedLoader().import(specifier,parentURL,attributes);
+}
 
 /**
  * Asynchronously imports a module dynamically using a callback function. The native callback.
  * @param{symbol} referrerSymbol - Referrer symbol of the registered script, function, module, or contextified object.
  * @param{string} specifier - The module specifier string.
  * @param{Record<string, string>} attributes - The import attributes object.
+ * @param{string|null|undefined} referrerName - name of the referrer.
  * @returns{Promise<import('internal/modules/esm/loader.js').ModuleExports>} - The imported module object.
  * @throws{ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING} - If the callback function is missing.
  */
-asyncfunctionimportModuleDynamicallyCallback(referrerSymbol,specifier,attributes){
+asyncfunctionimportModuleDynamicallyCallback(referrerSymbol,specifier,attributes,referrerName){
+// For user-provided vm.constants.USE_MAIN_CONTEXT_DEFAULT_LOADER, emit the warning
+// and fall back to the default loader.
+if(referrerSymbol===vm_dynamic_import_main_context_default){
+emitExperimentalWarning('vm.USE_MAIN_CONTEXT_DEFAULT_LOADER');
+returndefaultImportModuleDynamically(specifier,attributes,referrerName);
+}
+// For script compiled internally that should use the default loader to handle dynamic
+// import, proxy the request to the default loader without the warning.
+if(referrerSymbol===vm_dynamic_import_default_internal){
+returndefaultImportModuleDynamically(specifier,attributes,referrerName);
+}
+
 if(moduleRegistries.has(referrerSymbol)){
 const{ importModuleDynamically, callbackReferrer }=moduleRegistries.get(referrerSymbol);
 if(importModuleDynamically!==undefined){
@@ -275,7 +293,6 @@ async function initializeHooks(){
 
 module.exports={
  registerModule,
- registerRealm,
  initializeESM,
  initializeHooks,
  getDefaultConditions,

lib/internal/modules/esm/utils.js

@@ -23,16 +23,19 @@ const{validateString } = require('internal/validators');
 constfs=require('fs');// Import all of `fs` so that it can be monkey-patched.
 constinternalFS=require('internal/fs/utils');
 constpath=require('path');
-const{ pathToFileURL, fileURLToPath,URL}=require('internal/url');
+const{ pathToFileURL, fileURLToPath }=require('internal/url');
+constassert=require('internal/assert');
 
 const{ getOptionValue }=require('internal/options');
 const{ setOwnProperty }=require('internal/util');
+const{ inspect }=require('internal/util/inspect');
 
 const{
 privateSymbols: {
  require_private_symbol,
 },
 }=internalBinding('util');
+const{canParse: URLCanParse}=internalBinding('url');
 
 letdebug=require('internal/util/debuglog').debuglog('module',(fn)=>{
 debug=fn;
@@ -288,14 +291,32 @@ function addBuiltinLibsToObject(object, dummyModuleName){
 }
 
 /**
- * If a referrer is an URL instance or absolute path, convert it into an URL string.
- * @param{string | URL} referrer
+ * Normalize the referrer name as a URL.
+ * If it's a string containing an absolute path or a URL it's normalized as
+ * a URL string.
+ * Otherwise it's returned as undefined.
+ * @param{string | null | undefined} referrerName
+ * @returns{string | undefined}
  */
-functionnormalizeReferrerURL(referrer){
-if(typeofreferrer==='string'&&path.isAbsolute(referrer)){
-returnpathToFileURL(referrer).href;
+functionnormalizeReferrerURL(referrerName){
+if(referrerName===null||referrerName===undefined){
+returnundefined;
 }
-returnnewURL(referrer).href;
+
+if(typeofreferrerName==='string'){
+if(path.isAbsolute(referrerName)){
+returnpathToFileURL(referrerName).href;
+}
+
+if(StringPrototypeStartsWith(referrerName,'file://')||
+URLCanParse(referrerName)){
+returnreferrerName;
+}
+
+returnundefined;
+}
+
+assert.fail('Unreachable code reached by '+inspect(referrerName));
 }
 
 module.exports={

lib/internal/modules/helpers.js

@@ -67,23 +67,31 @@ function prepareWorkerThreadExecution(){
 }
 
 functionprepareShadowRealmExecution(){
-const{ registerRealm }=require('internal/modules/esm/utils');
 // Patch the process object with legacy properties and normalizations.
 // Do not expand argv1 as it is not available in ShadowRealm.
 patchProcessObject(false);
 setupDebugEnv();
 
 // Disable custom loaders in ShadowRealm.
 setupUserModules(true);
-registerRealm(globalThis,{
-__proto__: null,
-importModuleDynamically: (specifier,_referrer,attributes)=>{
-// The handler for `ShadowRealm.prototype.importValue`.
-const{ esmLoader }=require('internal/process/esm_loader');
-// `parentURL` is not set in the case of a ShadowRealm top-level import.
-returnesmLoader.import(specifier,undefined,attributes);
+const{
+privateSymbols: {
+ host_defined_option_symbol,
 },
-});
+}=internalBinding('util');
+const{
+ vm_dynamic_import_default_internal,
+}=internalBinding('symbols');
+
+// For ShadowRealm.prototype.importValue(), the referrer name is
+// always null, so the native ImportModuleDynamically() callback would
+// always fallback to look up the host-defined option from the
+// global object using host_defined_option_symbol. Using
+// vm_dynamic_import_default_internal as the host-defined option
+// instructs the JS-land importModuleDynamicallyCallback() to
+// proxy the request to defaultImportModuleDynamically().
+globalThis[host_defined_option_symbol]=
+vm_dynamic_import_default_internal;
 }
 
 functionprepareExecution(options){

lib/internal/process/pre_execution.js

@@ -107,12 +107,10 @@ function extractSourceMapURLMagicComment(content){
 functionmaybeCacheSourceMap(filename,content,cjsModuleInstance,isGeneratedSource,sourceURL,sourceMapURL){
 constsourceMapsEnabled=getSourceMapsEnabled();
 if(!(process.env.NODE_V8_COVERAGE||sourceMapsEnabled))return;
-try{
-const{ normalizeReferrerURL }=require('internal/modules/helpers');
-filename=normalizeReferrerURL(filename);
-}catch(err){
+const{ normalizeReferrerURL }=require('internal/modules/helpers');
+filename=normalizeReferrerURL(filename);
+if(filename===undefined){
 // This is most likely an invalid filename in sourceURL of [eval]-wrapper.
-debug(err);
 return;
 }