esm: detect ESM syntax in ambiguous JavaScript

benchmark/esm/detect-esm-syntax.js

@@ -0,0 +1,37 @@
+'use strict'
+
+// This benchmarks the cost of running `containsModuleSyntax` on a CommonJS module being imported.
+// We use the TypeScript fixture because it's a very large CommonJS file with no ESM syntax: the worst case.
+const common = require('../common.js');
+const tmpdir = require('../../test/common/tmpdir.js');
+const fixtures = require('../../test/common/fixtures.js');
+const scriptPath = fixtures.path('snapshot', 'typescript.js');
+const fs = require('node:fs');
+
+const bench = common.createBenchmark(main,{
+ type: ['with-module-syntax-detection', 'without-module-syntax-detection'],
+ n: [1e4],
+},{
+ flags: ['--experimental-detect-module'],
+});
+
+const benchmarkDirectory = tmpdir.fileURL('benchmark-detect-esm-syntax');
+const ambiguousURL = new URL('./typescript.js', benchmarkDirectory);
+const explicitURL = new URL('./typescript.cjs', benchmarkDirectory);
+
+async function main({n, type }){
+ tmpdir.refresh();
+
+ fs.mkdirSync(benchmarkDirectory,{recursive: true });
+ fs.cpSync(scriptPath, ambiguousURL);
+ fs.cpSync(scriptPath, explicitURL);
+
+ bench.start();
+
+ for (let i = 0; i < n; i++){
+ const url = type === 'with-module-syntax-detection' ? ambiguousURL : explicitURL;
+ await import(url);
+ }
+
+ bench.end(n);
+}

doc/api/cli.md

@@ -620,6 +620,32 @@ files with no extension will be treated as WebAssembly if they begin with the
 WebAssembly magic number (`\0asm`); otherwise they will be treated as ES module
 JavaScript.
 
+### `--experimental-detect-module`
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+> Stability: 1.0 - Early development
+
+Node.js will inspect the source code of ambiguous input to determine whether it
+contains ES module syntax; if such syntax is detected, the input will be treated
+as an ES module.
+
+Ambiguous input is defined as:
+
+* Files with a `.js` extension or no extension; and either no controlling
+ `package.json` file or one that lacks a `type` field; and
+ `--experimental-default-type` is not specified.
+* String input (`--eval` or STDIN) when neither `--input-type` nor
+ `--experimental-default-type` are specified.
+
+ES module syntax is defined as syntax that would throw when evaluated as
+CommonJS. This includes `import` and `export` statements and `import.meta`
+references. It does _not_ include `import()` expressions, which are valid in
+CommonJS.
+
 ### `--experimental-import-meta-resolve`
 
 <!-- YAML
@@ -2286,6 +2312,7 @@ Node.js options that are allowed are:
 * `--enable-source-maps`
 * `--experimental-abortcontroller`
 * `--experimental-default-type`
+* `--experimental-detect-module`
 * `--experimental-import-meta-resolve`
 * `--experimental-json-modules`
 * `--experimental-loader`

doc/api/esm.md

@@ -109,11 +109,21 @@ provides interoperability between them and its original module format,
 
 Node.js has two module systems: [CommonJS][] modules and ECMAScript modules.
 
-Authors can tell Node.js to use the ECMAScript modules loader via the `.mjs`
-file extension, the `package.json` [`"type"`][] field, the [`--input-type`][]
-flag, or the [`--experimental-default-type`][] flag. Outside of those cases,
-Node.js will use the CommonJS module loader. See [Determining module system][]
-for more details.
+Authors can tell Node.js to interpret JavaScript as an ES module via the `.mjs`
+file extension, the `package.json` [`"type"`][] field with a value `"module"`,
+the [`--input-type`][] flag with a value of `"module"`, or the
+[`--experimental-default-type`][] flag with a value of `"module"`. These are
+explicit markers of code being intended to run as an ES module.
+
+Inversely, authors can tell Node.js to interpret JavaScript as CommonJS via the
+`.cjs` file extension, the `package.json` [`"type"`][] field with a value
+`"commonjs"`, the [`--input-type`][] flag with a value of `"commonjs"`, or the
+[`--experimental-default-type`][] flag with a value of `"commonjs"`.
+
+When code lacks explicit markers for either module system, Node.js will inspect
+the source code of a module to look for ES module syntax. If such syntax is
+found, Node.js will run the code as an ES module; otherwise it will run the
+module as CommonJS. See [Determining module system][] for more details.
 
 <!-- Anchors to make sure old links find a target -->
 
@@ -1019,18 +1029,33 @@ _isImports_, _conditions_)
 > 1. Return _"commonjs"_.
 > 4. If _url_ ends in _".json"_, then
 > 1. Return _"json"_.
-> 5. Let _packageURL_ be the result of **LOOKUP\_PACKAGE\_SCOPE**(_url_).
-> 6. Let _pjson_ be the result of **READ\_PACKAGE\_JSON**(_packageURL_).
-> 7. If _pjson?.type_ exists and is _"module"_, then
-> 1. If _url_ ends in _".js"_ or has no file extension, then
-> 1. If `--experimental-wasm-modules` is enabled and the file at _url_
-> contains the header for a WebAssembly module, then
-> 1. Return _"wasm"_.
-> 2. Otherwise,
-> 1. Return _"module"_.
-> 2. Return **undefined**.
-> 8. Otherwise,
-> 1. Return **undefined**.
+> 5. If `--experimental-wasm-modules` is enabled and _url_ ends in
+> _".wasm"_, then
+> 1. Return _"wasm"_.
+> 6. Let _packageURL_ be the result of **LOOKUP\_PACKAGE\_SCOPE**(_url_).
+> 7. Let _pjson_ be the result of **READ\_PACKAGE\_JSON**(_packageURL_).
+> 8. Let _packageType_ be **null**.
+> 9. If _pjson?.type_ is _"module"_ or _"commonjs"_, then
+> 1. Set _packageType_ to _pjson.type_.
+> 10. If _url_ ends in _".js"_, then
+> 1. If _packageType_ is not **null**, then
+> 1. Return _packageType_.
+> 2. If `--experimental-detect-module` is enabled and the source of
+> module contains static import or export syntax, then
+> 1. Return _"module"_.
+> 3. Return _"commonjs"_.
+> 11. If _url_ does not have any extension, then
+> 1. If _packageType_ is _"module"_ and `--experimental-wasm-modules` is
+> enabled and the file at _url_ contains the header for a WebAssembly
+> module, then
+> 1. Return _"wasm"_.
+> 2. If _packageType_ is not **null**, then
+> 1. Return _packageType_.
+> 3. If `--experimental-detect-module` is enabled and the source of
+> module contains static import or export syntax, then
+> 1. Return _"module"_.
+> 4. Return _"commonjs"_.
+> 12. Return **undefined** (will throw during load phase).
 
 **LOOKUP\_PACKAGE\_SCOPE**(_url_)
 

doc/api/modules.md

@@ -80,8 +80,10 @@ By default, Node.js will treat the following as CommonJS modules:
 * Files with a `.js` extension when the nearest parent `package.json` file
  contains a top-level field [`"type"`][] with a value of `"commonjs"`.
 
-* Files with a `.js` extension when the nearest parent `package.json` file
- doesn't contain a top-level field [`"type"`][]. Package authors should include
+* Files with a `.js` extension or without an extension, when the nearest parent
+ `package.json` file doesn't contain a top-level field [`"type"`][] or there is
+ no `package.json` in any parent folder; unless the file contains syntax that
+ errors unless it is evaluated as an ES module. Package authors should include
  the [`"type"`][] field, even in packages where all sources are CommonJS. Being
  explicit about the `type` of the package will make things easier for build
  tools and loaders to determine how the files in the package should be

doc/api/packages.md

@@ -69,6 +69,14 @@ expressions:
 * Strings passed in as an argument to `--eval`, or piped to `node` via `STDIN`,
  with the flag `--input-type=module`.
 
+* Code that contains syntax that only parses successfully as [ES modules][],
+ such as `import` or `export` statements or `import.meta`, when the code has no
+ explicit marker of how it should be interpreted. Explicit markers are `.mjs`
+ or `.cjs` extensions, `package.json` `"type"` fields with either `"module"` or
+ `"commonjs"` values, or `--input-type` or `--experimental-default-type` flags.
+ Dynamic `import()` expressions are supported in either CommonJS or ES modules
+ and would not cause a file to be treated as an ES module.
+
 Node.js will treat the following as [CommonJS][] when passed to `node` as the
 initial input, or when referenced by `import` statements or `import()`
 expressions:

lib/internal/modules/esm/get_format.js

@@ -18,9 +18,7 @@ const{
 
 const experimentalNetworkImports =
  getOptionValue('--experimental-network-imports');
-const defaultTypeFlag = getOptionValue('--experimental-default-type');
-// The next line is where we flip the default to ES modules someday.
-const defaultType = defaultTypeFlag === 'module' ? 'module' : 'commonjs'
+const{containsModuleSyntax } = internalBinding('contextify');
 const{getPackageType } = require('internal/modules/esm/resolve');
 const{fileURLToPath } = require('internal/url');
 const{ERR_UNKNOWN_FILE_EXTENSION } = require('internal/errors').codes;
@@ -85,42 +83,66 @@ function underNodeModules(url){
 
 /**
  * @param{URL} url
- * @param{{parentURL: string}} context
+ * @param{{parentURL: string; source?: Buffer}} context
  * @param{boolean} ignoreErrors
  * @returns{string}
  */
-function getFileProtocolModuleFormat(url, context, ignoreErrors){
+function getFileProtocolModuleFormat(url, context ={__proto__: null }, ignoreErrors){
+ const{source } = context;
  const ext = extname(url);
 
  if (ext === '.js'){
  const packageType = getPackageType(url);
  if (packageType !== 'none'){
  return packageType;
  }
+
  // The controlling `package.json` file has no `type` field.
- if (defaultType === 'module'){
- // An exception to the type flag making ESM the default everywhere is that package scopes under `node_modules`
- // should retain the assumption that a lack of a `type` field means CommonJS.
- return underNodeModules(url) ? 'commonjs' : 'module'
+ switch (getOptionValue('--experimental-default-type')){
+ case 'module':{// The user explicitly passed `--experimental-default-type=module`.
+ // An exception to the type flag making ESM the default everywhere is that package scopes under `node_modules`
+ // should retain the assumption that a lack of a `type` field means CommonJS.
+ return underNodeModules(url) ? 'commonjs' : 'module'
+ }
+ case 'commonjs':{// The user explicitly passed `--experimental-default-type=commonjs`.
+ return 'commonjs'
+ }
+ default:{// The user did not pass `--experimental-default-type`.
+ // `source` is undefined when this is called from `defaultResolve`;
+ // but this gets called again from `defaultLoad`/`defaultLoadSync`.
+ if (source && getOptionValue('--experimental-detect-module')){
+ return containsModuleSyntax(`${source}`, fileURLToPath(url)) ? 'module' : 'commonjs'
+ }
+ return 'commonjs'
+ }
  }
- return 'commonjs'
  }
 
  if (ext === ''){
  const packageType = getPackageType(url);
- if (defaultType === 'commonjs'){// Legacy behavior
- if (packageType === 'none' || packageType === 'commonjs'){
- return 'commonjs'
- } // Else packageType === 'module'
+ if (packageType === 'module'){
  return getFormatOfExtensionlessFile(url);
- } // Else defaultType === 'module'
- if (underNodeModules(url)){// Exception for package scopes under `node_modules`
- return packageType === 'module' ? getFormatOfExtensionlessFile(url) : 'commonjs'
  }
- if (packageType === 'none' || packageType === 'module'){
- return getFormatOfExtensionlessFile(url);
- } // Else packageType === 'commonjs'
- return 'commonjs'
+ if (packageType !== 'none'){
+ return packageType; // 'commonjs' or future package types
+ }
+
+ // The controlling `package.json` file has no `type` field.
+ switch (getOptionValue('--experimental-default-type')){
+ case 'module':{// The user explicitly passed `--experimental-default-type=module`.
+ return underNodeModules(url) ? 'commonjs' : getFormatOfExtensionlessFile(url);
+ }
+ case 'commonjs':{// The user explicitly passed `--experimental-default-type=commonjs`.
+ return 'commonjs'
+ }
+ default:{// The user did not pass `--experimental-default-type`.
+ if (source && getOptionValue('--experimental-detect-module') &&
+ getFormatOfExtensionlessFile(url) === 'module'){
+ return containsModuleSyntax(`${source}`, fileURLToPath(url)) ? 'module' : 'commonjs'
+ }
+ return 'commonjs'
+ }
+ }
  }
 
  const format = extensionFormatMap[ext];

lib/internal/modules/esm/load.js

@@ -33,7 +33,7 @@ const DATA_URL_PATTERN = /^[^/]+\/[^,;]+(?:[^,]*?)(;base64)?,([\s\S]*)$/;
 /**
  * @param{URL} url URL to the module
  * @param{ESModuleContext} context used to decorate error messages
- * @returns{{responseURL: string, source: string | BufferView }}
+ * @returns{Promise<{responseURL: string, source: string | BufferView }>}
  */
 async function getSource(url, context){
  const{protocol, href } = url;
@@ -105,7 +105,7 @@ function getSourceSync(url, context){
  * @param{LoadContext} context
  * @returns{LoadReturn}
  */
-async function defaultLoad(url, context = kEmptyObject){
+async function defaultLoad(url, context = {__proto__: null }){
  let responseURL = url;
  let{
  importAttributes,
@@ -127,19 +127,24 @@ async function defaultLoad(url, context = kEmptyObject){
 
  throwIfUnsupportedURLScheme(urlInstance, experimentalNetworkImports);
 
- format ??= await defaultGetFormat(urlInstance, context);
-
- validateAttributes(url, format, importAttributes);
-
- if (
- format === 'builtin' ||
- format === 'commonjs'
- ){
+ if (urlInstance.protocol === 'node:'){
  source = null;
  } else if (source == null){
  ({responseURL, source } = await getSource(urlInstance, context));
+ context.source = source;
+ }
+
+ if (format == null || format === 'commonjs'){
+ // Now that we have the source for the module, run `defaultGetFormat` again in case we detect ESM syntax.
+ format = await defaultGetFormat(urlInstance, context);
+ }
+
+ if (format === 'commonjs'){
+ source = null; // Let the CommonJS loader handle it (for now)
  }
 
+ validateAttributes(url, format, importAttributes);
+
  return{
  __proto__: null,
  format,
@@ -178,16 +183,17 @@ function defaultLoadSync(url, context = kEmptyObject){
 
  throwIfUnsupportedURLScheme(urlInstance, false);
 
- format ??= defaultGetFormat(urlInstance, context);
-
- validateAttributes(url, format, importAttributes);
-
- if (format === 'builtin'){
+ if (urlInstance.protocol === 'node:'){
  source = null;
  } else if (source == null){
  ({responseURL, source } = getSourceSync(urlInstance, context));
+ context.source = source;
  }
 
+ format ??= defaultGetFormat(urlInstance, context);
+
+ validateAttributes(url, format, importAttributes);
+
  return{
  __proto__: null,
  format,

lib/internal/modules/run_main.js

@@ -4,6 +4,7 @@ const{
  StringPrototypeEndsWith,
 } = primordials;
 
+const{containsModuleSyntax } = internalBinding('contextify');
 const{getOptionValue } = require('internal/options');
 const path = require('path');
 
@@ -70,7 +71,19 @@ function shouldUseESMLoader(mainPath){
  const{readPackageScope } = require('internal/modules/package_json_reader');
  const pkg = readPackageScope(mainPath);
  // No need to guard `pkg` as it can only be an object or `false`.
- return pkg.data?.type === 'module' || getOptionValue('--experimental-default-type') === 'module'
+ switch (pkg.data?.type){
+ case 'module':
+ return true;
+ case 'commonjs':
+ return false;
+ default:{// No package.json or no `type` field.
+ if (getOptionValue('--experimental-detect-module')){
+ // If the first argument of `containsModuleSyntax` is undefined, it will read `mainPath` from the file system.
+ return containsModuleSyntax(undefined, mainPath);
+ }
+ return false;
+ }
+ }
 }
 
 /**