gh-135801: Add the module parameter to compile() etc

Doc/library/ast.rst

@@ -2205,10 +2205,10 @@ Async and await
 Apart from the node classes, the :mod:`ast` module defines these utility functions
 and classes for traversing abstract syntax trees:
 
-.. function:: parse(source, filename='<unknown>', mode='exec', *, type_comments=False, feature_version=None, optimize=-1)
+.. function:: parse(source, filename='<unknown>', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None)
 
  Parse the source into an AST node. Equivalent to ``compile(source,
- filename, mode, flags=FLAGS_VALUE, optimize=optimize)``,
+ filename, mode, flags=FLAGS_VALUE, optimize=optimize, module=module)``,
  where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0``
  and ``ast.PyCF_OPTIMIZED_AST`` otherwise.
 
@@ -2261,6 +2261,9 @@ and classes for traversing abstract syntax trees:
  The minimum supported version for ``feature_version`` is now ``(3, 7)``.
  The ``optimize`` argument was added.
 
+ .. versionadded:: next
+ Added the *module* parameter.
+
 
 .. function:: unparse(ast_obj)
 

Doc/library/functions.rst

@@ -292,7 +292,9 @@ are always available. They are listed here in alphabetical order.
  :func:`property`.
 
 
-.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1)
+.. function:: compile(source, filename, mode, flags=0, \
+ dont_inherit=False, optimize=-1, \
+ *, module=None)
 
  Compile the *source* into a code or AST object. Code objects can be executed
  by :func:`exec` or :func:`eval`. *source* can either be a normal string, a
@@ -334,6 +336,10 @@ are always available. They are listed here in alphabetical order.
  ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false)
  or ``2`` (docstrings are removed too).
 
+ The optional argument *module* specifies the module name.
+ It is needed to unambiguous :ref:`filter <warning-filter>` syntax warnings
+ by module name.
+
  This function raises :exc:`SyntaxError` if the compiled source is invalid,
  and :exc:`ValueError` if the source contains null bytes.
 
@@ -371,6 +377,9 @@ are always available. They are listed here in alphabetical order.
  ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable
  support for top-level ``await``, ``async for``, and ``async with``.
 
+ .. versionadded:: next
+ Added the *module* parameter.
+
 
 .. class:: complex(number=0, /)
  complex(string, /)

Doc/library/importlib.rst

@@ -459,7 +459,7 @@ ABC hierarchy::
  .. versionchanged:: 3.4
  Raises :exc:`ImportError` instead of :exc:`NotImplementedError`.
 
- .. staticmethod:: source_to_code(data, path='<string>')
+ .. staticmethod:: source_to_code(data, path='<string>', fullname=None)
 
  Create a code object from Python source.
 
@@ -471,11 +471,19 @@ ABC hierarchy::
  With the subsequent code object one can execute it in a module by
  running ``exec(code, module.__dict__)``.
 
+ The optional argument *fullname* specifies the module name.
+ It is needed to unambiguous :ref:`filter <warning-filter>` syntax
+ warnings by module name.
+
  .. versionadded:: 3.4
 
  .. versionchanged:: 3.5
  Made the method static.
 
+ .. versionadded:: next
+ Added the *fullname* parameter.
+
+
  .. method:: exec_module(module)
 
  Implementation of :meth:`Loader.exec_module`.

Doc/library/symtable.rst

@@ -21,11 +21,17 @@ tables.
 Generating Symbol Tables
 ------------------------
 
-.. function:: symtable(code, filename, compile_type)
+.. function:: symtable(code, filename, compile_type, *, module=None)
 
  Return the toplevel :class:`SymbolTable` for the Python source *code*.
  *filename* is the name of the file containing the code. *compile_type* is
  like the *mode* argument to :func:`compile`.
+ The optional argument *module* specifies the module name.
+ It is needed to unambiguous :ref:`filter <warning-filter>` syntax warnings
+ by module name.
+
+ .. versionadded:: next
+ Added the *module* parameter.
 
 
 Examining Symbol Tables

Doc/whatsnew/3.15.rst

@@ -307,6 +307,13 @@ Other language changes
  not only integers or floats, although this does not improve precision.
  (Contributed by Serhiy Storchaka in :gh:`67795`.)
 
+* Many functions related to compiling or parsing Python code, such as
+ :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`,
+ and :func:`importlib.abc.InspectLoader.source_to_code`, now allow to pass
+ the module name. It is needed to unambiguous :ref:`filter <warning-filter>`
+ syntax warnings by module name.
+ (Contributed by Serhiy Storchaka in :gh:`135801`.)
+
 
 New modules
 ===========

Include/internal/pycore_compile.h

@@ -32,7 +32,8 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile(
  PyObject *filename,
  PyCompilerFlags *flags,
  int optimize,
- struct _arena *arena);
+ struct _arena *arena,
+ PyObject *module);
 
 /* AST preprocessing */
 extern int _PyCompile_AstPreprocess(
@@ -41,7 +42,8 @@ extern int _PyCompile_AstPreprocess(
  PyCompilerFlags *flags,
  int optimize,
  struct _arena *arena,
- int syntax_check_only);
+ int syntax_check_only,
+ PyObject *module);
 
 extern int _PyAST_Preprocess(
  struct _mod *,
@@ -50,7 +52,8 @@ extern int _PyAST_Preprocess(
  int optimize,
  int ff_features,
  int syntax_check_only,
- int enable_warnings);
+ int enable_warnings,
+ PyObject *module);
 
 
 typedef struct{

Include/internal/pycore_parser.h

@@ -48,7 +48,8 @@ extern struct _mod* _PyParser_ASTFromString(
  PyObject* filename,
  int mode,
  PyCompilerFlags *flags,
- PyArena *arena);
+ PyArena *arena,
+ PyObject *module);
 
 extern struct _mod* _PyParser_ASTFromFile(
  FILE *fp,

Include/internal/pycore_pyerrors.h

@@ -123,7 +123,8 @@ extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception);
 extern PyObject* _PyErr_NoMemory(PyThreadState *tstate);
 
 extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset,
- int end_lineno, int end_col_offset);
+ int end_lineno, int end_col_offset,
+ PyObject *module);
 extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset,
  int end_lineno, int end_col_offset);
 

Include/internal/pycore_pythonrun.h

@@ -33,6 +33,12 @@ extern const char* _Py_SourceAsString(
  PyCompilerFlags *cf,
  PyObject **cmd_copy);
 
+extern PyObject * _Py_CompileStringObjectWithModule(
+ const char *str,
+ PyObject *filename, int start,
+ PyCompilerFlags *flags, int optimize,
+ PyObject *module);
+
 
 /* Stack size, in "pointers". This must be large enough, so
  * no two calls to check recursion depth are more than this far

Include/internal/pycore_symtable.h

@@ -188,7 +188,8 @@ extern struct symtable* _Py_SymtableStringObjectFlags(
  const char *str,
  PyObject *filename,
  int start,
- PyCompilerFlags *flags);
+ PyCompilerFlags *flags,
+ PyObject *module);
 
 int _PyFuture_FromAST(
  struct _mod * mod,

Lib/ast.py

@@ -24,7 +24,7 @@
 
 
 def parse(source, filename='<unknown>', mode='exec', *,
- type_comments=False, feature_version=None, optimize=-1):
+ type_comments=False, feature_version=None, optimize=-1, module=None):
  """
  Parse the source into an AST node.
  Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
@@ -44,7 +44,8 @@ def parse(source, filename='<unknown>', mode='exec', *,
  feature_version = minor
  # Else it should be an int giving the minor version for 3.x.
  return compile(source, filename, mode, flags,
- _feature_version=feature_version, optimize=optimize)
+ _feature_version=feature_version, optimize=optimize,
+ module=module)
 
 
 def literal_eval(node_or_string):

Lib/importlib/_bootstrap_external.py

@@ -819,13 +819,14 @@ def get_source(self, fullname):
  name=fullname) from exc
  return decode_source(source_bytes)
 
- def source_to_code(self, data, path, *, _optimize=-1):
+ def source_to_code(self, data, path, fullname=None, *, _optimize=-1):
  """Return the code object compiled from source.
 
  The 'data' argument can be any object type that compile() supports.
  """
  return _bootstrap._call_with_frames_removed(compile, data, path, 'exec',
- dont_inherit=True, optimize=_optimize)
+ dont_inherit=True, optimize=_optimize,
+ module=fullname)
 
  def get_code(self, fullname):
  """Concrete implementation of InspectLoader.get_code.
@@ -894,7 +895,7 @@ def get_code(self, fullname):
  source_path=source_path)
  if source_bytes is None:
  source_bytes = self.get_data(source_path)
- code_object = self.source_to_code(source_bytes, source_path)
+ code_object = self.source_to_code(source_bytes, source_path, fullname)
  _bootstrap._verbose_message('code object from{}', source_path)
  if (not sys.dont_write_bytecode and bytecode_path is not None and
  source_mtime is not None):
@@ -1186,7 +1187,7 @@ def get_source(self, fullname):
  return ''
 
  def get_code(self, fullname):
- return compile('', '<string>', 'exec', dont_inherit=True)
+ return compile('', '<string>', 'exec', dont_inherit=True, module=fullname)
 
  def create_module(self, spec):
  """Use default semantics for module creation."""

Lib/importlib/abc.py

@@ -108,7 +108,7 @@ def get_code(self, fullname):
  source = self.get_source(fullname)
  if source is None:
  return None
- return self.source_to_code(source)
+ return self.source_to_code(source, '<string>', fullname)
 
  @abc.abstractmethod
  def get_source(self, fullname):
@@ -120,12 +120,12 @@ def get_source(self, fullname):
  raise ImportError
 
  @staticmethod
- def source_to_code(data, path='<string>'):
+ def source_to_code(data, path='<string>', fullname=None):
  """Compile 'data' into a code object.
 
  The 'data' argument can be anything that compile() can handle. The'path'
  argument should be where the data was retrieved (when applicable)."""
- return compile(data, path, 'exec', dont_inherit=True)
+ return compile(data, path, 'exec', dont_inherit=True, module=fullname)
 
  exec_module = _bootstrap_external._LoaderBasics.exec_module
  load_module = _bootstrap_external._LoaderBasics.load_module
@@ -163,9 +163,8 @@ def get_code(self, fullname):
  try:
  path = self.get_filename(fullname)
  except ImportError:
- return self.source_to_code(source)
- else:
- return self.source_to_code(source, path)
+ path = '<string>'
+ return self.source_to_code(source, path, fullname)
 
 _register(
  ExecutionLoader,

Lib/modulefinder.py

@@ -334,7 +334,7 @@ def load_module(self, fqname, fp, pathname, file_info):
  self.msgout(2, "load_module ->", m)
  return m
  if type == _PY_SOURCE:
- co = compile(fp.read(), pathname, 'exec')
+ co = compile(fp.read(), pathname, 'exec', module=fqname)
  elif type == _PY_COMPILED:
  try:
  data = fp.read()

Lib/profiling/sampling/_sync_coordinator.py

@@ -182,7 +182,7 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None:
 
  try:
  # Compile and execute the script
- code = compile(source_code, script_path, 'exec')
+ code = compile(source_code, script_path, 'exec', module='__main__')
  exec(code,{'__name__': '__main__', '__file__': script_path})
  except SyntaxError as e:
  raise TargetError(f"Syntax error in script{script_path}:{e}") from e

Lib/profiling/tracing/__init__.py

@@ -185,7 +185,7 @@ def main():
  progname = args[0]
  sys.path.insert(0, os.path.dirname(progname))
  with io.open_code(progname) as fp:
- code = compile(fp.read(), progname, 'exec')
+ code = compile(fp.read(), progname, 'exec', module='__main__')
  spec = importlib.machinery.ModuleSpec(name='__main__', loader=None,
  origin=progname)
  module = importlib.util.module_from_spec(spec)

Lib/runpy.py

@@ -247,7 +247,7 @@ def _get_main_module_details(error=ImportError):
  sys.modules[main_name] = saved_main
 
 
-def _get_code_from_file(fname):
+def _get_code_from_file(fname, module):
  # Check for a compiled file first
  from pkgutil import read_code
  code_path = os.path.abspath(fname)
@@ -256,7 +256,7 @@ def _get_code_from_file(fname):
  if code is None:
  # That didn't work, so try it as normal source code
  with io.open_code(code_path) as f:
- code = compile(f.read(), fname, 'exec')
+ code = compile(f.read(), fname, 'exec', module=module)
  return code
 
 def run_path(path_name, init_globals=None, run_name=None):
@@ -283,7 +283,7 @@ def run_path(path_name, init_globals=None, run_name=None):
  if isinstance(importer, type(None)):
  # Not a valid sys.path entry, so run the code directly
  # execfile() doesn't help as we want to allow compiled files
- code = _get_code_from_file(path_name)
+ code = _get_code_from_file(path_name, run_name)
  return _run_module_code(code, init_globals, run_name,
  pkg_name=pkg_name, script_name=path_name)
  else:

Lib/symtable.py

@@ -17,13 +17,13 @@
 
 __all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"]
 
-def symtable(code, filename, compile_type):
+def symtable(code, filename, compile_type, *, module=None):
  """ Return the toplevel *SymbolTable* for the source code.
 
  *filename* is the name of the file with the code
  and *compile_type* is the *compile()* mode argument.
  """
- top = _symtable.symtable(code, filename, compile_type)
+ top = _symtable.symtable(code, filename, compile_type, module=module)
  return _newSymbolTable(top, filename)
 
 class SymbolTableFactory:

Lib/test/test_ast/test_ast.py

@@ -1083,6 +1083,16 @@ def test_filter_syntax_warnings_by_module(self):
  self.assertEqual(wm.filename, '<unknown>')
  self.assertIs(wm.category, SyntaxWarning)
 
+ with warnings.catch_warnings(record=True) as wlog:
+ warnings.simplefilter('error')
+ warnings.filterwarnings('always', module=r'package\.module\z')
+ warnings.filterwarnings('error', module=r'<unknown>')
+ ast.parse(source, filename, module='package.module')
+ self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10])
+ for wm in wlog:
+ self.assertEqual(wm.filename, filename)
+ self.assertIs(wm.category, SyntaxWarning)
+
 
 class CopyTests(unittest.TestCase):
  """Test copying and pickling AST nodes."""

Lib/test/test_builtin.py

@@ -1103,7 +1103,8 @@ def test_exec_filter_syntax_warnings_by_module(self):
 
  with warnings.catch_warnings(record=True) as wlog:
  warnings.simplefilter('error')
- warnings.filterwarnings('always', module=r'<string>\z')
+ warnings.filterwarnings('always', module=r'package.module\z')
+ warnings.filterwarnings('error', module=r'<string>')
  exec(source,{'__name__': 'package.module', '__file__': filename})
  self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21])
  for wm in wlog: