gh-127833: Docs: Add a grammar-snippet directive & replace productionlist

Doc/conf.py

@@ -27,6 +27,7 @@
  'c_annotations',
  'changes',
  'glossary_search',
+ 'grammar_snippet',
  'lexers',
  'pydoc_topics',
  'pyspecific',

Doc/reference/toplevel_components.rst

@@ -66,7 +66,9 @@ File input
 
 All input read from non-interactive files has the same form:
 
-.. productionlist:: python-grammar
+.. grammar-snippet::
+ :group: python-grammar
+
  file_input: (NEWLINE | `statement`)*
 
 This syntax is used in the following situations:
@@ -85,7 +87,9 @@ Interactive input
 
 Input in interactive mode is parsed using the following grammar:
 
-.. productionlist:: python-grammar
+.. grammar-snippet::
+ :group: python-grammar
+
  interactive_input: [`stmt_list`] NEWLINE | `compound_stmt` NEWLINE
 
 Note that a (top-level) compound statement must be followed by a blank line in

Doc/tools/extensions/grammar_snippet.py

@@ -0,0 +1,219 @@
+"""Support for documenting Python's grammar."""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from docutils import nodes
+from docutils.parsers.rst import directives
+from sphinx import addnodes
+from sphinx.domains.std import token_xrefs
+from sphinx.util.docutils import SphinxDirective
+from sphinx.util.nodes import make_id
+
+if TYPE_CHECKING:
+ from collections.abc import Sequence
+ from typing import Any
+
+ from docutils.nodes import Node
+ from sphinx.application import Sphinx
+ from sphinx.util.typing import ExtensionMetadata
+
+
+class snippet_string_node(nodes.inline): # noqa: N801 (snake_case is fine)
+ """Node for a string literal in a grammar snippet."""
+
+ def __init__(
+ self,
+ rawsource: str = '',
+ text: str = '',
+ *children: Node,
+ **attributes: Any,
+ ) -> None:
+ super().__init__(rawsource, text, *children, **attributes)
+ # Use the Pygments highlight class for `Literal.String.Other`
+ self['classes'].append('sx')
+
+
+class GrammarSnippetBase(SphinxDirective):
+ """Common functionality for GrammarSnippetDirective & CompatProductionList."""
+
+ # The option/argument handling is left to the individual classes.
+
+ def make_grammar_snippet(
+ self, options: dict[str, Any], content: Sequence[str]
+ ) -> list[nodes.paragraph]:
+ """Create a literal block from options & content."""
+
+ group_name = options['group']
+
+ # Docutils elements have a `rawsource` attribute that is supposed to be
+ # set to the original ReST source.
+ # Sphinx does the following with it:
+ # - if it's empty, set it to `self.astext()`
+ # - if it matches `self.astext()` when generating the output,
+ # apply syntax highlighting (which is based on the plain-text content
+ # and thus discards internal formatting, like references).
+ # To get around this, we set it to this non-empty string:
+ rawsource = 'You should not see this.'
+
+ literal = nodes.literal_block(
+ rawsource,
+ '',
+ classes=['highlight'],
+ )
+
+ grammar_re = re.compile(
+ r"""
+ (?P<rule_name>^[a-zA-Z0-9_]+) # identifier at start of line
+ (?=:) # ... followed by a colon
+ |
+ (?P<rule_ref>`[^\s`]+`) # identifier in backquotes
+ |
+ (?P<single_quoted>'[^']*') # string in 'quotes'
+ |
+ (?P<double_quoted>"[^"]*") # string in "quotes"
+ """,
+ re.VERBOSE,
+ )
+
+ for line in content:
+ last_pos = 0
+ for match in grammar_re.finditer(line):
+ # Handle text between matches
+ if match.start() > last_pos:
+ literal += nodes.Text(line[last_pos : match.start()])
+ last_pos = match.end()
+
+ # Handle matches
+ group_dict ={
+ name: content
+ for name, content in match.groupdict().items()
+ if content is not None
+ }
+ match group_dict:
+ case{'rule_name': name}:
+ literal += self.make_link_target_for_token(
+ group_name, name
+ )
+ case{'rule_ref': ref_text}:
+ literal += token_xrefs(ref_text, group_name)
+ case{'single_quoted': name} |{'double_quoted': name}:
+ literal += snippet_string_node('', name)
+ case _:
+ raise ValueError('unhandled match')
+ literal += nodes.Text(line[last_pos:] + '\n')
+
+ node = nodes.paragraph(
+ '',
+ '',
+ literal,
+ )
+
+ return [node]
+
+ def make_link_target_for_token(
+ self, group_name: str, name: str
+ ) -> addnodes.literal_strong:
+ """Return a literal node which is a link target for the given token."""
+ name_node = addnodes.literal_strong()
+
+ # Cargo-culted magic to make `name_node` a link target
+ # similar to Sphinx `production`.
+ # This needs to be the same as what Sphinx does
+ # to avoid breaking existing links.
+ domain = self.env.domains['std']
+ obj_name = f"{group_name}:{name}"
+ prefix = f'grammar-token-{group_name}'
+ node_id = make_id(self.env, self.state.document, prefix, name)
+ name_node['ids'].append(node_id)
+ self.state.document.note_implicit_target(name_node, name_node)
+ domain.note_object('token', obj_name, node_id, location=name_node)
+
+ text_node = nodes.Text(name)
+ name_node += text_node
+ return name_node
+
+
+class GrammarSnippetDirective(GrammarSnippetBase):
+ """Transform a grammar-snippet directive to a Sphinx literal_block
+
+ That is, turn something like:
+
+ .. grammar-snippet:: file
+ :group: python-grammar
+
+ file: (NEWLINE | statement)*
+
+ into something similar to Sphinx productionlist, but better suited
+ for our needs:
+ - Instead of `::=`, use a colon, as in `Grammar/python.gram`
+ - Show the listing almost as is, with no auto-aligment.
+ The only special character is the backtick, which marks tokens.
+
+ Unlike Sphinx's productionlist, this directive supports options.
+ The "group" must be given as a named option.
+ The content must be preceded by a blank line (like with most ReST
+ directives).
+ """
+
+ has_content = True
+ option_spec ={
+ 'group': directives.unchanged_required,
+ }
+
+ # We currently ignore arguments.
+ required_arguments = 0
+ optional_arguments = 1
+ final_argument_whitespace = True
+
+ def run(self) -> list[nodes.paragraph]:
+ return self.make_grammar_snippet(self.options, self.content)
+
+
+class CompatProductionList(GrammarSnippetBase):
+ """Create grammar snippets from reST productionlist syntax
+
+ This is intended to be a transitional directive, used while we switch
+ from productionlist to grammar-snippet.
+ It makes existing docs that use the ReST syntax look like grammar-snippet,
+ as much as possible.
+ """
+
+ has_content = False
+ required_arguments = 1
+ optional_arguments = 0
+ final_argument_whitespace = True
+ option_spec ={}
+
+ def run(self) -> list[nodes.paragraph]:
+ # The "content" of a productionlist is actually the first and only
+ # argument. The first line is the group; the rest is the content lines.
+ lines = self.arguments[0].splitlines()
+ group = lines[0].strip()
+ options ={'group': group}
+ # We assume there's a colon in each line; align on it.
+ align_column = max(line.index(':') for line in lines[1:]) + 1
+ content = []
+ for line in lines[1:]:
+ rule_name, _colon, text = line.partition(':')
+ rule_name = rule_name.strip()
+ if rule_name:
+ name_part = rule_name + ':'
+ else:
+ name_part = ''
+ content.append(f'{name_part:<{align_column}}{text}')
+ return self.make_grammar_snippet(options, content)
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+ app.add_directive('grammar-snippet', GrammarSnippetDirective)
+ app.add_directive_to_domain(
+ 'std', 'productionlist', CompatProductionList, override=True
+ )
+ return{
+ 'version': '1.0',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }