cpython#121277: Replace next versions in docs by the just-released version

release.py

@@ -209,6 +209,21 @@ def committed_at(self) -> datetime.datetime:
  int(proc.stdout.decode().strip()), tz=datetime.timezone.utc
  )
 
+ @property
+ def includes_docs(self) -> bool:
+ """True if docs should be included in the release"""
+ return self.is_final or self.is_release_candidate
+
+ @property
+ def doc_version(self) -> str:
+ """Text used for notes in docs like 'Added in x.y'"""
+ # - ignore levels (alpha/beta/rc are preparation for the full release)
+ # - use just X.Y for patch 0
+ if self.patch == 0:
+ return f"{self.major}.{self.minor}"
+ else:
+ return f"{self.major}.{self.minor}.{self.patch}"
+
 
 def error(*msgs: str) -> None:
  print("**ERROR**", file=sys.stderr)

run_release.py

@@ -31,6 +31,7 @@
 
 import release as release_mod
 import sbom
+import update_version_next
 from buildbotapi import BuildBotAPI, Builder
 from release import ReleaseShelf, Tag, Task
 
@@ -497,6 +498,13 @@ def bump_version(db: ReleaseShelf) -> None:
  )
 
 
+def bump_version_in_docs(db: ReleaseShelf) -> None:
+ update_version_next.main([db["release"].doc_version, str(db["git_repo"])])
+ subprocess.check_call(
+ ["git", "commit", "-a", "--amend", "--no-edit"], cwd=db["git_repo"]
+ )
+
+
 def create_tag(db: ReleaseShelf) -> None:
  with cd(db["git_repo"]):
  if not release_mod.make_tag(db["release"], sign_gpg=db["sign_gpg"]):
@@ -509,7 +517,7 @@ def create_tag(db: ReleaseShelf) -> None:
 def wait_for_source_and_docs_artifacts(db: ReleaseShelf) -> None:
  # Determine if we need to wait for docs or only source artifacts.
  release_tag = db["release"]
- should_wait_for_docs = release_tag.is_final or release_tag.is_release_candidate
+ should_wait_for_docs = release_tag.includes_docs
 
  # Create the directory so it's easier to place the artifacts there.
  release_path = Path(db["git_repo"] / str(release_tag))
@@ -549,6 +557,31 @@ def wait_for_source_and_docs_artifacts(db: ReleaseShelf) -> None:
  time.sleep(1)
 
 
+def check_doc_unreleased_version(db: ReleaseShelf) -> None:
+ print("Checking built docs for '(unreleased)'")
+ # This string is generated when a `versionadded:: next` directive is
+ # left in the docs, which means the `bump_version_in_docs` step
+ # didn't do its job.
+ # But, there could also be a false positive.
+ release_tag = db["release"]
+ docs_path = Path(db["git_repo"]) / str(release_tag) / "docs"
+ archive_path = docs_path / f"python-{release_tag}-docs-html.tar.bz2"
+ if release_tag.includes_docs:
+ assert archive_path.exists()
+ if archive_path.exists():
+ proc = subprocess.run(
+ [
+ "tar",
+ "-xjf",
+ archive_path,
+ '--to-command=! grep -Hn --label="$TAR_FILENAME" "[(]unreleased[)]"',
+ ],
+ )
+ if proc.returncode != 0:
+ if not ask_question("Are these `(unreleased)` strings in built docs OK?"):
+ raise AssertionError("`(unreleased)` strings found in docs")
+
+
 def sign_source_artifacts(db: ReleaseShelf) -> None:
  print("Signing tarballs with GPG")
  uid = os.environ.get("GPG_KEY_FOR_RELEASE")
@@ -1251,6 +1284,7 @@ def _api_key(api_key: str) -> str:
  Task(check_cpython_repo_is_clean, "Checking Git repository is clean"),
  Task(prepare_pydoc_topics, "Preparing pydoc topics"),
  Task(bump_version, "Bump version"),
+ Task(bump_version_in_docs, "Bump version in docs"),
  Task(check_cpython_repo_is_clean, "Checking Git repository is clean"),
  Task(run_autoconf, "Running autoconf"),
  Task(check_cpython_repo_is_clean, "Checking Git repository is clean"),
@@ -1270,6 +1304,7 @@ def _api_key(api_key: str) -> str:
  wait_for_source_and_docs_artifacts,
  "Wait for source and docs artifacts to build",
  ),
+ Task(check_doc_unreleased_version, "Check docs for `(unreleased)`"),
  Task(build_sbom_artifacts, "Building SBOM artifacts"),
  *([] if no_gpg else [Task(sign_source_artifacts, "Sign source artifacts")]),
  Task(upload_files_to_server, "Upload files to the PSF server"),

tests/test_release_tag.py

@@ -91,3 +91,25 @@ def test_tag_invalid() -> None:
  # Act / Assert
  with pytest.raises(SystemExit):
  release.Tag(tag_name)
+
+
+def test_tag_docs_attributes() -> None:
+ # Arrange
+ alpha = release.Tag("3.13.0a7")
+ beta = release.Tag("3.13.0b1")
+ rc = release.Tag("3.13.0rc3")
+ final_zero = release.Tag("3.13.0")
+ final_3 = release.Tag("3.13.3")
+
+ # Act / Assert
+ assert alpha.includes_docs is False
+ assert beta.includes_docs is False
+ assert rc.includes_docs is True
+ assert final_zero.includes_docs is True
+ assert final_3.includes_docs is True
+
+ assert alpha.doc_version == "3.13"
+ assert beta.doc_version == "3.13"
+ assert rc.doc_version == "3.13"
+ assert final_zero.doc_version == "3.13"
+ assert final_3.doc_version == "3.13.3"

tests/test_run_release.py

@@ -1,3 +1,7 @@
+import builtins
+import contextlib
+import io
+import tarfile
 from pathlib import Path
 from typing import cast
 
@@ -37,3 +41,86 @@ def test_check_magic_number() -> None:
  run_release.ReleaseException, match="Magic numbers in .* don't match"
  ):
  run_release.check_magic_number(cast(ReleaseShelf, db))
+
+
+def prepare_fake_docs(tmp_path: Path, content: str) -> None:
+ docs_path = tmp_path / "3.13.0rc1/docs"
+ docs_path.mkdir(parents=True)
+ tarball = tarfile.open(docs_path / "python-3.13.0rc1-docs-html.tar.bz2", "w:bz2")
+ with tarball:
+ tarinfo = tarfile.TarInfo("index.html")
+ tarinfo.size = len(content)
+ tarball.addfile(tarinfo, io.BytesIO(content.encode()))
+
+
+@contextlib.contextmanager
+def fake_answers(monkeypatch: pytest.MonkeyPatch, answers: list[str]) -> None:
+ """Monkey-patch input() to give the given answers. All must be consumed."""
+
+ answers_left = list(answers)
+
+ def fake_input(question):
+ print(question, "--", answers_left[0])
+ return answers_left.pop(0)
+
+ with monkeypatch.context() as ctx:
+ ctx.setattr(builtins, "input", fake_input)
+ yield
+ assert answers_left == []
+
+
+def test_check_doc_unreleased_version_no_file(tmp_path: Path) -> None:
+ db ={
+ "release": Tag("3.13.0rc1"),
+ "git_repo": str(tmp_path),
+ }
+ with pytest.raises(AssertionError):
+ # There should be a docs artefact available
+ run_release.check_doc_unreleased_version(cast(ReleaseShelf, db))
+
+
+def test_check_doc_unreleased_version_no_file_alpha(tmp_path: Path) -> None:
+ db ={
+ "release": Tag("3.13.0a1"),
+ "git_repo": str(tmp_path),
+ }
+ # No docs artefact needed for alphas
+ run_release.check_doc_unreleased_version(cast(ReleaseShelf, db))
+
+
+def test_check_doc_unreleased_version_ok(monkeypatch, tmp_path: Path) -> None:
+ prepare_fake_docs(
+ tmp_path,
+ "<div>New in 3.13</div>",
+ )
+ db ={
+ "release": Tag("3.13.0rc1"),
+ "git_repo": str(tmp_path),
+ }
+ run_release.check_doc_unreleased_version(cast(ReleaseShelf, db))
+
+
+def test_check_doc_unreleased_version_not_ok(monkeypatch, tmp_path: Path) -> None:
+ prepare_fake_docs(
+ tmp_path,
+ "<div>New in 3.13.0rc1 (unreleased)</div>",
+ )
+ db ={
+ "release": Tag("3.13.0rc1"),
+ "git_repo": str(tmp_path),
+ }
+ with fake_answers(monkeypatch, ["no"]), pytest.raises(AssertionError):
+ run_release.check_doc_unreleased_version(cast(ReleaseShelf, db))
+
+
+def test_check_doc_unreleased_version_waived(monkeypatch, tmp_path: Path) -> None:
+ prepare_fake_docs(
+ tmp_path,
+ "<div>New in 3.13.0rc1 (unreleased)</div>",
+ )
+ db ={
+ "release": Tag("3.13.0rc1"),
+ "git_repo": str(tmp_path),
+ }
+ with fake_answers(monkeypatch, ["yes"]):
+ run_release.check_doc_unreleased_version(cast(ReleaseShelf, db))

tests/test_update_version_next.py

@@ -0,0 +1,74 @@
+"""Tests for the update_version_next tool."""
+
+from pathlib import Path
+
+import update_version_next
+
+TO_CHANGE = """
+Directives to change
+--------------------
+
+Here, all occurences of NEXT (lowercase) should be changed:
+
+.. versionadded:: next
+
+.. versionchanged:: next
+
+.. deprecated:: next
+
+.. deprecated-removed:: next 4.0
+
+whitespace:
+
+.. versionchanged:: next
+
+.. versionchanged :: next
+
+ .. versionadded:: next
+
+arguments:
+
+.. versionadded:: next
+ Foo bar
+
+.. versionadded:: next as ``previousname``
+"""
+
+UNCHANGED = """
+Unchanged
+---------
+
+Here, the word "next" should NOT be changed:
+
+.. versionchanged:: NEXT
+
+..versionchanged:: NEXT
+
+... versionchanged:: next
+
+foo .. versionchanged:: next
+
+.. otherdirective:: next
+
+.. VERSIONCHANGED: next
+
+.. deprecated-removed: 3.0 next
+"""
+
+EXPECTED_CHANGED = TO_CHANGE.replace("next", "VER")
+
+
+def test_freeze_simple_script(tmp_path: Path) -> None:
+ p = tmp_path.joinpath
+
+ p("source.rst").write_text(TO_CHANGE + UNCHANGED)
+ p("subdir").mkdir()
+ p("subdir/change.rst").write_text(".. versionadded:: next")
+ p("subdir/keep.not-rst").write_text(".. versionadded:: next")
+ p("subdir/keep.rst").write_text("nothing to see here")
+ args = ["VER", str(tmp_path)]
+ update_version_next.main(args)
+ assert p("source.rst").read_text() == EXPECTED_CHANGED + UNCHANGED
+ assert p("subdir/change.rst").read_text() == ".. versionadded:: VER"
+ assert p("subdir/keep.not-rst").read_text() == ".. versionadded:: next"
+ assert p("subdir/keep.rst").read_text() == "nothing to see here"

update_version_next.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""
+Replace `.. versionchanged:: next` lines in docs files by the given version.
+
+Run this at release time to replace `next` with the just-released version
+in the sources.
+
+No backups are made; add/commit to Git before running the script.
+
+Applies to all the VersionChange directives. For deprecated-removed, only
+handle the first argument (deprecation version, not the removal version).
+
+"""
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+DIRECTIVE_RE = re.compile(
+ r"""
+ (?P<before>
+ \s*\.\.\s+
+ (version(added|changed|removed)|deprecated(-removed)?)
+ \s*::\s*
+ )
+ next
+ (?P<after>
+ .*
+ )
+ """,
+ re.VERBOSE | re.DOTALL,
+)
+
+parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+)
+parser.add_argument(
+ "version",
+ help='String to replace "next" with. Usually `x.y`, but can be anything.',
+)
+parser.add_argument(
+ "directory",
+ type=Path,
+ help="Directory to process",
+)
+parser.add_argument(
+ "--verbose",
+ "-v",
+ action="count",
+ default=0,
+ help="Increase verbosity. Can be repeated (`-vv`).",
+)
+
+
+def main(argv: list[str]) -> None:
+ args = parser.parse_args(argv)
+ version = args.version
+ if args.verbose:
+ print(
+ f'Updating "next" versions in{args.directory} to{version!r}',
+ file=sys.stderr,
+ )
+ for path in Path(args.directory).glob("**/*.rst"):
+ num_changed_lines = 0
+ lines = []
+ with open(path, encoding="utf-8") as file:
+ for lineno, line in enumerate(file, start=1):
+ try:
+ if match := DIRECTIVE_RE.fullmatch(line):
+ line = match["before"] + version + match["after"]
+ num_changed_lines += 1
+ lines.append(line)
+ except Exception as exc:
+ exc.add_note(f"processing line{path}:{lineno}")
+ raise
+ if num_changed_lines:
+ if args.verbose:
+ s = "" if num_changed_lines == 1 else "s"
+ print(
+ f"Updating file{path} ({num_changed_lines} change{s})",
+ file=sys.stderr,
+ )
+ with open(path, "w", encoding="utf-8") as file:
+ file.writelines(lines)
+ else:
+ if args.verbose > 1:
+ print(f"Unchanged file{path}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+ main(sys.argv[1:])