GH-80789: Bundle ensurepip wheels at build time

.cirrus-DISABLED.yml

@@ -23,6 +23,9 @@ freebsd_task:
  pythoninfo_script:
  - cd build
  - make pythoninfo
+ # Download wheels for the venv step
+ bundle_ensurepip_script:
+ - cd build && make download-ensurepip-wheels
  test_script:
  - cd build
  # dtrace fails to build on FreeBSD - see gh-73263

.github/workflows/build.yml

@@ -435,6 +435,12 @@ jobs:
  - name: Remount sources writable for tests
  # some tests write to srcdir, lack of pyc files slows down testing
  run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
+ # Download wheels for the venv step
+ - name: Download ensurepip wheels
+ env:
+ SSL_CERT_DIR: /etc/ssl/certs
+ run: make download-ensurepip-wheels
+ working-directory: ${{env.CPYTHON_BUILDDIR }}
  - name: Setup directory envs for out-of-tree builds
  run: |
  echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV

.github/workflows/reusable-docs.yml

@@ -101,6 +101,9 @@ jobs:
  run: ./configure --with-pydebug
  - name: 'Build CPython'
  run: make -j4
+ # Download wheels for the venv step
+ - name: 'Download ensurepip wheels'
+ run: make download-ensurepip-wheels
  - name: 'Install build dependencies'
  run: make -C Doc/ PYTHON=../python venv
  # Use "xvfb-run" since some doctest tests open GUI windows

Doc/whatsnew/3.13.rst

@@ -907,6 +907,10 @@ Build Changes
 * Building CPython now requires a compiler with support for the C11 atomic
  library, GCC built-in atomic functions, or MSVC interlocked intrinsics.
 
+* Wheels for :mod:`ensurepip` are no longer bundled in the CPython source
+ tree. Distributors should bundle these as part of the build process by
+ running :file:`Tools/build/bundle_ensurepip_wheels.py`.
+
 
 C API Changes
 =============

Lib/ensurepip/__init__.py

@@ -11,8 +11,9 @@
 __all__ = ["version", "bootstrap"]
 _PACKAGE_NAMES = ('pip',)
 _PIP_VERSION = "23.2.1"
+_PIP_SHA_256 = "7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be"
 _PROJECTS = [
- ("pip", _PIP_VERSION, "py3"),
+ ("pip", _PIP_VERSION, _PIP_SHA_256),
 ]
 
 # Packages bundled in ensurepip._bundled have wheel_name set.
@@ -62,8 +63,8 @@ def _get_packages():
  return _PACKAGES
 
  packages ={}
- for name, version, py_tag in _PROJECTS:
- wheel_name = f"{name}-{version}-{py_tag}-none-any.whl"
+ for name, version, _checksum in _PROJECTS:
+ wheel_name = f"{name}-{version}-py3-none-any.whl"
  packages[name] = _Package(version, wheel_name, None)
  if _WHEEL_PKG_DIR:
  dir_packages = _find_packages(_WHEEL_PKG_DIR)

Lib/ensurepip/_bundled/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!README.rst

Lib/ensurepip/_bundled/README.rst

@@ -0,0 +1,10 @@
+Bundling ensurepip wheels
+=========================
+
+``ensurepip`` expects wheels for ``pip`` to be located in this directory.
+These need to be 'bundled' by each distributor of Python,
+ordinarily by running the ``Tools/build/bundle_ensurepip_wheels.py`` script.
+
+This will download the version of ``pip`` specified by the
+``ensurepip._PIP_VERSION`` variable to this directory,
+and verify it against the stored SHA-256 checksum.

Lib/test/test_ensurepip.py

@@ -76,6 +76,12 @@ def setUp(self):
  patched_os.path = os.path
  self.os_environ = patched_os.environ = os.environ.copy()
 
+ # ensure the pip wheel exists
+ pip_filename = os.path.join(test.support.STDLIB_DIR, 'ensurepip', '_bundled',
+ f'pip-{ensurepip._PIP_VERSION}-py3-none-any.whl')
+ if not os.path.isfile(pip_filename):
+ open(pip_filename, "wb").close()
+
 
 class TestBootstrap(EnsurepipMixin, unittest.TestCase):
 

Lib/test/test_tools/test_bundle_ensurepip_wheels.py

@@ -0,0 +1,100 @@
+importsys
+importtempfile
+importunittest
+importunittest.mock
+importurllib.request
+fromhashlibimportsha256
+fromioimportBytesIO
+frompathlibimportPath
+fromrandomimportrandbytes
+fromtestimportsupport, test_tools
+fromtest.supportimportcaptured_stderr
+
+importensurepip
+
+test_tools.skip_if_missing('build')
+withtest_tools.imports_under_tool('build'):
+importbundle_ensurepip_wheelsasbew
+
+
+# Disable fancy GitHub actions output during the tests
+@unittest.mock.patch.object(bew, 'GITHUB_ACTIONS', False)
+classTestBundle(unittest.TestCase):
+contents=randbytes(512)
+checksum=sha256(contents).hexdigest()
+projects= [('pip', '1.2.3', checksum)]
+pip_filename="pip-1.2.3-py3-none-any.whl"
+
+deftest__wheel_url(self):
+self.assertEqual(
+bew._wheel_url('pip', '1.2.3'),
+'https://files.pythonhosted.org/packages/py3/p/pip/pip-1.2.3-py3-none-any.whl',
+ )
+
+deftest__get_projects(self):
+self.assertListEqual(
+bew._get_projects(),
+ [('pip', ensurepip._PIP_VERSION, ensurepip._PIP_SHA_256)],
+ )
+
+deftest__is_valid_wheel(self):
+self.assertTrue(bew._is_valid_wheel(self.contents, checksum=self.checksum))
+
+deftest_cached_wheel(self):
+withtempfile.TemporaryDirectory() astmpdir:
+tmpdir=Path(tmpdir)
+ (tmpdir/self.pip_filename).write_bytes(self.contents)
+with (
+captured_stderr() asstderr,
+unittest.mock.patch.object(bew, 'WHEEL_DIR', tmpdir),
+unittest.mock.patch.object(bew, '_get_projects', lambda: self.projects),
+ ):
+exit_code=bew.download_wheels()
+self.assertEqual(exit_code, 0)
+stderr=stderr.getvalue()
+self.assertIn("A valid 'pip' wheel already exists!", stderr)
+
+deftest_invalid_checksum(self):
+classMockedHTTPSOpener:
+@staticmethod
+defopen(url, data, timeout):
+assert'pip'inurl
+assertdataisNone# HTTP GET
+# Intentionally corrupt the wheel:
+returnBytesIO(self.contents[:-1])
+
+with (
+captured_stderr() asstderr,
+unittest.mock.patch.object(urllib.request, '_opener', None),
+unittest.mock.patch.object(bew, '_get_projects', lambda: self.projects),
+ ):
+urllib.request.install_opener(MockedHTTPSOpener())
+exit_code=bew.download_wheels()
+self.assertEqual(exit_code, 1)
+stderr=stderr.getvalue()
+self.assertIn("Failed to validate checksum for", stderr)
+
+deftest_download_wheel(self):
+classMockedHTTPSOpener:
+@staticmethod
+defopen(url, data, timeout):
+assert'pip'inurl
+assertdataisNone# HTTP GET
+returnBytesIO(self.contents)
+
+withtempfile.TemporaryDirectory() astmpdir:
+tmpdir=Path(tmpdir)
+with (
+captured_stderr() asstderr,
+unittest.mock.patch.object(urllib.request, '_opener', None),
+unittest.mock.patch.object(bew, 'WHEEL_DIR', tmpdir),
+unittest.mock.patch.object(bew, '_get_projects', lambda: self.projects),
+ ):
+urllib.request.install_opener(MockedHTTPSOpener())
+exit_code=bew.download_wheels()
+self.assertEqual((tmpdir/self.pip_filename).read_bytes(), self.contents)
+self.assertEqual(exit_code, 0)
+stderr=stderr.getvalue()
+self.assertIn("Downloading 'https://files.pythonhosted.org/packages/py3/p/pip/pip-1.2.3-py3-none-any.whl'",
+stderr)
+self.assertIn("Writing 'pip-1.2.3-py3-none-any.whl' to disk", stderr)

Makefile.pre.in

@@ -897,6 +897,14 @@ python.html: $(srcdir)/Tools/wasm/python.html python.worker.js
 python.worker.js: $(srcdir)/Tools/wasm/python.worker.js
  @cp $(srcdir)/Tools/wasm/python.worker.js $@
 
+##########################################################################
+# Bundle wheels for ensurepip
+.PHONY: download-ensurepip-wheels
+download-ensurepip-wheels: $(BUILDPYTHON) platform
+ if test "x$(ENSUREPIP)" != "xno" then \
+ $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/bundle_ensurepip_wheels.py ; \
+ fi
+
 ##########################################################################
 # Build static libmpdec.a
 LIBMPDEC_CFLAGS=@LIBMPDEC_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED)

Misc/NEWS.d/next/Library/2019-04-11-22-21-28.bpo-36608.HFoazc.rst

@@ -0,0 +1,2 @@
+Replaced vendored ``pip`` wheels for :mod:`ensurepip` with a new bundler script,
+:file:`Tools/build/bundle_ensurepip_wheels.py`, to be run by distributors.

Tools/build/bundle_ensurepip_wheels.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+"""
+Download wheels for 'ensurepip' packages from PyPI.
+
+When GitHub Actions executes the script, output is formatted accordingly.
+https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-notice-message
+"""
+
+importimportlib.util
+importos
+importsys
+fromhashlibimportsha256
+frompathlibimportPath
+fromurllib.errorimportURLError
+fromurllib.requestimporturlopen
+
+HOST='https://files.pythonhosted.org'
+ENSURE_PIP_ROOT=Path(__file__, "..", "..", "..", "Lib", "ensurepip").resolve()
+WHEEL_DIR=ENSURE_PIP_ROOT/"_bundled"
+ENSURE_PIP_INIT=ENSURE_PIP_ROOT/"__init__.py"
+GITHUB_ACTIONS="GITHUB_ACTIONS"inos.environ
+
+
+defprint_notice(message: str) ->None:
+ifGITHUB_ACTIONS:
+print(f"::notice::{message}", end="\n\n")
+else:
+print(message, file=sys.stderr)
+
+
+defprint_error(message: str) ->None:
+ifGITHUB_ACTIONS:
+print(f"::error::{message}", end="\n\n")
+else:
+print(message, file=sys.stderr)
+
+
+defdownload_wheels() ->int:
+"""Download wheels into bundle if they are not there yet."""
+
+try:
+projects=_get_projects()
+except (AttributeError, TypeError):
+print_error(f"Could not find '_PROJECTS' in {ENSURE_PIP_INIT}.")
+return1
+
+errors=0
+forname, version, checksuminprojects:
+wheel_filename=f'{name}-{version}-py3-none-any.whl'
+wheel_path=WHEEL_DIR/wheel_filename
+ifwheel_path.exists():
+if_is_valid_wheel(wheel_path.read_bytes(), checksum=checksum):
+print_notice(f"A valid '{name}' wheel already exists!")
+continue
+else:
+print_error(f"An invalid '{name}' wheel exists.")
+os.remove(wheel_path)
+
+wheel_url=_wheel_url(name, version)
+print_notice(f"Downloading {wheel_url!r}")
+try:
+withurlopen(wheel_url) asresponse:
+whl=response.read()
+exceptURLErrorasexc:
+print_error(f"Failed to download {wheel_url!r}: {exc}")
+errors=1
+continue
+ifnot_is_valid_wheel(whl, checksum=checksum):
+print_error(f"Failed to validate checksum for {wheel_url!r}!")
+errors=1
+continue
+
+print_notice(f"Writing {wheel_filename!r} to disk")
+wheel_path.write_bytes(whl)
+
+returnerrors
+
+
+def_get_projects() ->list[tuple[str, str, str]]:
+spec=importlib.util.spec_from_file_location("ensurepip", ENSURE_PIP_INIT)
+ensurepip=importlib.util.module_from_spec(spec)
+spec.loader.exec_module(ensurepip)
+returnensurepip._PROJECTS
+
+
+def_wheel_url(name: str, version: str, /) ->str:
+# The following code was adapted from
+# https://warehouse.pypa.io/api-reference/integration-guide.html#predictable-urls
+#
+# We rely on the fact that pip is, as a matter of policy, portable code.
+# We can therefore guarantee that we'll always have the values:
+# build_tag = ""
+# python_tag = "py3"
+# abi_tag = "none"
+# platform_tag = "any"
+
+# https://www.python.org/dev/peps/pep-0491/#file-name-convention
+filename=f'{name}-{version}-py3-none-any.whl'
+returnf'{HOST}/packages/py3/{name[0]}/{name}/{filename}'
+
+
+def_is_valid_wheel(content: bytes, *, checksum: str) ->bool:
+returnchecksum==sha256(content, usedforsecurity=False).hexdigest()
+
+
+if__name__=='__main__':
+raiseSystemExit(download_wheels())