diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index d3e842d9f31d01..9b3415fd2b5861 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -1,11 +1,11 @@ -trigger: ['main', '3.13', '3.12', '3.11', '3.10', '3.9', '3.8'] +trigger: ['main', '3.*'] jobs: - job: Prebuild displayName: Pre-build checks pool: - vmImage: ubuntu-22.04 + vmImage: ubuntu-24.04 steps: - template: ./prebuild-checks.yml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 64c85c1101e6e6..fb11c82bb96d9c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,13 +1,10 @@ { - "image": "ghcr.io/python/devcontainer:2024.09.25.11038928730", + "image": "ghcr.io/python/devcontainer:latest", "onCreateCommand": [ // Install common tooling. "dnf", "install", "-y", - "which", - "zsh", - "fish", // For umask fix below. "/usr/bin/setfacl" ], diff --git a/.devcontainer/wasi/devcontainer.json b/.devcontainer/wasi/devcontainer.json new file mode 100644 index 00000000000000..4266144ce47639 --- /dev/null +++ b/.devcontainer/wasi/devcontainer.json @@ -0,0 +1,73 @@ +{ + "image": "ghcr.io/python/wasicontainer:latest", + "onCreateCommand": [ + // Install common tooling. + "dnf", + "install", + "-y", + // For umask fix below. + "/usr/bin/setfacl" + ], + "updateContentCommand": { + // Using the shell for `nproc` usage. + "python": "python3 Tools/wasm/wasi build --quiet -- --with-pydebug -C" + }, + "postCreateCommand": { + // https://github.com/orgs/community/discussions/26026 + "umask fix: workspace": ["sudo", "setfacl", "-bnR", "."], + "umask fix: /tmp": ["sudo", "setfacl", "-bnR", "/tmp"] + }, + "customizations": { + "vscode": { + "extensions": [ + // Highlighting for Parser/Python.asdl. + "brettcannon.zephyr-asdl", + // Highlighting for configure.ac. + "maelvalais.autoconf", + // C auto-complete. + "ms-vscode.cpptools", + // Python auto-complete. + "ms-python.python" + ], + "settings": { + "C_Cpp.default.compilerPath": "/usr/bin/clang", + "C_Cpp.default.cStandard": "c11", + "C_Cpp.default.defines": [ + "CONFIG_64", + "Py_BUILD_CORE" + ], + "C_Cpp.default.includePath": [ + "${workspaceFolder}/*", + "${workspaceFolder}/Include/**" + ], + // https://github.com/microsoft/vscode-cpptools/issues/10732 + "C_Cpp.errorSquiggles": "disabled", + "editor.insertSpaces": true, + "editor.rulers": [ + 80 + ], + "editor.tabSize": 4, + "editor.trimAutoWhitespace": true, + "files.associations": { + "*.h": "c" + }, + "files.encoding": "utf8", + "files.eol": "\n", + "files.insertFinalNewline": true, + "files.trimTrailingWhitespace": true, + "python.analysis.diagnosticSeverityOverrides": { + // Complains about shadowing the stdlib w/ the stdlib. + "reportShadowedImports": "none", + // Doesn't like _frozen_importlib. + "reportMissingImports": "none" + }, + "python.analysis.extraPaths": [ + "Lib" + ], + "[restructuredtext]": { + "editor.tabSize": 3 + } + } + } + } +} diff --git a/.editorconfig b/.editorconfig index a6187d64f3ce46..25bc5935258bd1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,15 +1,15 @@ root = true -[*.{py,c,cpp,h,js,rst,md,yml}] +[*.{py,c,cpp,h,js,rst,md,yml,yaml,gram}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space -[*.{py,c,cpp,h}] +[*.{py,c,cpp,h,gram}] indent_size = 4 [*.rst] indent_size = 3 -[*.{js,yml}] +[*.{js,yml,yaml}] indent_size = 2 diff --git a/.gitattributes b/.gitattributes index 2f5a030981fb94..14c1eced0cfed4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,6 +10,7 @@ *.ico binary *.jpg binary *.pck binary +*.pdf binary *.png binary *.psd binary *.tar binary @@ -67,6 +68,8 @@ PCbuild/readme.txt dos **/clinic/*.cpp.h generated **/clinic/*.h.h generated *_db.h generated +Doc/_static/tachyon-example-*.html generated +Doc/c-api/lifecycle.dot.svg generated Doc/data/stable_abi.dat generated Doc/library/token-list.inc generated Include/internal/pycore_ast.h generated @@ -80,13 +83,19 @@ Include/opcode.h generated Include/opcode_ids.h generated Include/token.h generated Lib/_opcode_metadata.py generated +Lib/idlelib/help.html generated Lib/keyword.py generated +Lib/pydoc_data/topics.py generated +Lib/pydoc_data/module_docs.py generated Lib/test/certdata/*.pem generated Lib/test/certdata/*.0 generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated +Lib/test/test_zoneinfo/data/*.json generated Lib/token.py generated Misc/sbom.spdx.json generated +Modules/_testinternalcapi/test_cases.c.h generated +Modules/_testinternalcapi/test_targets.h generated Objects/typeslots.inc generated PC/python3dll.c generated Parser/parser.c generated @@ -101,3 +110,4 @@ Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated aclocal.m4 generated configure generated +*.min.js generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e929ac8c474b2f..6b6074be0a5728 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,115 +1,265 @@ # See https://help.github.com/articles/about-codeowners/ -# for more info about CODEOWNERS file +# for further details about the .github/CODEOWNERS file. # It uses the same pattern rule for gitignore file # https://git-scm.com/docs/gitignore#_pattern_format +# Notably, a later match overrides earlier matches, so order matters. +# If using a wildcard pattern, try to be as specific as possible to avoid +# matching unintended files or overriding previous entries. +# To exclude a file from ownership, add a line with only the file. +# See the exclusions section at the end of the file for examples. + +# ======= +# Purpose +# ======= +# +# An entry in this file does not imply 'ownership', despite the name of the +# file, but instead that those listed take an interest in that part of the +# project and will automatically be added as reviewers to PRs that affect +# the matching files. +# See also the Experts Index in the Python Developer's Guide: +# https://devguide.python.org/core-developers/experts/ +# +# ========= +# Structure +# ========= +# +# The CODEOWNERS file is organised by topic area. +# Please add new entries in alphabetical order within the relevant section. +# Where possible, keep related files together. For example, documentation, +# code, and tests for a given item should all be listed in the same place. +# +# GitHub usernames should be aligned to column 31, or the next multiple +# of three if the relevant paths are too long to fit. +# +# Top-level sections are: +# +# * Buildbots, Continuous Integration, and Testing +# project-wide configuration files, internal tools for use in CI, +# linting. +# * Build System +# the Makefile, autoconf, and other autotools files. +# * Documentation +# broader sections of documentation, documentation tools +# * Internal Tools & Data +# internal tools, integration with external systems, +# entries that don't fit elsewhere +# * Platform Support +# relating to support for specific platforms +# * Interpreter Core +# the grammar, parser, compiler, interpreter, etc. +# * Standard Library +# standard library modules (from both Lib and Modules) +# and related files (such as their tests and docs) +# * Exclusions +# exclusions from .github/CODEOWNERS should go at the very end +# because the final matching pattern will take precedence. + +# ---------------------------------------------------------------------------- +# Buildbots, Continuous Integration, and Testing +# ---------------------------------------------------------------------------- + +# Azure Pipelines +.azure-pipelines/ @AA-Turner + +# GitHub & related scripts +.github/ @ezio-melotti @hugovk @AA-Turner @webknjaz +Tools/build/compute-changes.py @AA-Turner @hugovk @webknjaz +Tools/build/verify_ensurepip_wheels.py @AA-Turner @pfmoore @pradyunsg + +# Pre-commit +.pre-commit-config.yaml @hugovk +.ruff.toml @hugovk @AlexWaygood @AA-Turner + +# Patchcheck +Tools/patchcheck/ @AA-Turner + + +# ---------------------------------------------------------------------------- +# Build System +# ---------------------------------------------------------------------------- + +# Autotools +configure* @erlend-aasland @corona10 @AA-Turner @emmatyping +Makefile.pre.in @erlend-aasland @AA-Turner @emmatyping +Modules/makesetup @erlend-aasland @AA-Turner @emmatyping +Modules/Setup* @erlend-aasland @AA-Turner @emmatyping +Tools/build/regen-configure.sh @AA-Turner + +# generate-build-details +Tools/build/generate-build-details.py @FFY00 +Lib/test/test_build_details.py @FFY00 + + +# ---------------------------------------------------------------------------- +# Documentation +# ---------------------------------------------------------------------------- + +# Internal Docs +InternalDocs/ @AA-Turner + +# Tools, Configuration, etc +Doc/Makefile @AA-Turner @hugovk +Doc/_static/ @AA-Turner @hugovk +Doc/conf.py @AA-Turner @hugovk +Doc/make.bat @AA-Turner @hugovk +Doc/requirements.txt @AA-Turner @hugovk +Doc/tools/ @AA-Turner @hugovk + +# PR Previews +.readthedocs.yml @AA-Turner + +# Sections +Doc/reference/ @willingc @AA-Turner +Doc/whatsnew/ @AA-Turner + + +# ---------------------------------------------------------------------------- +# Internal Tools and Data +# ---------------------------------------------------------------------------- -# GitHub -.github/** @ezio-melotti @hugovk +# Argument Clinic +Tools/clinic/ @erlend-aasland @AA-Turner +Lib/test/test_clinic.py @erlend-aasland @AA-Turner +Doc/howto/clinic.rst @erlend-aasland @AA-Turner + +# C Analyser +Tools/c-analyzer/ @ericsnowcurrently + +# C API Documentation Checks +Tools/check-c-api-docs/ @ZeroIntensity + +# Fuzzing +Modules/_xxtestfuzz/ @ammaraskar + +# Limited C API & Stable ABI +Doc/c-api/stable.rst @encukou +Doc/data/*.abi @encukou +Misc/stable_abi.toml @encukou +Tools/build/stable_abi.py @encukou + +# SBOM +Misc/externals.spdx.json @sethmlarson +Misc/sbom.spdx.json @sethmlarson +Tools/build/generate_sbom.py @sethmlarson + +# ABI check +Misc/libabigail.abignore @encukou + + +# ---------------------------------------------------------------------------- +# Platform Support +# ---------------------------------------------------------------------------- + +# Android +Android/ @mhsmith @freakboy3742 +Doc/using/android.rst @mhsmith @freakboy3742 +Lib/_android_support.py @mhsmith @freakboy3742 +Lib/test/test_android.py @mhsmith @freakboy3742 + +# iOS +Doc/using/ios.rst @freakboy3742 +Lib/_ios_support.py @freakboy3742 +Apple/ @freakboy3742 +iOS/ @freakboy3742 + +# macOS +Mac/ @python/macos-team +Lib/_osx_support.py @python/macos-team +Lib/test/test__osx_support.py @python/macos-team + +# WebAssembly +Tools/wasm/README.md @brettcannon @freakboy3742 @emmatyping + +# WebAssembly (Emscripten) +Tools/wasm/config.site-wasm32-emscripten @freakboy3742 @emmatyping +Tools/wasm/emscripten @freakboy3742 @emmatyping + +# WebAssembly (WASI) +Platforms/WASI @brettcannon @emmatyping @savannahostrowski +Tools/wasm/wasi-env @brettcannon @emmatyping @savannahostrowski +Tools/wasm/wasi.py @brettcannon @emmatyping @savannahostrowski +Tools/wasm/wasi @brettcannon @emmatyping @savannahostrowski -# pre-commit -.pre-commit-config.yaml @hugovk @AlexWaygood -.ruff.toml @hugovk @AlexWaygood +# Windows +PC/ @python/windows-team +PCbuild/ @python/windows-team + +# Windows installer packages +Tools/msi/ @python/windows-team +Tools/nuget/ @python/windows-team + +# Windows Launcher +PC/launcher.c @python/windows-team @vsajip -# Build system -configure* @erlend-aasland @corona10 -Makefile.pre.in @erlend-aasland -Modules/Setup* @erlend-aasland -# asyncio -**/*asyncio* @1st1 @asvetlov @kumaraditya303 @willingc +# ---------------------------------------------------------------------------- +# Interpreter Core +# ---------------------------------------------------------------------------- -# Core -**/*context* @1st1 +# AST +Lib/_ast_unparse.py @isidentical @JelleZijlstra @eclips4 @tomasr8 +Lib/ast.py @isidentical @JelleZijlstra @eclips4 @tomasr8 +Lib/test/test_ast/ @eclips4 @tomasr8 +Parser/asdl.py @isidentical @JelleZijlstra @eclips4 @tomasr8 +Parser/asdl_c.py @isidentical @JelleZijlstra @eclips4 @tomasr8 +Python/ast.c @isidentical @JelleZijlstra @eclips4 @tomasr8 +Python/ast_preprocess.c @isidentical @eclips4 @tomasr8 + +# Built-in types +Objects/call.c @markshannon +Objects/codeobject.c @markshannon +Objects/dict* @methane @markshannon +Objects/frameobject.c @markshannon **/*genobject* @markshannon -**/*hamt* @1st1 -**/*jit* @brandtbucher +Objects/object.c @ZeroIntensity Objects/set* @rhettinger -Objects/dict* @methane @markshannon -Objects/typevarobject.c @JelleZijlstra Objects/type* @markshannon -Objects/codeobject.c @markshannon -Objects/frameobject.c @markshannon -Objects/call.c @markshannon -Python/ceval*.c @markshannon -Python/ceval*.h @markshannon +Objects/typevarobject.c @JelleZijlstra +Objects/unionobject.c @JelleZijlstra + +# Byte code interpreter ('the eval loop') +Python/bytecodes.c @markshannon +Python/ceval* @markshannon +Tools/cases_generator/ @markshannon + +# Compiler (AST to byte code) +Python/assemble.c @markshannon @iritkatriel Python/codegen.c @markshannon @iritkatriel Python/compile.c @markshannon @iritkatriel -Python/assemble.c @markshannon @iritkatriel Python/flowgraph.c @markshannon @iritkatriel Python/instruction_sequence.c @iritkatriel -Python/bytecodes.c @markshannon -Python/optimizer*.c @markshannon -Python/optimizer_analysis.c @Fidget-Spinner -Python/optimizer_bytecodes.c @Fidget-Spinner Python/symtable.c @JelleZijlstra @carljm -Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv -Lib/test/test_patma.py @brandtbucher -Lib/test/test_type_*.py @JelleZijlstra -Lib/test/test_capi/test_misc.py @markshannon -Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv -Tools/c-analyzer/ @ericsnowcurrently -# dbm -**/*dbm* @corona10 @erlend-aasland @serhiy-storchaka +# Context variables & HAMT +**/contextvars* @1st1 +**/*hamt* @1st1 +Include/cpython/context.h @1st1 +Include/internal/pycore_context.h @1st1 +Lib/test/test_context.py @1st1 +Python/context.c @1st1 -# runtime state/lifecycle -**/*pylifecycle* @ericsnowcurrently -**/*pystate* @ericsnowcurrently -**/*preconfig* @ericsnowcurrently -**/*initconfig* @ericsnowcurrently -**/*pathconfig* @ericsnowcurrently -**/*sysmodule* @ericsnowcurrently +# Core Modules **/*bltinmodule* @ericsnowcurrently -**/*gil* @ericsnowcurrently -Include/internal/pycore_runtime.h @ericsnowcurrently -Include/internal/pycore_interp.h @ericsnowcurrently -Include/internal/pycore_tstate.h @ericsnowcurrently -Include/internal/pycore_*_state.h @ericsnowcurrently -Include/internal/pycore_*_init.h @ericsnowcurrently -Include/internal/pycore_atexit.h @ericsnowcurrently -Include/internal/pycore_freelist.h @ericsnowcurrently -Include/internal/pycore_global_objects.h @ericsnowcurrently -Include/internal/pycore_obmalloc.h @ericsnowcurrently -Include/internal/pycore_pymem.h @ericsnowcurrently -Include/internal/pycore_stackref.h @Fidget-Spinner -Modules/main.c @ericsnowcurrently -Programs/_bootstrap_python.c @ericsnowcurrently -Programs/python.c @ericsnowcurrently -Tools/build/generate_global_objects.py @ericsnowcurrently +**/*sysmodule* @ericsnowcurrently # Exceptions Lib/test/test_except*.py @iritkatriel Objects/exceptions.c @iritkatriel -# Hashing -**/*hashlib* @gpshead @tiran -**/*pyhash* @gpshead @tiran -**/sha* @gpshead @tiran -Modules/md5* @gpshead @tiran -**/*blake* @gpshead @tiran -Modules/_blake2/** @gpshead @tiran -Modules/_hacl/** @gpshead - -# logging -**/*logging* @vsajip +# Getpath +Lib/test/test_getpath.py @FFY00 +Modules/getpath* @FFY00 -# venv -**/*venv* @vsajip +# Hashing / ``hash()`` and related +Include/cpython/pyhash.h @gpshead @picnixz +Include/internal/pycore_pyhash.h @gpshead @picnixz +Include/pyhash.h @gpshead @picnixz +Python/pyhash.c @gpshead @picnixz -# Launcher -/PC/launcher.c @vsajip - -# HTML -/Lib/html/ @ezio-melotti -/Lib/_markupbase.py @ezio-melotti -/Lib/test/test_html*.py @ezio-melotti -/Tools/build/parse_html5_entities.py @ezio-melotti - -# Import (including importlib). +# The import system (including importlib) **/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw -/Python/import.c @kumaraditya303 -Python/dynload_*.c @ericsnowcurrently +Python/import.c @brettcannon @ericsnowcurrently @ncoghlan @warsaw @kumaraditya303 **/*freeze* @ericsnowcurrently **/*frozen* @ericsnowcurrently **/*modsupport* @ericsnowcurrently @@ -120,19 +270,171 @@ Python/dynload_*.c @ericsnowcurrently **/*pythonrun* @ericsnowcurrently **/*runpy* @ericsnowcurrently **/*singlephase* @ericsnowcurrently -Lib/test/test_module/ @ericsnowcurrently Doc/c-api/module.rst @ericsnowcurrently -**/*importlib/resources/* @jaraco @warsaw @FFY00 -**/*importlib/metadata/* @jaraco @warsaw +Lib/test/test_module/ @ericsnowcurrently +Python/dynload_*.c @ericsnowcurrently + +# Initialisation +**/*initconfig* @ericsnowcurrently +**/*pathconfig* @ericsnowcurrently +**/*preconfig* @ericsnowcurrently +Doc/library/sys_path_init.rst @FFY00 +Doc/c-api/init_config.rst @FFY00 + +# Interpreter main program +Modules/main.c @ericsnowcurrently +Programs/_bootstrap_python.c @ericsnowcurrently +Programs/python.c @ericsnowcurrently + +# JIT +Include/internal/pycore_jit.h @brandtbucher @savannahostrowski @diegorusso +Python/jit.c @brandtbucher @savannahostrowski @diegorusso +Tools/jit/ @brandtbucher @savannahostrowski @diegorusso +InternalDocs/jit.md @brandtbucher @savannahostrowski @diegorusso @AA-Turner + +# Micro-op / μop / Tier 2 Optimiser +Python/optimizer.c @markshannon @Fidget-Spinner +Python/optimizer_analysis.c @markshannon @tomasr8 @Fidget-Spinner @savannahostrowski +Python/optimizer_bytecodes.c @markshannon @tomasr8 @Fidget-Spinner @savannahostrowski +Python/optimizer_symbols.c @markshannon @tomasr8 @Fidget-Spinner @savannahostrowski + +# Parser, Lexer, and Grammar +Grammar/python.gram @pablogsal @lysnikolaou +Lib/test/test_peg_generator/ @pablogsal @lysnikolaou +Lib/test/test_tokenize.py @pablogsal @lysnikolaou +Lib/tokenize.py @pablogsal @lysnikolaou +Parser/ @pablogsal @lysnikolaou +Tools/peg_generator/ @pablogsal @lysnikolaou + +# Runtime state/lifecycle +**/*gil* @ericsnowcurrently +**/*pylifecycle* @ericsnowcurrently @ZeroIntensity +**/*pystate* @ericsnowcurrently @ZeroIntensity +Include/internal/pycore_*_init.h @ericsnowcurrently +Include/internal/pycore_*_state.h @ericsnowcurrently +Include/internal/pycore_atexit.h @ericsnowcurrently +Include/internal/pycore_freelist.h @ericsnowcurrently +Include/internal/pycore_global_objects.h @ericsnowcurrently +Include/internal/pycore_interp.h @ericsnowcurrently +Include/internal/pycore_obmalloc.h @ericsnowcurrently +Include/internal/pycore_pymem.h @ericsnowcurrently +Include/internal/pycore_runtime.h @ericsnowcurrently +Include/internal/pycore_stackref.h @Fidget-Spinner +Include/internal/pycore_tstate.h @ericsnowcurrently +Tools/build/generate_global_objects.py @ericsnowcurrently + +# Remote Debugging +Python/remote_debug.h @pablogsal +Python/remote_debugging.c @pablogsal +Modules/_remote_debugging/ @pablogsal + +# Sub-Interpreters +**/*crossinterp* @ericsnowcurrently +**/*interpreteridobject.* @ericsnowcurrently +Doc/library/concurrent.interpreters.rst @ericsnowcurrently +Lib/concurrent/futures/interpreter.py @ericsnowcurrently +Lib/concurrent/interpreters/ @ericsnowcurrently +Lib/test/support/channels.py @ericsnowcurrently +Lib/test/test__interp*.py @ericsnowcurrently +Lib/test/test_interpreters/ @ericsnowcurrently +Modules/_interp*module.c @ericsnowcurrently + +# Template string literals (t-strings) +Lib/test/test_tstring.py @lysnikolaou +Objects/interpolationobject.c @lysnikolaou +Objects/templateobject.c @lysnikolaou + +# Tests +Lib/test/test_patma.py @brandtbucher +Lib/test/test_type_*.py @JelleZijlstra +Lib/test/test_capi/test_misc.py @markshannon + + +# ---------------------------------------------------------------------------- +# Standard Library +# ---------------------------------------------------------------------------- + +# Annotationlib +Doc/library/annotationlib.rst @JelleZijlstra +Lib/annotationlib.py @JelleZijlstra +Lib/test/test_annotationlib.py @JelleZijlstra + +# Argparse +Doc/**/argparse*.rst @savannahostrowski +Lib/argparse.py @savannahostrowski +Lib/test/test_argparse.py @savannahostrowski + +# Asyncio +Doc/library/asyncio*.rst @1st1 @asvetlov @kumaraditya303 @willingc +InternalDocs/asyncio.md @1st1 @asvetlov @kumaraditya303 @willingc @AA-Turner +Lib/asyncio/ @1st1 @asvetlov @kumaraditya303 @willingc +Lib/test/test_asyncio/ @1st1 @asvetlov @kumaraditya303 @willingc +Modules/_asynciomodule.c @1st1 @asvetlov @kumaraditya303 @willingc + +# Bisect +Doc/library/bisect.rst @rhettinger +Lib/bisect.py @rhettinger +Lib/test/test_bisect.py @rhettinger +Modules/_bisectmodule.c @rhettinger + +# Calendar +Lib/calendar.py @AA-Turner +Lib/test/test_calendar.py @AA-Turner + +# Cryptographic Primitives and Applications +**/*hashlib* @gpshead @picnixz +**/*hashopenssl* @gpshead @picnixz +**/*hmac* @gpshead @picnixz +**/*ssl* @gpshead @picnixz +Modules/_hacl/ @gpshead @picnixz +Modules/*blake* @gpshead @picnixz +Modules/*md5* @gpshead @picnixz +Modules/*sha* @gpshead @picnixz + +# Codecs +Modules/cjkcodecs/ @corona10 +Tools/unicode/gencjkcodecs.py @corona10 + +# Collections +Doc/library/collections.abc.rst @rhettinger +Doc/library/collections.rst @rhettinger +Lib/_collections_abc.py @rhettinger +Lib/collections/ @rhettinger +Lib/test/test_collections.py @rhettinger +Modules/_collectionsmodule.c @rhettinger + +# Colorize +Lib/_colorize.py @hugovk +Lib/test/test__colorize.py @hugovk + +# Config Parser +Lib/configparser.py @jaraco +Lib/test/test_configparser.py @jaraco + +# Dataclasses +Doc/library/dataclasses.rst @ericvsmith +Lib/dataclasses.py @ericvsmith +Lib/test/test_dataclasses/ @ericvsmith # Dates and times -**/*datetime* @pganssle @abalkin -**/*str*time* @pganssle @abalkin -Doc/library/time.rst @pganssle @abalkin -Lib/test/test_time.py @pganssle @abalkin -Modules/timemodule.c @pganssle @abalkin -Python/pytime.c @pganssle @abalkin -Include/internal/pycore_time.h @pganssle @abalkin +Doc/**/*time.rst @pganssle @abalkin +Doc/library/zoneinfo.rst @pganssle +Include/datetime.h @pganssle @abalkin +Include/internal/pycore_time.h @pganssle @abalkin +Lib/test/test_zoneinfo/ @pganssle +Lib/zoneinfo/ @pganssle +Lib/*time.py @pganssle @abalkin +Lib/test/datetimetester.py @pganssle @abalkin +Lib/test/test_*time.py @pganssle @abalkin +Modules/*zoneinfo* @pganssle +Modules/*time* @pganssle @abalkin +Python/pytime.c @pganssle @abalkin + +# Dbm +Doc/library/dbm.rst @corona10 @erlend-aasland @serhiy-storchaka +Lib/dbm/ @corona10 @erlend-aasland @serhiy-storchaka +Lib/test/test_dbm*.py @corona10 @erlend-aasland @serhiy-storchaka +Modules/*dbm* @corona10 @erlend-aasland @serhiy-storchaka # Email and related **/*mail* @python/email-team @@ -141,144 +443,202 @@ Include/internal/pycore_time.h @pganssle @abalkin **/*imap* @python/email-team **/*poplib* @python/email-team -# Garbage collector -/Modules/gcmodule.c @pablogsal -/Doc/library/gc.rst @pablogsal - -# Parser -/Parser/ @pablogsal @lysnikolaou -/Tools/peg_generator/ @pablogsal @lysnikolaou -/Lib/test/test_peg_generator/ @pablogsal @lysnikolaou -/Grammar/python.gram @pablogsal @lysnikolaou -/Lib/tokenize.py @pablogsal @lysnikolaou -/Lib/test/test_tokenize.py @pablogsal @lysnikolaou +# Ensurepip +Doc/library/ensurepip.rst @pfmoore @pradyunsg +Lib/ensurepip/ @pfmoore @pradyunsg +Lib/test/test_ensurepip.py @pfmoore @pradyunsg + +# Enum +Doc/howto/enum.rst @ethanfurman +Doc/library/enum.rst @ethanfurman +Lib/enum.py @ethanfurman +Lib/test/test_enum.py @ethanfurman +Lib/test/test_json/test_enum.py @ethanfurman + +# FTP +Doc/library/ftplib.rst @giampaolo +Lib/ftplib.py @giampaolo +Lib/test/test_ftplib.py @giampaolo + +# Functools +Doc/library/functools.rst @rhettinger +Lib/functools.py @rhettinger +Lib/test/test_functools.py @rhettinger +Modules/_functoolsmodule.c @rhettinger -# Code generator -/Tools/cases_generator/ @markshannon +# Garbage collector +Modules/gcmodule.c @pablogsal +Doc/library/gc.rst @pablogsal -# AST -Python/ast.c @isidentical @JelleZijlstra @eclips4 -Python/ast_opt.c @isidentical @eclips4 -Parser/asdl.py @isidentical @JelleZijlstra @eclips4 -Parser/asdl_c.py @isidentical @JelleZijlstra @eclips4 -Lib/ast.py @isidentical @JelleZijlstra @eclips4 -Lib/test/test_ast/ @eclips4 +# Gettext +Doc/library/gettext.rst @tomasr8 +Lib/gettext.py @tomasr8 +Lib/test/test_gettext.py @tomasr8 +Tools/i18n/pygettext.py @tomasr8 -# Mock -/Lib/unittest/mock.py @cjw296 -/Lib/test/test_unittest/testmock/* @cjw296 +# Heapq +Doc/library/heapq* @rhettinger +Lib/heapq.py @rhettinger +Lib/test/test_heapq.py @rhettinger +Modules/_heapqmodule.c @rhettinger -# multiprocessing -**/*multiprocessing* @gpshead +# HTML +Doc/library/html* @ezio-melotti +Lib/html/ @ezio-melotti +Lib/_markupbase.py @ezio-melotti +Lib/test/test_html*.py @ezio-melotti +Tools/build/parse_html5_entities.py @ezio-melotti + +# IDLE +Doc/library/idle.rst @terryjreedy +Lib/idlelib/ @terryjreedy +Lib/turtledemo/ @terryjreedy + +# importlib.metadata +Doc/library/importlib.metadata.rst @jaraco @warsaw +Lib/importlib/metadata/ @jaraco @warsaw +Lib/test/test_importlib/metadata/ @jaraco @warsaw + +# importlib.resources +Doc/library/importlib.resources.abc.rst @jaraco @warsaw +Doc/library/importlib.resources.rst @jaraco @warsaw +Lib/importlib/resources/ @jaraco @warsaw @FFY00 +Lib/test/test_importlib/resources/ @jaraco @warsaw @FFY00 + +# Itertools +Doc/library/itertools.rst @rhettinger +Lib/test/test_itertools.py @rhettinger +Modules/itertoolsmodule.c @rhettinger + +# Logging +Doc/**/logging* @vsajip +Lib/logging/ @vsajip +Lib/test/test_logging.py @vsajip + +# Multiprocessing +Doc/library/multiprocessing*.rst @gpshead +Lib/multiprocessing/ @gpshead +Lib/test/*multiprocessing.py @gpshead +Lib/test/test_multiprocessing*/ @gpshead +Modules/_multiprocessing/ @gpshead + +# Pathlib +Doc/library/pathlib.rst @barneygale +Lib/pathlib/ @barneygale +Lib/test/test_pathlib/ @barneygale + +# Pdb & Bdb +Doc/library/bdb.rst @gaogaotiantian +Doc/library/pdb.rst @gaogaotiantian +Lib/bdb.py @gaogaotiantian +Lib/pdb.py @gaogaotiantian +Lib/test/test_bdb.py @gaogaotiantian +Lib/test/test_pdb.py @gaogaotiantian +Lib/test/test_remote_pdb.py @gaogaotiantian + +# Pydoc +Lib/pydoc.py @AA-Turner +Lib/pydoc_data/ @AA-Turner +Lib/test/test_pydoc/ @AA-Turner + +# Profiling (Sampling) +Doc/library/profiling*.rst @pablogsal +Lib/profiling/ @pablogsal +Lib/test/test_profiling/ @pablogsal + +# PyREPL +Lib/_pyrepl/ @pablogsal @lysnikolaou @ambv +Lib/test/test_pyrepl/ @pablogsal @lysnikolaou @ambv + +# Random +Doc/library/random.rst @rhettinger +Lib/random.py @rhettinger +Lib/test/test_random.py @rhettinger +Modules/_randommodule.c @rhettinger + +# Shutil +Doc/library/shutil.rst @giampaolo +Lib/shutil.py @giampaolo +Lib/test/test_shutil.py @giampaolo + +# Site +Lib/site.py @FFY00 +Lib/test/test_site.py @FFY00 +Doc/library/site.rst @FFY00 + +# string.templatelib +Doc/library/string.templatelib.rst @lysnikolaou @AA-Turner +Lib/string/templatelib.py @lysnikolaou @AA-Turner +Lib/test/test_string/test_templatelib.py @lysnikolaou @AA-Turner + +# Sysconfig +**/*sysconfig* @FFY00 # SQLite 3 -**/*sqlite* @berkerpeksag @erlend-aasland - -# subprocess -/Lib/subprocess.py @gpshead -/Lib/test/test_subprocess.py @gpshead -/Modules/*subprocess* @gpshead - -# debugger -**/*pdb* @gaogaotiantian -**/*bdb* @gaogaotiantian - -# Limited C API & stable ABI -Tools/build/stable_abi.py @encukou -Misc/stable_abi.toml @encukou -Doc/data/*.abi @encukou -Doc/c-api/stable.rst @encukou - -# Windows -/PC/ @python/windows-team -/PCbuild/ @python/windows-team +Doc/library/sqlite3.rst @berkerpeksag @erlend-aasland +Lib/sqlite3/ @berkerpeksag @erlend-aasland +Lib/test/test_sqlite3/ @berkerpeksag @erlend-aasland +Modules/_sqlite/ @berkerpeksag @erlend-aasland + +# Subprocess +Lib/subprocess.py @gpshead +Lib/test/test_subprocess.py @gpshead +Modules/*subprocess* @gpshead + +# Tarfile +Doc/library/tarfile.rst @ethanfurman +Lib/tarfile.py @ethanfurman +Lib/test/test_tarfile.py @ethanfurman + +# TOML +Doc/library/tomllib.rst @encukou @hauntsaninja +Lib/test/test_tomllib/ @encukou @hauntsaninja +Lib/tomllib/ @encukou @hauntsaninja + +# Typing +Doc/library/typing.rst @JelleZijlstra @AlexWaygood +Lib/test/test_typing.py @JelleZijlstra @AlexWaygood +Lib/test/typinganndata/ @JelleZijlstra @AlexWaygood +Lib/typing.py @JelleZijlstra @AlexWaygood +Modules/_typingmodule.c @JelleZijlstra @AlexWaygood + +# Types +Lib/test/test_types.py @AA-Turner +Lib/types.py @AA-Turner +Modules/_typesmodule.c @AA-Turner + +# Unittest +Lib/unittest/mock.py @cjw296 +Lib/test/test_unittest/testmock/ @cjw296 # Urllib **/*robotparser* @berkerpeksag -# Windows installer packages -/Tools/msi/ @python/windows-team -/Tools/nuget/ @python/windows-team - -# Misc -**/*itertools* @rhettinger -**/*collections* @rhettinger -**/*random* @rhettinger -**/*bisect* @rhettinger -**/*heapq* @rhettinger -**/*functools* @rhettinger - -**/*dataclasses* @ericvsmith - -**/*ensurepip* @pfmoore @pradyunsg - -/Doc/library/idle.rst @terryjreedy -**/*idlelib* @terryjreedy -**/*turtledemo* @terryjreedy - -**/*annotationlib* @JelleZijlstra -**/*typing* @JelleZijlstra @AlexWaygood - -**/*ftplib @giampaolo -**/*shutil @giampaolo - -**/*enum* @ethanfurman -**/*cgi* @ethanfurman -**/*tarfile* @ethanfurman - -**/*tomllib* @encukou @hauntsaninja - -**/*sysconfig* @FFY00 - -**/*cjkcodecs* @corona10 +# Venv +**/*venv* @vsajip @FFY00 -# macOS -/Mac/ @python/macos-team -**/*osx_support* @python/macos-team - -# pathlib -**/*pathlib* @barneygale - -# zipfile.Path -**/*zipfile/_path/* @jaraco - -# Argument Clinic -/Tools/clinic/** @erlend-aasland -/Lib/test/test_clinic.py @erlend-aasland -Doc/howto/clinic.rst @erlend-aasland - -# Subinterpreters -**/*interpreteridobject.* @ericsnowcurrently -**/*crossinterp* @ericsnowcurrently -Lib/test/support/interpreters/ @ericsnowcurrently -Modules/_interp*module.c @ericsnowcurrently -Lib/test/test_interpreters/ @ericsnowcurrently - -# Android -**/*Android* @mhsmith -**/*android* @mhsmith +# Weakref +**/*weakref* @kumaraditya303 -# iOS (but not termios) -**/iOS* @freakboy3742 -**/ios* @freakboy3742 -**/*_iOS* @freakboy3742 -**/*_ios* @freakboy3742 -**/*-iOS* @freakboy3742 -**/*-ios* @freakboy3742 +# Zipfile.Path +Lib/test/test_zipfile/_path/ @jaraco +Lib/zipfile/_path/ @jaraco -# WebAssembly -/Tools/wasm/ @brettcannon +# Zstandard +Lib/compression/zstd/ @AA-Turner @emmatyping +Lib/test/test_zstd.py @AA-Turner @emmatyping +Modules/_zstd/ @AA-Turner @emmatyping -# SBOM -/Misc/externals.spdx.json @sethmlarson -/Misc/sbom.spdx.json @sethmlarson -/Tools/build/generate_sbom.py @sethmlarson +# ---------------------------------------------------------------------------- -# Config Parser -Lib/configparser.py @jaraco -Lib/test/test_configparser.py @jaraco +# Exclusions from .github/CODEOWNERS should go at the very end +# because the final matching pattern will take precedence. -# Doc sections -Doc/reference/ @willingc +# Exclude .mailmap from being owned by @python/email-team +.mailmap -**/*weakref* @kumaraditya303 +# Exclude Argument Clinic directories +Modules/**/clinic/ +Objects/**/clinic/ +PC/**/clinic/ +Python/**/clinic/ diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 2ef9cdc191481e..94b67ce3dbe341 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -4,7 +4,7 @@ Contributing to Python Build Status ------------ -- `Buildbot status overview `_ +- `Buildbot status overview `_ - `GitHub Actions status `_ @@ -28,23 +28,23 @@ Please be aware that our workflow does deviate slightly from the typical GitHub project. Details on how to properly submit a pull request are covered in `Lifecycle of a Pull Request `_. We utilize various bots and status checks to help with this, so do follow the -comments they leave and their "Details" links, respectively. The key points of -our workflow that are not covered by a bot or status check are: +comments they leave and their "Details" links, respectively. -- All discussions that are not directly related to the code in the pull request - should happen on `GitHub Issues `_. -- Upon your first non-trivial pull request (which includes documentation changes), - feel free to add yourself to ``Misc/ACKS`` +The final key part of our workflow is that all discussions that are not +directly related to the code in the pull request should happen on +`GitHub Issues `__, generally in the +pull request's parent issue. Setting Expectations -------------------- -Due to the fact that this project is entirely volunteer-run (i.e. no one is paid -to work on Python full-time), we unfortunately can make no guarantees as to if +Due to the fact that this project is run by volunteers, +unfortunately we cannot make any guarantees as to if or when a core developer will get around to reviewing your pull request. If no core developer has done a review or responded to changes made because of a -"changes requested" review, please feel free to email python-dev to ask if -someone could take a look at your pull request. +"changes requested" review within a month, you can ask for someone to +review your pull request via a post in the `Core Development Discourse +category `__. Code of Conduct diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 7b7810cf6965fd..da70710b7ecfa3 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -40,6 +40,7 @@ body: - "3.12" - "3.13" - "3.14" + - "3.15" - "CPython main branch" validations: required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 75d174307ce160..de6e8756b03d80 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,3 +5,6 @@ contact_links: - name: "Proposing new features" about: "Submit major feature proposal (e.g. syntax changes) to an ideas forum first." url: "https://discuss.python.org/c/ideas/6" + - name: "Python Install Manager issues" + about: "Report issues with the Python Install Manager (for Windows)" + url: "https://github.com/python/pymanager/issues" diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml index 58da2dfe0c7354..470ad581367b10 100644 --- a/.github/ISSUE_TEMPLATE/crash.yml +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -33,6 +33,7 @@ body: - "3.12" - "3.13" - "3.14" + - "3.15" - "CPython main branch" validations: required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4cc2f461dbe2f2..03ea959ca14e28 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,10 +7,10 @@ Please read this comment in its entirety. It's quite important. It should be in the following format: ``` -gh-NNNNN: Summary of the changes made +gh-NNNNNN: Summary of the changes made ``` -Where: gh-NNNNN refers to the GitHub issue number. +Where: gh-NNNNNN refers to the GitHub issue number. Most PRs will require an issue number. Trivial changes, like fixing a typo, do not need an issue. @@ -20,11 +20,11 @@ If this is a backport PR (PR made against branches other than `main`), please ensure that the PR title is in the following format: ``` -[X.Y] (GH-NNNN) +[X.Y] <title from the original PR> (GH-NNNNNN) ``` -Where: [X.Y] is the branch name, e.g. [3.6]. +Where: [X.Y] is the branch name, for example: [3.13]. -GH-NNNN refers to the PR number from `main`. +GH-NNNNNN refers to the PR number from `main`. --> diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000000000..267ff6b42a8655 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,12 @@ +self-hosted-runner: + # Pending https://github.com/rhysd/actionlint/issues/533 + # and https://github.com/rhysd/actionlint/issues/571 + labels: ["windows-11-arm", "macos-15-intel"] + +config-variables: null + +paths: + .github/workflows/**/*.yml: + ignore: + - 1st argument of function call is not assignable + - SC2(015|038|086|091|097|098|129|155) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c8a3165d690364..7f3376f8ddb1e2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,6 +12,11 @@ updates: update-types: - "version-update:semver-minor" - "version-update:semver-patch" + cooldown: + # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns + # Cooldowns protect against supply chain attacks by avoiding the + # highest-risk window immediately after new releases. + default-days: 14 - package-ecosystem: "pip" directory: "/Tools/" schedule: @@ -19,3 +24,5 @@ updates: labels: - "skip issue" - "skip news" + cooldown: + default-days: 14 diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml index 570b8779994a0f..c404bc519300e2 100644 --- a/.github/workflows/add-issue-header.yml +++ b/.github/workflows/add-issue-header.yml @@ -18,8 +18,9 @@ jobs: runs-on: ubuntu-latest permissions: issues: write + timeout-minutes: 5 steps: - - uses: actions/github-script@v7 + - uses: actions/github-script@v8 with: # language=JavaScript script: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d1e9fff37d3c2d..e7f7aa5172e082 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,54 +15,62 @@ permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable + # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#concurrency + # 'group' must be a key uniquely representing a PR or push event. + # github.workflow is the workflow name + # github.actor is the user invoking the workflow + # github.head_ref is the source branch of the PR or otherwise blank + # github.run_id is a unique number for the current run + group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: - check_source: + build-context: name: Change detection # To use boolean outputs from this job, parse them as JSON. # Here's some examples: # - # if: fromJSON(needs.check_source.outputs.run-docs) + # if: fromJSON(needs.build-context.outputs.run-docs) # # ${{ - # fromJSON(needs.check_source.outputs.run_tests) + # fromJSON(needs.build-context.outputs.run-tests) # && 'truthy-branch' # || 'falsy-branch' # }} # - uses: ./.github/workflows/reusable-change-detection.yml + uses: ./.github/workflows/reusable-context.yml check-docs: name: Docs - needs: check_source - if: fromJSON(needs.check_source.outputs.run-docs) + needs: build-context + if: fromJSON(needs.build-context.outputs.run-docs) uses: ./.github/workflows/reusable-docs.yml - check_autoconf_regen: + check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job # reproducible: to get the same tools versions (autoconf, aclocal, ...) runs-on: ubuntu-24.04 container: - image: ghcr.io/python/autoconf:2024.10.16.11360930377 + image: ghcr.io/python/autoconf:2025.01.02.12581854023 timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' steps: - name: Install Git run: | - apt install git -yq + apt update && apt install git -yq git config --global --add safe.directory "$GITHUB_WORKSPACE" - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 - - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" + persist-credentials: false - name: Check Autoconf and aclocal versions run: | - grep "Generated by GNU Autoconf 2.71" configure + grep "Generated by GNU Autoconf 2.72" configure grep "aclocal 1.16.5" aclocal.m4 grep -q "runstatedir" configure grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 @@ -76,7 +84,7 @@ jobs: # Check for changes in regenerated files if test -n "$changes"; then echo "Generated files not up to date." - echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)" + echo "Perhaps you forgot to run make regen-configure ;)" echo "configure files must be regenerated with a specific version of autoconf." echo "$changes" echo "" @@ -84,35 +92,27 @@ jobs: exit 1 fi - check_generated_files: + check-generated-files: name: 'Check if generated files are up to date' # Don't use ubuntu-latest but a specific version to make the job # reproducible: to get the same tools versions (autoconf, aclocal, ...) - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 with: python-version: '3.x' - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} - - name: Install Dependencies + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | # Build Python with the libpython dynamic library @@ -143,32 +143,45 @@ jobs: if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals - build_windows: + check-c-api-docs: + name: C API Docs + needs: build-context + if: >- + needs.build-context.outputs.run-tests == 'true' + || needs.build-context.outputs.run-docs == 'true' + uses: ./.github/workflows/reusable-check-c-api-docs.yml + + build-windows: name: >- Windows ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: check_source - if: fromJSON(needs.check_source.outputs.run_tests) + needs: build-context + if: fromJSON(needs.build-context.outputs.run-windows-tests) strategy: + fail-fast: false matrix: arch: - - Win32 - - x64 - - arm64 + - x64 + - Win32 + - arm64 free-threading: - - false - - true + - false + - true + exclude: + # Skip Win32 on free-threaded builds + - { arch: Win32, free-threading: true } uses: ./.github/workflows/reusable-windows.yml with: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} - build_windows_msi: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Windows MSI${{ '' }} - needs: check_source - if: fromJSON(needs.check_source.outputs.run-win-msi) + build-windows-msi: + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Windows MSI${{ '' }} # zizmor: ignore[obfuscation] + needs: build-context + if: fromJSON(needs.build-context.outputs.run-windows-msi) strategy: + fail-fast: false matrix: arch: - x86 @@ -178,84 +191,92 @@ jobs: with: arch: ${{ matrix.arch }} - build_macos: + build-macos: name: >- macOS ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-macos == 'true' strategy: fail-fast: false matrix: - # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. - # macOS 13 only runs tests against the GIL-enabled CPython. - # Cirrus used for upstream, macos-14 for forks. + # macos-14 is M1, macos-15-intel is Intel. + # macos-15-intel only runs tests against the GIL-enabled CPython. os: - - ghcr.io/cirruslabs/macos-runner:sonoma - macos-14 - - macos-13 - is-fork: # only used for the exclusion trick - - ${{ github.repository_owner != 'python' }} + - macos-15-intel free-threading: - false - true exclude: - - os: ghcr.io/cirruslabs/macos-runner:sonoma - is-fork: true - - os: macos-14 - is-fork: false - - os: macos-13 + - os: macos-15-intel free-threading: true uses: ./.github/workflows/reusable-macos.yml with: - config_hash: ${{ needs.check_source.outputs.config_hash }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} - build_ubuntu: + build-ubuntu: name: >- Ubuntu ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }} + needs: build-context + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: + fail-fast: false matrix: + bolt: + - false + - true free-threading: - false - true + os: + - ubuntu-24.04 + - ubuntu-24.04-arm + exclude: + # Do not test BOLT with free-threading, to conserve resources + - bolt: true + free-threading: true + # BOLT currently crashes during instrumentation on aarch64 + - os: ubuntu-24.04-arm + bolt: true uses: ./.github/workflows/reusable-ubuntu.yml with: - config_hash: ${{ needs.check_source.outputs.config_hash }} + bolt-optimizations: ${{ matrix.bolt }} free-threading: ${{ matrix.free-threading }} + os: ${{ matrix.os }} - build_ubuntu_ssltests: + build-ubuntu-ssltests-openssl: name: 'Ubuntu SSL tests with OpenSSL' runs-on: ${{ matrix.os }} timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: - os: [ubuntu-22.04] - openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2] + os: [ubuntu-24.04] + # Keep 1.1.1w in our list despite it being upstream EOL and otherwise + # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs + # supported by important vendors such as AWS-LC. + openssl_ver: [1.1.1w, 3.0.18, 3.3.5, 3.4.3, 3.5.4, 3.6.0] + # See Tools/ssl/make_ssl_data.py for notes on adding a new version env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib steps: - - uses: actions/checkout@v4 - - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 + - uses: actions/checkout@v6 with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies + - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | @@ -264,7 +285,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -274,10 +295,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - name: Build CPython @@ -287,28 +304,139 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py - build_wasi: + build-ubuntu-ssltests-awslc: + name: 'Ubuntu SSL tests with AWS-LC' + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-ubuntu == 'true' + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04] + awslc_ver: [1.55.0] + env: + AWSLC_VER: ${{ matrix.awslc_ver}} + MULTISSL_DIR: ${{ github.workspace }}/multissl + OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure SSL lib env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" + - name: 'Restore AWS-LC build' + id: cache-aws-lc + uses: actions/cache@v5 + with: + path: ./multissl/aws-lc/${{ matrix.awslc_ver }} + key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} + - name: Install AWS-LC + if: steps.cache-aws-lc.outputs.cache-hit != 'true' + run: | + python3 Tools/ssl/multissltests.py \ + --steps=library \ + --base-directory "$MULTISSL_DIR" \ + --awslc ${{ matrix.awslc_ver }} \ + --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure CPython + run: | + ./configure CFLAGS="-fdiagnostics-format=json" \ + --config-cache \ + --enable-slower-safety \ + --with-pydebug \ + --with-openssl="$OPENSSL_DIR" \ + --with-builtin-hashlib-hashes=blake2 \ + --with-ssl-default-suites=openssl + - name: Build CPython + run: make -j + - name: Display build info + run: make pythoninfo + - name: Verify python is linked to AWS-LC + run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC + - name: SSL tests + run: ./python Lib/test/ssltests.py + + build-android: + name: Android (${{ matrix.arch }}) + needs: build-context + if: needs.build-context.outputs.run-android == 'true' + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - arch: aarch64 + runs-on: macos-14 + - arch: x86_64 + runs-on: ubuntu-24.04 + + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Build and test + run: ./Android/android.py ci --fast-ci ${{ matrix.arch }}-linux-android + + build-ios: + name: iOS + needs: build-context + if: needs.build-context.outputs.run-ios == 'true' + timeout-minutes: 60 + runs-on: macos-14 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + # GitHub recommends explicitly selecting the desired Xcode version: + # https://github.com/actions/runner-images/issues/12541#issuecomment-3083850140 + # This became a necessity as a result of + # https://github.com/actions/runner-images/issues/12541 and + # https://github.com/actions/runner-images/issues/12751. + - name: Select Xcode version + run: | + sudo xcode-select --switch /Applications/Xcode_15.4.app + + - name: Build and test + run: python3 Apple ci iOS --fast-ci --simulator 'iPhone SE (3rd generation),OS=17.5' + + build-wasi: name: 'WASI' - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-wasi == 'true' uses: ./.github/workflows/reusable-wasi.yml - with: - config_hash: ${{ needs.check_source.outputs.config_hash }} - test_hypothesis: + test-hypothesis: name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' + needs: build-context + if: needs.build-context.outputs.run-ubuntu == 'true' env: - OPENSSL_VER: 3.0.15 + OPENSSL_VER: 3.0.18 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + with: + persist-credentials: false - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies + - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | @@ -317,7 +445,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -327,10 +455,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -340,12 +464,7 @@ jobs: - name: Bind mount sources read-only run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -376,7 +495,7 @@ jobs: ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} @@ -389,10 +508,11 @@ jobs: # # (GH-104097) test_sysconfig is skipped because it has tests that are # failing when executed from inside a virtual environment. - ${{ env.VENV_PYTHON }} -m test \ + "${VENV_PYTHON}" -m test \ -W \ - -o \ + --slowest \ -j4 \ + --timeout 900 \ -x test_asyncio \ -x test_multiprocessing_fork \ -x test_multiprocessing_forkserver \ @@ -402,41 +522,38 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v6 if: always() with: name: hypothesis-example-db path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ - - build_asan: + build-asan: name: 'Address sanitizer' runs-on: ${{ matrix.os }} timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: + fail-fast: false matrix: - os: [ubuntu-22.04] + os: [ubuntu-24.04] env: - OPENSSL_VER: 3.0.15 + OPENSSL_VER: 3.0.18 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - - uses: actions/checkout@v4 - - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 + - uses: actions/checkout@v6 with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies + - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 + uses: egor-tensin/setup-gcc@v2 with: version: 10 - name: Configure OpenSSL env vars @@ -446,7 +563,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -456,11 +573,6 @@ jobs: - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - name: Build CPython @@ -468,138 +580,181 @@ jobs: - name: Display build info run: make pythoninfo - name: Tests - run: xvfb-run make test + run: xvfb-run make ci - build_tsan: - name: 'Thread sanitizer' - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - uses: ./.github/workflows/reusable-tsan.yml - with: - config_hash: ${{ needs.check_source.outputs.config_hash }} - options: ./configure --config-cache --with-thread-sanitizer --with-pydebug - suppressions_path: Tools/tsan/supressions.txt - tsan_logs_artifact_name: tsan-logs-default - - build_tsan_free_threading: - name: 'Thread sanitizer (free-threading)' - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - uses: ./.github/workflows/reusable-tsan.yml + build-san: + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Sanitizers${{ '' }} # zizmor: ignore[obfuscation] + needs: build-context + if: needs.build-context.outputs.run-ubuntu == 'true' + strategy: + fail-fast: false + matrix: + check-name: + - Thread + free-threading: + - false + - true + sanitizer: + - TSan + include: + - check-name: Undefined behavior + sanitizer: UBSan + free-threading: false + uses: ./.github/workflows/reusable-san.yml with: - config_hash: ${{ needs.check_source.outputs.config_hash }} - options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug - suppressions_path: Tools/tsan/suppressions_free_threading.txt - tsan_logs_artifact_name: tsan-logs-free-threading + sanitizer: ${{ matrix.sanitizer }} + free-threading: ${{ matrix.free-threading }} - # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ - cifuzz: - name: CIFuzz + cross-build-linux: + name: Cross build Linux runs-on: ubuntu-latest timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_cifuzz == 'true' + needs: build-context + if: needs.build-context.outputs.run-ubuntu == 'true' + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Set build dir + run: + # an absolute path outside of the working directoy + echo "BUILD_DIR=$(realpath ${{ github.workspace }}/../build)" >> "$GITHUB_ENV" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure host build + run: ./configure --prefix="$BUILD_DIR/host-python" + - name: Install host Python + run: make -j8 install + - name: Run test subset with host build + run: | + "$BUILD_DIR/host-python/bin/python3" -m test test_sysconfig test_site test_embed + - name: Configure cross build + run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3" + - name: Install cross Python + run: make -j8 install + - name: Run test subset with host build + run: | + "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed + + cifuzz: + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: CIFuzz${{ '' }} # zizmor: ignore[obfuscation] + needs: build-context + if: >- + needs.build-context.outputs.run-ci-fuzz == 'true' + || needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' permissions: security-events: write strategy: fail-fast: false matrix: - sanitizer: [address, undefined, memory] - steps: - - name: Build fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: cpython3 - sanitizer: ${{ matrix.sanitizer }} - - name: Run fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - fuzz-seconds: 600 - oss-fuzz-project-name: cpython3 - output-sarif: true - sanitizer: ${{ matrix.sanitizer }} - - name: Upload crash - uses: actions/upload-artifact@v4 - if: failure() && steps.build.outcome == 'success' - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts - - name: Upload SARIF - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif + sanitizer: + - address + - undefined + - memory + oss-fuzz-project-name: + - cpython3 + - python3-libraries + exclude: + # Note that the 'no-exclude' sentinel below is to prevent + # an empty string value from excluding all jobs and causing + # GHA to create a 'default' matrix entry with all empty values. + - oss-fuzz-project-name: >- + ${{ + needs.build-context.outputs.run-ci-fuzz == 'true' + && 'no-exclude' + || 'cpython3' + }} + - oss-fuzz-project-name: >- + ${{ + needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' + && 'no-exclude' + || 'python3-libraries' + }} + uses: ./.github/workflows/reusable-cifuzz.yml + with: + oss-fuzz-project-name: ${{ matrix.oss-fuzz-project-name }} + sanitizer: ${{ matrix.sanitizer }} all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass - if: always() - + runs-on: ubuntu-latest + timeout-minutes: 5 needs: - - check_source # Transitive dependency, needed to access `run_tests` value + - build-context # Transitive dependency, needed to access `run-tests` value - check-docs - - check_autoconf_regen - - check_generated_files - - build_macos - - build_ubuntu - - build_ubuntu_ssltests - - build_wasi - - build_windows - - build_windows_msi - - test_hypothesis - - build_asan - - build_tsan - - build_tsan_free_threading + - check-autoconf-regen + - check-generated-files + - check-c-api-docs + - build-windows + - build-windows-msi + - build-macos + - build-ubuntu + - build-ubuntu-ssltests-awslc + - build-ubuntu-ssltests-openssl + - build-ios + - build-wasi + - test-hypothesis + - build-asan + - build-san + - cross-build-linux - cifuzz - - runs-on: ubuntu-latest + if: always() steps: - name: Check whether the needed jobs succeeded or failed uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: allowed-failures: >- - build_ubuntu_ssltests, - build_windows_msi, + build-android, + build-windows-msi, + build-ubuntu-ssltests-awslc, + build-ubuntu-ssltests-openssl, + test-hypothesis, cifuzz, - test_hypothesis, allowed-skips: >- + ${{ !fromJSON(needs.build-context.outputs.run-docs) && 'check-docs,' || '' }} ${{ - !fromJSON(needs.check_source.outputs.run-docs) + needs.build-context.outputs.run-tests != 'true' && ' - check-docs, + check-autoconf-regen, + check-generated-files, ' || '' }} ${{ - needs.check_source.outputs.run_tests != 'true' - && ' - check_autoconf_regen, - check_generated_files, - build_macos, - build_ubuntu, - build_ubuntu_ssltests, - build_wasi, - build_windows, - build_asan, - build_tsan, - build_tsan_free_threading, - ' + !fromJSON(needs.build-context.outputs.run-tests) + && !fromJSON(needs.build-context.outputs.run-docs) + && 'check-c-api-docs,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-windows-tests) && 'build-windows,' || '' }} ${{ - !fromJSON(needs.check_source.outputs.run_cifuzz) - && ' - cifuzz, - ' - || '' + !fromJSON(needs.build-context.outputs.run-ci-fuzz) + && !fromJSON(needs.build-context.outputs.run-ci-fuzz-stdlib) + && 'cifuzz,' || + '' }} + ${{ !fromJSON(needs.build-context.outputs.run-macos) && 'build-macos,' || '' }} ${{ - !fromJSON(needs.check_source.outputs.run_hypothesis) + !fromJSON(needs.build-context.outputs.run-ubuntu) && ' - test_hypothesis, + build-ubuntu, + build-ubuntu-ssltests-awslc, + build-ubuntu-ssltests-openssl, + test-hypothesis, + build-asan, + build-san, + cross-build-linux, ' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-android) && 'build-android,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-ios) && 'build-ios,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-wasi) && 'build-wasi,' || '' }} jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index 43a7afec73884e..a09a30587b35eb 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -10,9 +10,6 @@ on: - 'Doc/**' - '.github/workflows/doc.yml' -permissions: - pull-requests: write - concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -20,6 +17,10 @@ concurrency: jobs: documentation-links: runs-on: ubuntu-latest + permissions: + pull-requests: write + timeout-minutes: 5 + steps: - uses: readthedocs/actions/preview@v1 with: diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 754f179f105591..cd6e9875d282d2 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -5,6 +5,9 @@ on: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' + - 'Python/executor_cases.c.h' + - 'Python/optimizer_cases.c.h' + - '**_testinternalcapi**' - '!Python/perf_jit_trampoline.c' - '!**/*.md' - '!**/*.ini' @@ -13,6 +16,9 @@ on: - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' + - 'Python/executor_cases.c.h' + - 'Python/optimizer_cases.c.h' + - '**_testinternalcapi**' - '!Python/perf_jit_trampoline.c' - '!**/*.md' - '!**/*.ini' @@ -25,13 +31,18 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: interpreter: name: Interpreter (Debug) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 90 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + with: + persist-credentials: false - name: Build tier two interpreter run: | ./configure --enable-experimental-jit=interpreter --with-pydebug @@ -54,128 +65,147 @@ jobs: - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc - - x86_64-unknown-linux-gnu/clang - aarch64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/clang debug: - true - false llvm: - - 18 + - 21 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 - runner: windows-latest - compiler: msvc + runner: windows-2022 - target: x86_64-pc-windows-msvc/msvc architecture: x64 - runner: windows-latest - compiler: msvc + runner: windows-2022 - target: aarch64-pc-windows-msvc/msvc architecture: ARM64 - runner: windows-latest - compiler: msvc + runner: windows-11-arm - target: x86_64-apple-darwin/clang architecture: x86_64 - runner: macos-13 - compiler: clang + runner: macos-15-intel - target: aarch64-apple-darwin/clang architecture: aarch64 runner: macos-14 - compiler: clang - target: x86_64-unknown-linux-gnu/gcc architecture: x86_64 - runner: ubuntu-latest - compiler: gcc - - target: x86_64-unknown-linux-gnu/clang - architecture: x86_64 - runner: ubuntu-latest - compiler: clang + runner: ubuntu-24.04 - target: aarch64-unknown-linux-gnu/gcc architecture: aarch64 - runner: ubuntu-latest - compiler: gcc - - target: aarch64-unknown-linux-gnu/clang - architecture: aarch64 - runner: ubuntu-latest - compiler: clang - env: - CC: ${{ matrix.compiler }} + runner: ubuntu-24.04-arm steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 with: python-version: '3.11' - - name: Native Windows - if: runner.os == 'Windows' && matrix.architecture != 'ARM64' - run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} - ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - # No PGO or tests (yet): - - name: Emulated Windows - if: runner.os == 'Windows' && matrix.architecture == 'ARM64' + # PCbuild downloads LLVM automatically: + - name: Windows + if: runner.os == 'Windows' run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Native macOS + - name: macOS if: runner.os == 'macOS' run: | brew update brew install llvm@${{ matrix.llvm }} - SDKROOT="$(xcrun --show-sdk-path)" \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + export SDKROOT="$(xcrun --show-sdk-path)" + # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to + # make sure we don't break downstream distributors (like uv): + export CFLAGS_JIT='-Werror=unguarded-availability' + export MACOSX_DEPLOYMENT_TARGET=10.15 + ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Native Linux - if: runner.os == 'Linux' && matrix.architecture == 'x86_64' + - name: Linux + if: runner.os == 'Linux' run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Emulated Linux - if: runner.os == 'Linux' && matrix.architecture != 'x86_64' - # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux. - run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --prefix="$(pwd)/../build" - make install --jobs 4 - make clean --jobs 4 - export HOST=${{ matrix.architecture }}-linux-gnu - sudo apt install --yes "gcc-$HOST" qemu-user - ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }} - ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }} - export QEMU_LD_PREFIX="/usr/$HOST" - CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \ - CPP="$CC --preprocess" \ - HOSTRUNNER=qemu-${{ matrix.architecture }} \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes - make all --jobs 4 - ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - jit-with-disabled-gil: name: Free-Threaded (Debug) needs: interpreter - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 21 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 with: python-version: '3.11' - name: Build with JIT enabled and GIL disabled run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 18 - export PATH="$(llvm-config-18 --bindir):$PATH" + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" ./configure --enable-experimental-jit --with-pydebug --disable-gil make all --jobs 4 - name: Run tests run: | ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + continue-on-error: true + + no-opt-jit: + name: JIT without optimizations (Debug) + needs: interpreter + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 21 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + - name: Build with JIT + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --enable-experimental-jit --with-pydebug + make all --jobs 4 + - name: Run tests without optimizations + run: | + PYTHON_UOPS_OPTIMIZE=0 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + tail-call-jit: + name: JIT with tail calling interpreter + needs: interpreter + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + llvm: + - 21 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + - name: Build with JIT and tailcall + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + CC=clang-${{ matrix.llvm }} ./configure --enable-experimental-jit --with-tail-call-interp --with-pydebug + make all --jobs 4 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ccde03f91983df..0ded53b00da0ef 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,8 +19,7 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 with: - python-version: "3.x" - - uses: pre-commit/action@v3.0.1 + persist-credentials: false + - uses: j178/prek-action@v1 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index e5b05302b5ac27..db363bef7a45ae 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,15 +8,29 @@ on: pull_request: paths: - ".github/workflows/mypy.yml" + - "Lib/_colorize.py" - "Lib/_pyrepl/**" - "Lib/test/libregrtest/**" + - "Lib/tomllib/**" + - "Misc/mypy/**" + - "Tools/build/check_extension_modules.py" + - "Tools/build/check_warnings.py" + - "Tools/build/compute-changes.py" + - "Tools/build/consts_getter.py" + - "Tools/build/deepfreeze.py" + - "Tools/build/generate-build-details.py" - "Tools/build/generate_sbom.py" + - "Tools/build/generate_stdlib_module_names.py" + - "Tools/build/mypy.ini" + - "Tools/build/umarshal.py" + - "Tools/build/update_file.py" + - "Tools/build/verify_ensurepip_wheels.py" - "Tools/cases_generator/**" + - "Tools/check-c-api-docs/**" - "Tools/clinic/**" - "Tools/jit/**" - "Tools/peg_generator/**" - "Tools/requirements-dev.txt" - - "Tools/wasm/**" workflow_dispatch: permissions: @@ -33,28 +47,32 @@ concurrency: jobs: mypy: + name: Run mypy on ${{ matrix.target }} + runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false matrix: target: [ "Lib/_pyrepl", "Lib/test/libregrtest", + "Lib/tomllib", "Tools/build", "Tools/cases_generator", + "Tools/check-c-api-docs", "Tools/clinic", "Tools/jit", "Tools/peg_generator", - "Tools/wasm", ] - name: Run mypy on ${{ matrix.target }} - runs-on: ubuntu-latest - timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 with: python-version: "3.13" cache: pip cache-dependency-path: Tools/requirements-dev.txt - run: pip install -r Tools/requirements-dev.txt + - run: python3 Misc/mypy/make_symlinks.py --symlink - run: mypy --config-file ${{ matrix.target }}/mypy.ini diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 9f1a8a824e5f19..b25750f0897de2 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: node-version: 20 - run: npm install mailgun.js form-data - name: Send notification - uses: actions/github-script@v7 + uses: actions/github-script@v8 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index bfc5a0874281bd..0b64367e6c4562 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -5,6 +5,7 @@ apt-get -yq install \ build-essential \ pkg-config \ ccache \ + cmake \ gdb \ lcov \ libb2-dev \ @@ -13,11 +14,11 @@ apt-get -yq install \ libgdbm-dev \ libgdbm-compat-dev \ liblzma-dev \ - libmpdec-dev \ libncurses5-dev \ libreadline6-dev \ libsqlite3-dev \ libssl-dev \ + libzstd-dev \ lzma \ lzma-dev \ strace \ @@ -25,3 +26,10 @@ apt-get -yq install \ uuid-dev \ xvfb \ zlib1g-dev + +# Workaround missing libmpdec-dev on ubuntu 24.04: +# https://launchpad.net/~ondrej/+archive/ubuntu/php +# https://deb.sury.org/ +sudo add-apt-repository ppa:ondrej/php +apt-get update +apt-get -yq install libmpdec-dev diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml deleted file mode 100644 index 066d8593a70cf6..00000000000000 --- a/.github/workflows/project-updater.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Update GH projects - -on: - issues: - types: - - opened - - labeled - -permissions: - contents: read - -jobs: - add-to-project: - name: Add issues to projects - runs-on: ubuntu-latest - timeout-minutes: 10 - strategy: - matrix: - include: - # if an issue has any of these labels, it will be added - # to the corresponding project - - { project: 2, label: "release-blocker, deferred-blocker" } - - { project: 32, label: sprint } - - steps: - - uses: actions/add-to-project@v1.0.0 - with: - project-url: https://github.com/orgs/python/projects/${{ matrix.project }} - github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} - labeled: ${{ matrix.label }} diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index bbedd22cc6d189..7e534c58c798d1 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,15 +4,13 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] -permissions: - issues: write - pull-requests: write - jobs: label-dnm: name: DO-NOT-MERGE if: github.repository_owner == 'python' runs-on: ubuntu-latest + permissions: + pull-requests: read timeout-minutes: 10 steps: @@ -28,6 +26,8 @@ jobs: name: Unresolved review if: github.repository_owner == 'python' runs-on: ubuntu-latest + permissions: + pull-requests: read timeout-minutes: 10 steps: diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml deleted file mode 100644 index 1a6fd33186840c..00000000000000 --- a/.github/workflows/reusable-change-detection.yml +++ /dev/null @@ -1,156 +0,0 @@ -name: Reusable change detection - -on: # yamllint disable-line rule:truthy - workflow_call: - outputs: - # Some of the referenced steps set outputs conditionally and there may be - # cases when referencing them evaluates to empty strings. It is nice to - # work with proper booleans so they have to be evaluated through JSON - # conversion in the expressions. However, empty strings used like that - # may trigger all sorts of undefined and hard-to-debug behaviors in - # GitHub Actions CI/CD. To help with this, all of the outputs set here - # that are meant to be used as boolean flags (and not arbitrary strings), - # MUST have fallbacks with default values set. A common pattern would be - # to add ` || false` to all such expressions here, in the output - # definitions. They can then later be safely used through the following - # idiom in job conditionals and other expressions. Here's some examples: - # - # if: fromJSON(needs.change-detection.outputs.run-docs) - # - # ${{ - # fromJSON(needs.change-detection.outputs.run-tests) - # && 'truthy-branch' - # || 'falsy-branch' - # }} - # - config_hash: - description: Config hash value for use in cache keys - value: ${{ jobs.compute-changes.outputs.config-hash }} # str - run-docs: - description: Whether to build the docs - value: ${{ jobs.compute-changes.outputs.run-docs || false }} # bool - run_tests: - description: Whether to run the regular tests - value: ${{ jobs.compute-changes.outputs.run-tests || false }} # bool - run-win-msi: - description: Whether to run the MSI installer smoke tests - value: >- # bool - ${{ jobs.compute-changes.outputs.run-win-msi || false }} - run_hypothesis: - description: Whether to run the Hypothesis tests - value: >- # bool - ${{ jobs.compute-changes.outputs.run-hypothesis || false }} - run_cifuzz: - description: Whether to run the CIFuzz job - value: >- # bool - ${{ jobs.compute-changes.outputs.run-cifuzz || false }} - -jobs: - compute-changes: - name: Compute changed files - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - config-hash: ${{ steps.config-hash.outputs.hash }} - run-cifuzz: ${{ steps.check.outputs.run-cifuzz }} - run-docs: ${{ steps.docs-changes.outputs.run-docs }} - run-hypothesis: ${{ steps.check.outputs.run-hypothesis }} - run-tests: ${{ steps.check.outputs.run-tests }} - run-win-msi: ${{ steps.win-msi-changes.outputs.run-win-msi }} - steps: - - run: >- - echo '${{ github.event_name }}' - - uses: actions/checkout@v4 - - name: Check for source changes - id: check - run: | - if [ -z "$GITHUB_BASE_REF" ]; then - echo "run-tests=true" >> "$GITHUB_OUTPUT" - else - git fetch origin "$GITHUB_BASE_REF" --depth=1 - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more - # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), - # but it requires to download more commits (this job uses - # "git fetch --depth=1"). - # - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git - # 2.26, but Git 2.28 is stricter and fails with "no merge base". - # - # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on - # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF - # into the PR branch anyway. - # - # https://github.com/python/core-workflow/issues/373 - git diff --name-only "origin/$GITHUB_BASE_REF.." | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$|\.md$|mypy\.ini$)' && echo "run-tests=true" >> "$GITHUB_OUTPUT" || true - fi - - # Check if we should run hypothesis tests - GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} - echo "$GIT_BRANCH" - if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then - echo "Branch too old for hypothesis tests" - echo "run-hypothesis=false" >> "$GITHUB_OUTPUT" - else - echo "Run hypothesis tests" - echo "run-hypothesis=true" >> "$GITHUB_OUTPUT" - fi - - # oss-fuzz maintains a configuration for fuzzing the main branch of - # CPython, so CIFuzz should be run only for code that is likely to be - # merged into the main branch; compatibility with older branches may - # be broken. - FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)' - if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only "origin/$GITHUB_BASE_REF.." | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then - # The tests are pretty slow so they are executed only for PRs - # changing relevant files. - echo "Run CIFuzz tests" - echo "run-cifuzz=true" >> "$GITHUB_OUTPUT" - else - echo "Branch too old for CIFuzz tests; or no C files were changed" - echo "run-cifuzz=false" >> "$GITHUB_OUTPUT" - fi - - name: Compute hash for config cache key - id: config-hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" - - name: Get a list of the changed documentation-related files - if: github.event_name == 'pull_request' - id: changed-docs-files - uses: Ana06/get-changed-files@v2.3.0 - with: - filter: | - Doc/** - Misc/** - .github/workflows/reusable-docs.yml - format: csv # works for paths with spaces - - name: Check for docs changes - # We only want to run this on PRs when related files are changed, - # or when user triggers manual workflow run. - if: >- - ( - github.event_name == 'pull_request' - && steps.changed-docs-files.outputs.added_modified_renamed != '' - ) || github.event_name == 'workflow_dispatch' - id: docs-changes - run: | - echo "run-docs=true" >> "${GITHUB_OUTPUT}" - - name: Get a list of the MSI installer-related files - if: github.event_name == 'pull_request' - id: changed-win-msi-files - uses: Ana06/get-changed-files@v2.3.0 - with: - filter: | - Tools/msi/** - .github/workflows/reusable-windows-msi.yml - format: csv # works for paths with spaces - - name: Check for changes in MSI installer-related files - # We only want to run this on PRs when related files are changed, - # or when user triggers manual workflow run. - if: >- - ( - github.event_name == 'pull_request' - && steps.changed-win-msi-files.outputs.added_modified_renamed != '' - ) || github.event_name == 'workflow_dispatch' - id: win-msi-changes - run: | - echo "run-win-msi=true" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/reusable-check-c-api-docs.yml b/.github/workflows/reusable-check-c-api-docs.yml new file mode 100644 index 00000000000000..bab1ca67d538ad --- /dev/null +++ b/.github/workflows/reusable-check-c-api-docs.yml @@ -0,0 +1,25 @@ +name: Reusable C API Docs Check + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + check-c-api-docs: + name: 'Check if all C APIs are documented' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Check for undocumented C APIs + run: python Tools/check-c-api-docs/main.py diff --git a/.github/workflows/reusable-cifuzz.yml b/.github/workflows/reusable-cifuzz.yml new file mode 100644 index 00000000000000..1986f5fb2cc640 --- /dev/null +++ b/.github/workflows/reusable-cifuzz.yml @@ -0,0 +1,46 @@ +# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ +name: Reusable CIFuzz + +on: + workflow_call: + inputs: + oss-fuzz-project-name: + description: OSS-Fuzz project name + required: true + type: string + sanitizer: + description: OSS-Fuzz sanitizer + required: true + type: string + +jobs: + cifuzz: + name: ${{ inputs.oss-fuzz-project-name }} (${{ inputs.sanitizer }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Build fuzzers (${{ inputs.sanitizer }}) + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + sanitizer: ${{ inputs.sanitizer }} + - name: Run fuzzers (${{ inputs.sanitizer }}) + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + fuzz-seconds: 600 + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + output-sarif: true + sanitizer: ${{ inputs.sanitizer }} + - name: Upload crash + if: failure() && steps.build.outcome == 'success' + uses: actions/upload-artifact@v6 + with: + name: ${{ inputs.sanitizer }}-artifacts + path: ./out/artifacts + - name: Upload SARIF + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml new file mode 100644 index 00000000000000..d958d729168e23 --- /dev/null +++ b/.github/workflows/reusable-context.yml @@ -0,0 +1,122 @@ +name: Reusable build context + +on: # yamllint disable-line rule:truthy + workflow_call: + outputs: + # Every referenced step MUST always set its output variable, + # either via ``Tools/build/compute-changes.py`` or in this workflow file. + # Boolean outputs (generally prefixed ``run-``) can then later be used + # safely through the following idiom in job conditionals and other + # expressions. Here's some examples: + # + # if: fromJSON(needs.build-context.outputs.run-tests) + # + # ${{ + # fromJSON(needs.build-context.outputs.run-tests) + # && 'truthy-branch' + # || 'falsy-branch' + # }} + # + run-android: + description: Whether to run the Android tests + value: ${{ jobs.compute-changes.outputs.run-android }} # bool + run-ci-fuzz: + description: Whether to run the CIFuzz job for 'cpython' fuzzer + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool + run-ci-fuzz-stdlib: + description: Whether to run the CIFuzz job for 'python3-libraries' fuzzer + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz-stdlib }} # bool + run-docs: + description: Whether to build the docs + value: ${{ jobs.compute-changes.outputs.run-docs }} # bool + run-ios: + description: Whether to run the iOS tests + value: ${{ jobs.compute-changes.outputs.run-ios }} # bool + run-macos: + description: Whether to run the macOS tests + value: ${{ jobs.compute-changes.outputs.run-macos }} # bool + run-tests: + description: Whether to run the regular tests + value: ${{ jobs.compute-changes.outputs.run-tests }} # bool + run-ubuntu: + description: Whether to run the Ubuntu tests + value: ${{ jobs.compute-changes.outputs.run-ubuntu }} # bool + run-wasi: + description: Whether to run the WASI tests + value: ${{ jobs.compute-changes.outputs.run-wasi }} # bool + run-windows-msi: + description: Whether to run the MSI installer smoke tests + value: ${{ jobs.compute-changes.outputs.run-windows-msi }} # bool + run-windows-tests: + description: Whether to run the Windows tests + value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool + +jobs: + compute-changes: + name: Create context from changed files + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + run-android: ${{ steps.changes.outputs.run-android }} + run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} + run-ci-fuzz-stdlib: ${{ steps.changes.outputs.run-ci-fuzz-stdlib }} + run-docs: ${{ steps.changes.outputs.run-docs }} + run-ios: ${{ steps.changes.outputs.run-ios }} + run-macos: ${{ steps.changes.outputs.run-macos }} + run-tests: ${{ steps.changes.outputs.run-tests }} + run-ubuntu: ${{ steps.changes.outputs.run-ubuntu }} + run-wasi: ${{ steps.changes.outputs.run-wasi }} + run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} + run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }} + steps: + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3" + + - run: >- + echo '${{ github.event_name }}' + + - uses: actions/checkout@v6 + with: + persist-credentials: false + ref: >- + ${{ + github.event_name == 'pull_request' + && github.event.pull_request.head.sha + || '' + }} + + # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 + - name: Fetch commits to get branch diff + if: github.event_name == 'pull_request' + run: | + set -eux + + # Fetch enough history to find a common ancestor commit (aka merge-base): + git fetch origin "${refspec_pr}" --depth=$(( commits + 1 )) \ + --no-tags --prune --no-recurse-submodules + + # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from): + COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 "${branch_pr}" ) + DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) + + # Get all commits since that commit date from the base branch (eg: main): + git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ + --no-tags --prune --no-recurse-submodules + env: + branch_pr: 'origin/${{ github.event.pull_request.head.ref }}' + commits: ${{ github.event.pull_request.commits }} + refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}' + refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' + + # We only want to run tests on PRs when related files are changed, + # or when someone triggers a manual workflow run. + - name: Compute changed files + id: changes + run: python Tools/build/compute-changes.py + env: + GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }} + CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 39a97392e898aa..fc68c040fca059 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -15,19 +15,21 @@ env: FORCE_COLOR: 1 jobs: - build_doc: + build-doc: name: 'Docs' runs-on: ubuntu-latest timeout-minutes: 60 env: branch_base: 'origin/${{ github.event.pull_request.base.ref }}' branch_pr: 'origin/${{ github.event.pull_request.head.ref }}' + commits: ${{ github.event.pull_request.commits }} refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}' refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: + persist-credentials: false ref: >- ${{ github.event_name == 'pull_request' @@ -39,18 +41,18 @@ jobs: if: github.event_name == 'pull_request' run: | # Fetch enough history to find a common ancestor commit (aka merge-base): - git fetch origin ${{ env.refspec_pr }} --depth=$(( ${{ github.event.pull_request.commits }} + 1 )) \ + git fetch origin "${refspec_pr}" --depth=$(( commits + 1 )) \ --no-tags --prune --no-recurse-submodules # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from): - COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 ${{ env.branch_pr }} ) + COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 "${branch_pr}" ) DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) # Get all commits since that commit date from the base branch (eg: master or main): - git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \ + git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3' cache: 'pip' @@ -63,43 +65,27 @@ jobs: continue-on-error: true run: | set -Eeuo pipefail - # Build docs with the '-n' (nit-picky) option; write warnings to file - make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going -w sphinx-warnings.txt" html + # Build docs with the nit-picky option; write warnings to file + make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --warning-file sphinx-warnings.txt" html - name: 'Check warnings' if: github.event_name == 'pull_request' run: | python Doc/tools/check-warnings.py \ - --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \ + --annotate-diff "${branch_base}" "${branch_pr}" \ --fail-if-regression \ --fail-if-improved \ --fail-if-new-news-nit - # This build doesn't use problem matchers or check annotations - build_doc_oldest_supported_sphinx: - name: 'Docs (Oldest Sphinx)' - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - - name: 'Set up Python' - uses: actions/setup-python@v5 - with: - python-version: '3.13' # known to work with Sphinx 7.2.6 - cache: 'pip' - cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt' - - name: 'Install build dependencies' - run: make -C Doc/ venv REQUIREMENTS="requirements-oldest-sphinx.txt" - - name: 'Build HTML documentation' - run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html - # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: name: 'Doctest' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/cache@v5 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} @@ -115,4 +101,31 @@ jobs: run: make -C Doc/ PYTHON=../python venv # Use "xvfb-run" since some doctest tests open GUI windows - name: 'Run documentation doctest' - run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="-W --keep-going" doctest + run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="--fail-on-warning" doctest + + check-epub: + name: 'Check EPUB' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: 'Set up Python' + uses: actions/setup-python@v6 + with: + python-version: '3' + cache: 'pip' + cache-dependency-path: 'Doc/requirements.txt' + - name: 'Install build dependencies' + run: | + make -C Doc/ venv + python -m pip install epubcheck + - name: 'Build EPUB documentation' + run: make -C Doc/ PYTHON=../python epub + - name: 'Run epubcheck' + continue-on-error: true + run: epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt + - run: cat Doc/epubcheck.txt + - name: 'Check for fatal errors in EPUB' + run: python Doc/tools/check-epub.py diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index bfa5f69defb87c..7eef66bd9d9324 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -3,9 +3,6 @@ name: Reusable macOS on: workflow_call: inputs: - config_hash: - required: true - type: string free-threading: required: false type: boolean @@ -15,9 +12,13 @@ on: required: true type: string +env: + FORCE_COLOR: 1 + jobs: - build_macos: + build-macos: name: build and test (${{ inputs.os }}) + runs-on: ${{ inputs.os }} timeout-minutes: 60 env: HOMEBREW_NO_ANALYTICS: 1 @@ -26,20 +27,20 @@ jobs: HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux - runs-on: ${{ inputs.os }} steps: - - uses: actions/checkout@v4 - - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 + - uses: actions/checkout@v6 with: - path: config.cache - key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Install Homebrew dependencies - run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk make + run: | + brew install pkg-config openssl@3.0 xz gdbm tcl-tk@9 make + # Because alternate versions are not symlinked into place by default: + brew link --overwrite tcl-tk@9 - name: Configure CPython run: | + MACOSX_DEPLOYMENT_TARGET=10.15 \ GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ ./configure \ @@ -51,15 +52,15 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - if : ${{ inputs.free-threading || inputs.os != 'macos-13' }} + if : ${{ inputs.free-threading || inputs.os != 'macos-15-intel' }} run: gmake -j8 - name: Build CPython for compiler warning check - if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} run: >- python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output_macos.txt @@ -69,4 +70,4 @@ jobs: --fail-on-improvement --path-prefix="./" - name: Tests - run: make test + run: make ci diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml new file mode 100644 index 00000000000000..49876cf49260d9 --- /dev/null +++ b/.github/workflows/reusable-san.yml @@ -0,0 +1,111 @@ +name: Reusable Sanitizer + +on: + workflow_call: + inputs: + sanitizer: + required: true + type: string + free-threading: + description: Whether to use free-threaded mode + required: false + type: boolean + default: false + +env: + FORCE_COLOR: 1 + +jobs: + build-san-reusable: + name: >- + ${{ inputs.sanitizer }}${{ + inputs.free-threading + && ' (free-threading)' + || '' + }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + # Install clang + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + + if [ "${SANITIZER}" = "TSan" ]; then + sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 + sudo update-alternatives --set clang /usr/bin/clang-17 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-17 + # Reduce ASLR to avoid TSan crashing + sudo sysctl -w vm.mmap_rnd_bits=28 + else + sudo ./llvm.sh 20 + fi + + - name: Sanitizer option setup + run: | + if [ "${SANITIZER}" = "TSan" ]; then + echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ + fromJSON(inputs.free-threading) + && '_free_threading' + || '' + }}.txt handle_segv=0" >> "$GITHUB_ENV" + else + echo "UBSAN_OPTIONS=${SAN_LOG_OPTION}" >> "$GITHUB_ENV" + fi + echo "CC=clang" >> "$GITHUB_ENV" + echo "CXX=clang++" >> "$GITHUB_ENV" + env: + SANITIZER: ${{ inputs.sanitizer }} + SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure CPython + run: >- + ./configure + --config-cache + ${{ + inputs.sanitizer == 'TSan' + && '--with-thread-sanitizer' + || '--with-undefined-behavior-sanitizer' + }} + --with-pydebug + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: >- + ./python -m test + ${{ inputs.sanitizer == 'TSan' && '--tsan' || '' }} + -j4 + - name: Parallel tests + if: >- + inputs.sanitizer == 'TSan' + && fromJSON(inputs.free-threading) + run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 + - name: Display logs + if: always() + run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 + - name: Archive logs + if: always() + uses: actions/upload-artifact@v6 + with: + name: >- + ${{ inputs.sanitizer }}-logs-${{ + fromJSON(inputs.free-threading) + && 'free-threading' + || 'default' + }} + path: san_log.* + if-no-files-found: ignore diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml deleted file mode 100644 index 65072efa8e9023..00000000000000 --- a/.github/workflows/reusable-tsan.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Reusable Thread Sanitizer - -on: - workflow_call: - inputs: - config_hash: - required: true - type: string - options: - required: true - type: string - suppressions_path: - description: 'A repo relative path to the suppressions file' - required: true - type: string - tsan_logs_artifact_name: - description: 'Name of the TSAN logs artifact. Must be unique for each job.' - required: true - type: string - -jobs: - build_tsan_reusable: - name: 'Thread sanitizer' - runs-on: ubuntu-22.04 - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - - name: Install Dependencies - run: | - sudo ./.github/workflows/posix-deps-apt.sh - # Install clang-18 - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 - # Reduce ASLR to avoid TSAN crashing - sudo sysctl -w vm.mmap_rnd_bits=28 - - name: TSAN Option Setup - run: | - echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }} handle_segv=0" >> "$GITHUB_ENV" - echo "CC=clang" >> "$GITHUB_ENV" - echo "CXX=clang++" >> "$GITHUB_ENV" - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: ${{ inputs.options }} - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: ./python -m test --tsan -j4 - - name: Display TSAN logs - if: always() - run: find "${GITHUB_WORKSPACE}" -name 'tsan_log.*' | xargs head -n 1000 - - name: Archive TSAN logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ inputs.tsan_logs_artifact_name }} - path: tsan_log.* - if-no-files-found: ignore diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 18e2e471e60e8d..ad725e92f2b20f 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -3,35 +3,47 @@ name: Reusable Ubuntu on: workflow_call: inputs: - config_hash: - required: true - type: string + bolt-optimizations: + description: Whether to enable BOLT optimizations + required: false + type: boolean + default: false free-threading: description: Whether to use free-threaded mode required: false type: boolean default: false + os: + description: OS to run the job + required: true + type: string + +env: + FORCE_COLOR: 1 jobs: - build_ubuntu_reusable: - name: 'build and test' + build-ubuntu-reusable: + name: build and test (${{ inputs.os }}) + runs-on: ${{ inputs.os }} timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-22.04] env: - FORCE_COLOR: 1 - OPENSSL_VER: 3.0.15 + OPENSSL_VER: 3.0.18 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + with: + persist-credentials: false - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Install Clang and BOLT + if: ${{ fromJSON(inputs.bolt-optimizations) }} + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 19 + sudo apt-get install bolt-19 + echo PATH="$(llvm-config-19 --bindir):$PATH" >> $GITHUB_ENV - name: Configure OpenSSL env vars run: | echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" @@ -39,21 +51,16 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -63,15 +70,13 @@ jobs: - name: Bind mount sources read-only run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version - run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} + # `test_unpickle_module_race` writes to the source directory, which is + # read-only during builds — so we exclude it from profiling with BOLT. run: >- + PROFILE_TASK='-m test --pgo --ignore test_unpickle_module_race' ../cpython-ro-srcdir/configure --config-cache --with-pydebug @@ -79,14 +84,15 @@ jobs: --enable-safety --with-openssl="$OPENSSL_DIR" ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + ${{ fromJSON(inputs.bolt-optimizations) && '--enable-bolt' || '' }} - name: Build CPython out-of-tree if: ${{ inputs.free-threading }} working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 + run: make -j - name: Build CPython out-of-tree (for compiler warning check) - if: ${{ !inputs.free-threading}} + if: ${{ !inputs.free-threading }} working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: set -o pipefail; make -j4 --output-sync 2>&1 | tee compiler_output_ubuntu.txt + run: set -o pipefail; make -j --output-sync 2>&1 | tee compiler_output_ubuntu.txt - name: Display build info working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo @@ -94,7 +100,7 @@ jobs: if: ${{ !inputs.free-threading }} run: >- python Tools/build/check_warnings.py - --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt + --compiler-output-file-path="${CPYTHON_BUILDDIR}/compiler_output_ubuntu.txt" --warning-ignore-file-path "${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu" --compiler-output-type=gcc --fail-on-regression @@ -105,4 +111,4 @@ jobs: run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make test + run: xvfb-run make ci diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index abc617a317cc0f..3c81f6ef82dc8c 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -2,24 +2,25 @@ name: Reusable WASI on: workflow_call: - inputs: - config_hash: - required: true - type: string + +env: + FORCE_COLOR: 1 jobs: - build_wasi_reusable: + build-wasi-reusable: name: 'build and test' + runs-on: ubuntu-24.04-arm timeout-minutes: 60 - runs-on: ubuntu-22.04 env: - WASMTIME_VERSION: 22.0.0 - WASI_SDK_VERSION: 24 + WASMTIME_VERSION: 38.0.3 + WASI_SDK_VERSION: 29 WASI_SDK_PATH: /opt/wasi-sdk CROSS_BUILD_PYTHON: cross-build/build - CROSS_BUILD_WASI: cross-build/wasm32-wasi + CROSS_BUILD_WASI: cross-build/wasm32-wasip1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + with: + persist-credentials: false # No problem resolver registered as one doesn't currently exist for Clang. - name: "Install wasmtime" uses: bytecodealliance/actions/wasmtime/setup@v1 @@ -27,51 +28,34 @@ jobs: version: ${{ env.WASMTIME_VERSION }} - name: "Restore WASI SDK" id: cache-wasi-sdk - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ${{ env.WASI_SDK_PATH }} key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} - - name: "Install WASI SDK" + - name: "Install WASI SDK" # Hard-coded to x64. if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' run: | - mkdir ${{ env.WASI_SDK_PATH }} && \ - curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-x86_64-linux.tar.gz | \ - tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip - - name: "Configure ccache action" - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" + mkdir "${WASI_SDK_PATH}" && \ + curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-arm64-linux.tar.gz" | \ + tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip - name: "Add ccache to PATH" run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: "Install Python" - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.x' - - name: "Restore Python build config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. - # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables. - # (Make sure to keep the key in sync with the other config.cache step below.) - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + - name: "Runner image version" + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: "Configure build Python" - run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + run: python3 Platforms/WASI configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi.py make-build-python - - name: "Restore host config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_WASI }}/config.cache - # Should be kept in sync with the other config.cache step above. - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + run: python3 Platforms/WASI make-build-python - name: "Configure host" # `--with-pydebug` inferred from configure-build-python - run: python3 Tools/wasm/wasi.py configure-host -- --config-cache + run: python3 Platforms/WASI configure-host -- --config-cache - name: "Make host" - run: python3 Tools/wasm/wasi.py make-host + run: python3 Platforms/WASI make-host - name: "Display build info" - run: make --directory ${{ env.CROSS_BUILD_WASI }} pythoninfo + run: make --directory "${CROSS_BUILD_WASI}" pythoninfo - name: "Test" - run: make --directory ${{ env.CROSS_BUILD_WASI }} test + run: make --directory "${CROSS_BUILD_WASI}" test diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index abdb1a1982fef8..c7611804369600 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -11,14 +11,21 @@ on: permissions: contents: read +env: + FORCE_COLOR: 1 + jobs: build: name: installer for ${{ inputs.arch }} - runs-on: windows-latest + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2022' }} timeout-minutes: 60 env: + ARCH: ${{ inputs.arch }} IncludeFreethreaded: true steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + with: + persist-credentials: false - name: Build CPython installer - run: .\Tools\msi\build.bat --doc -${{ inputs.arch }} + run: ./Tools/msi/build.bat --doc -"${ARCH}" + shell: bash diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index dcfc62d7f5d145..82ea819867ef6d 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -14,34 +14,37 @@ on: default: false env: + FORCE_COLOR: 1 IncludeUwp: >- true jobs: build: - name: >- - build${{ inputs.arch != 'arm64' && ' and test' || '' }} - (${{ inputs.arch }}) - runs-on: windows-latest + name: Build and test (${{ inputs.arch }}) + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2022' }} timeout-minutes: 60 + env: + ARCH: ${{ inputs.arch }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + with: + persist-credentials: false - name: Register MSVC problem matcher if: inputs.arch != 'Win32' run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython run: >- - .\PCbuild\build.bat + .\\PCbuild\\build.bat -e -d -v - -p ${{ inputs.arch }} + -p "${ARCH}" ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + shell: bash - name: Display build info - if: inputs.arch != 'arm64' - run: .\python.bat -m test.pythoninfo + run: .\\python.bat -m test.pythoninfo - name: Tests - if: inputs.arch != 'arm64' run: >- - .\PCbuild\rt.bat - -p ${{ inputs.arch }} + .\\PCbuild\\rt.bat + -p "${ARCH}" -d -q --fast-ci ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + shell: bash diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index f97587e68cbbe4..febb2dd823a8fe 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,14 +4,12 @@ on: schedule: - cron: "0 */6 * * *" -permissions: - pull-requests: write - jobs: stale: if: github.repository_owner == 'python' - runs-on: ubuntu-latest + permissions: + pull-requests: write timeout-minutes: 10 steps: diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml new file mode 100644 index 00000000000000..335e1a93dce4ea --- /dev/null +++ b/.github/workflows/tail-call.yml @@ -0,0 +1,131 @@ +name: Tail calling interpreter +on: + pull_request: + paths: + - '.github/workflows/tail-call.yml' + - 'Python/bytecodes.c' + - 'Python/ceval.c' + - 'Python/ceval_macros.h' + - 'Python/generated_cases.c.h' + push: + paths: + - '.github/workflows/tail-call.yml' + - 'Python/bytecodes.c' + - 'Python/ceval.c' + - 'Python/ceval_macros.h' + - 'Python/generated_cases.c.h' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 + +jobs: + tail-call: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + target: +# Un-comment as we add support for more platforms for tail-calling interpreters. +# - i686-pc-windows-msvc/msvc + - x86_64-pc-windows-msvc/msvc +# - aarch64-pc-windows-msvc/msvc + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + - x86_64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/gcc + - free-threading + llvm: + - 20 + include: +# - target: i686-pc-windows-msvc/msvc +# architecture: Win32 +# runner: windows-2022 + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-2022 +# - target: aarch64-pc-windows-msvc/msvc +# architecture: ARM64 +# runner: windows-2022 + - target: x86_64-apple-darwin/clang + architecture: x86_64 + runner: macos-15-intel + - target: aarch64-apple-darwin/clang + architecture: aarch64 + runner: macos-14 + - target: x86_64-unknown-linux-gnu/gcc + architecture: x86_64 + runner: ubuntu-24.04 + - target: aarch64-unknown-linux-gnu/gcc + architecture: aarch64 + runner: ubuntu-24.04-arm + - target: free-threading + architecture: x86_64 + runner: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + + - name: Native Windows MSVC (release) + if: runner.os == 'Windows' && matrix.architecture != 'ARM64' + shell: pwsh + run: | + choco install visualstudio2026buildtools --no-progress -y --force --params "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 --locale en-US --passive" + $env:PATH = "C:\Program Files (x86)\Microsoft Visual Studio\18\BuildTools\MSBuild\Current\bin;$env:PATH" + $env:PlatformToolset = "v145" + ./PCbuild/build.bat --tail-call-interp -c Release -p ${{ matrix.architecture }} + ./PCbuild/rt.bat -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + # No tests (yet): + - name: Emulated Windows Clang (release) + if: runner.os == 'Windows' && matrix.architecture == 'ARM64' + shell: pwsh + run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 + $env:PlatformToolset = "clangcl" + $env:LLVMToolsVersion = "${{ matrix.llvm }}.1.0" + $env:LLVMInstallDir = "C:\Program Files\LLVM" + ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }} + + - name: Native macOS (release) + if: runner.os == 'macOS' + run: | + brew update + brew install llvm@${{ matrix.llvm }} + export SDKROOT="$(xcrun --show-sdk-path)" + export PATH="/usr/local/opt/llvm@${{ matrix.llvm }}/bin:$PATH" + export PATH="/opt/homebrew/opt/llvm@${{ matrix.llvm }}/bin:$PATH" + CC=clang-20 ./configure --with-tail-call-interp + make all --jobs 4 + ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + - name: Native Linux (debug) + if: runner.os == 'Linux' && matrix.target != 'free-threading' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + CC=clang-20 ./configure --with-tail-call-interp --with-pydebug + make all --jobs 4 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + - name: Native Linux with free-threading (release) + if: matrix.target == 'free-threading' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + CC=clang-20 ./configure --with-tail-call-interp --disable-gil + make all --jobs 4 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 83b007f1c9c2ef..135979078710cc 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -25,8 +25,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 with: python-version: '3' - name: Compare checksum of bundled wheels to the ones published on PyPI diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 00000000000000..fab3abcb355dfe --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,10 @@ +# Configuration for the zizmor static analysis tool, run via prek in CI +# https://woodruffw.github.io/zizmor/configuration/ +rules: + dangerous-triggers: + ignore: + - documentation-links.yml + unpinned-uses: + config: + policies: + "*": ref-pin diff --git a/.gitignore b/.gitignore index 8872e9d5508ff1..e234d86e8d5532 100644 --- a/.gitignore +++ b/.gitignore @@ -38,12 +38,14 @@ tags TAGS .vs/ .vscode/ +.cache/ gmon.out .coverage .mypy_cache/ .pytest_cache/ .ruff_cache/ .DS_Store +.pixi/ *.exe @@ -70,16 +72,15 @@ Lib/test/data/* /Makefile /Makefile.pre /iOSTestbed.* -iOS/Frameworks/ -iOS/Resources/Info.plist -iOS/testbed/build -iOS/testbed/Python.xcframework/ios-*/bin -iOS/testbed/Python.xcframework/ios-*/include -iOS/testbed/Python.xcframework/ios-*/lib -iOS/testbed/Python.xcframework/ios-*/Python.framework -iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace -iOS/testbed/iOSTestbed.xcodeproj/xcuserdata -iOS/testbed/iOSTestbed.xcodeproj/xcshareddata +Apple/iOS/Frameworks/ +Apple/iOS/Resources/Info.plist +Apple/testbed/build +Apple/testbed/Python.xcframework/*/bin +Apple/testbed/Python.xcframework/*/include +Apple/testbed/Python.xcframework/*/lib +Apple/testbed/Python.xcframework/*/Python.framework +Apple/testbed/*Testbed.xcodeproj/project.xcworkspace +Apple/testbed/*Testbed.xcodeproj/xcuserdata Mac/Makefile Mac/PythonLauncher/Info.plist Mac/PythonLauncher/Makefile @@ -130,18 +131,19 @@ Tools/unicode/data/ /autom4te.cache /build/ /builddir/ +/compile_commands.json /config.cache /config.log /config.status /config.status.lineno -# hendrikmuhs/ccache-action@v1 /.ccache /cross-build/ -/jit_stencils.h +/jit_stencils*.h /platform /profile-clean-stamp /profile-run-stamp /profile-bolt-stamp +/profile-gen-stamp /pybuilddir.txt /pyconfig.h /python-config @@ -169,5 +171,10 @@ Python/frozen_modules/MANIFEST /python !/Python/ +# People's custom https://docs.anthropic.com/en/docs/claude-code/memory configs. +/.claude/ +CLAUDE.local.md + +#### main branch only stuff below this line, things to backport go above. #### # main branch only: ABI files are not checked/maintained. Doc/data/python*.abi diff --git a/.mailmap b/.mailmap index 013c839ed6b7a4..8f11bebb18fc17 100644 --- a/.mailmap +++ b/.mailmap @@ -1,3 +1,4 @@ # This file sets the canonical name for contributors to the repository. # Documentation: https://git-scm.com/docs/gitmailmap +Willow Chargin <wchargin@gmail.com> Amethyst Reese <amethyst@n7.gg> <john@noswap.com> diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f1db7886d8261c..ed88e9ca81b49c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,43 +1,71 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.14.10 hooks: - - id: ruff + - id: ruff-check + name: Run Ruff (lint) on Apple/ + args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] + files: ^Apple/ + - id: ruff-check name: Run Ruff (lint) on Doc/ args: [--exit-non-zero-on-fix] files: ^Doc/ - - id: ruff + - id: ruff-check name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] files: ^Lib/test/ - - id: ruff - name: Run Ruff (lint) on Tools/build/check_warnings.py + - id: ruff-check + name: Run Ruff (lint) on Tools/build/ args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] - files: ^Tools/build/check_warnings.py - - id: ruff + files: ^Tools/build/ + - id: ruff-check + name: Run Ruff (lint) on Tools/i18n/ + args: [--exit-non-zero-on-fix, --config=Tools/i18n/.ruff.toml] + files: ^Tools/i18n/ + - id: ruff-check name: Run Ruff (lint) on Argument Clinic args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] files: ^Tools/clinic/|Lib/test/test_clinic.py + - id: ruff-check + name: Run Ruff (lint) on Tools/peg_generator/ + args: [--exit-non-zero-on-fix, --config=Tools/peg_generator/.ruff.toml] + files: ^Tools/peg_generator/ + - id: ruff-check + name: Run Ruff (lint) on Tools/wasm/ + args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ + - id: ruff-format + name: Run Ruff (format) on Apple/ + args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] + files: ^Apple - id: ruff-format name: Run Ruff (format) on Doc/ - args: [--check] + args: [--exit-non-zero-on-fix] files: ^Doc/ + - id: ruff-format + name: Run Ruff (format) on Tools/build/check_warnings.py + args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] + files: ^Tools/build/check_warnings.py + - id: ruff-format + name: Run Ruff (format) on Tools/wasm/ + args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.10.0 + rev: 25.12.0 hooks: - - id: black - name: Run Black on Tools/build/check_warnings.py - files: ^Tools/build/check_warnings.py - language_version: python3.12 - args: [--line-length=79] - id: black name: Run Black on Tools/jit/ files: ^Tools/jit/ - language_version: python3.12 + + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: v1.5.5 + hooks: + - id: remove-tabs + types: [python] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -45,33 +73,55 @@ repos: exclude: ^Lib/test/test_tomllib/ - id: check-yaml - id: end-of-file-fixer - types: [python] + types_or: [python, yaml] exclude: Lib/test/tokenizedata/coding20731.py + - id: end-of-file-fixer + files: '^\.github/CODEOWNERS$' - id: trailing-whitespace - types_or: [c, inc, python, rst] + types_or: [c, inc, python, rst, yaml] + - id: trailing-whitespace + files: '^\.github/CODEOWNERS|\.(gram)$' - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.4 + rev: 0.36.0 hooks: - id: check-dependabot - id: check-github-workflows + - id: check-readthedocs - repo: https://github.com/rhysd/actionlint - rev: v1.7.3 + rev: v1.7.9 hooks: - id: actionlint - args: [ - -ignore=1st argument of function call is not assignable, - -ignore=SC2(015|038|086|091|097|098|129|155), - ] + + - repo: https://github.com/woodruffw/zizmor-pre-commit + rev: v1.19.0 + hooks: + - id: zizmor - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v1.0.0 + rev: v1.0.2 hooks: - id: sphinx-lint args: [--enable=default-role] files: ^Doc/|^Misc/NEWS.d/ + - repo: local + hooks: + - id: blurb-no-space-c-api + name: Check C API news entries + language: fail + entry: Space found in path, move to Misc/NEWS.d/next/C_API/ + files: Misc/NEWS.d/next/C API/20.*.rst + + - repo: local + hooks: + - id: blurb-no-space-core-and-builtins + name: Check Core and Builtins news entries + language: fail + entry: Space found in path, move to Misc/NEWS.d/next/Core_and_Builtins/ + files: Misc/NEWS.d/next/Core and Builtins/20.*.rst + - repo: meta hooks: - id: check-hooks-apply diff --git a/.readthedocs.yml b/.readthedocs.yml index a57de00544e4e3..0a2c3f8345367f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -32,4 +32,3 @@ build: - make -C Doc venv html - mkdir _readthedocs - mv Doc/build/html _readthedocs/html - diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 00000000000000..1c015fa88415bc --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,12 @@ +# Default settings for Ruff in CPython + +# PYTHON_FOR_REGEN +target-version = "py310" + +# PEP 8 +line-length = 79 + +# Enable automatic fixes by default. +# To override this, use ``fix = false`` in a subdirectory's config file +# or ``--no-fix`` on the command line. +fix = true diff --git a/.well-known/funding-manifest-urls b/.well-known/funding-manifest-urls new file mode 100644 index 00000000000000..d56ca9eb52f004 --- /dev/null +++ b/.well-known/funding-manifest-urls @@ -0,0 +1 @@ +https://www.python.org/funding.json diff --git a/Android/README.md b/Android/README.md index 3daa545cc93746..9f71aeb934f386 100644 --- a/Android/README.md +++ b/Android/README.md @@ -1,19 +1,22 @@ # Python for Android -These instructions are only needed if you're planning to compile Python for -Android yourself. Most users should *not* need to do this. Instead, use one of -the tools listed in `Doc/using/android.rst`, which will provide a much easier -experience. +If you obtained this README as part of a release package, then the only +applicable sections are "Prerequisites", "Testing", and "Using in your own app". +If you obtained this README as part of the CPython source tree, then you can +also follow the other sections to compile Python for Android yourself. + +However, most app developers should not need to do any of these things manually. +Instead, use one of the tools listed +[here](https://docs.python.org/3/using/android.html), which will provide a much +easier experience. -## Prerequisites -First, make sure you have all the usual tools and libraries needed to build -Python for your development machine. +## Prerequisites -Second, you'll need an Android SDK. If you already have the SDK installed, -export the `ANDROID_HOME` environment variable to point at its location. -Otherwise, here's how to install it: +If you already have an Android SDK installed, export the `ANDROID_HOME` +environment variable to point at its location. Otherwise, here's how to install +it: * Download the "Command line tools" from <https://developer.android.com/studio>. * Create a directory `android-sdk/cmdline-tools`, and unzip the command line @@ -22,20 +25,23 @@ Otherwise, here's how to install it: `android-sdk/cmdline-tools/latest`. * `export ANDROID_HOME=/path/to/android-sdk` -The `android.py` script also requires the following commands to be on the `PATH`: +The `android.py` script will automatically use the SDK's `sdkmanager` to install +any packages it needs. + +The script also requires the following commands to be on the `PATH`: * `curl` * `java` (or set the `JAVA_HOME` environment variable) -* `tar` -* `unzip` ## Building Python can be built for Android on any POSIX platform supported by the Android -development tools, which currently means Linux or macOS. This involves doing a -cross-build where you use a "build" Python (for your development machine) to -help produce a "host" Python for Android. +development tools, which currently means Linux or macOS. + +First we'll make a "build" Python (for your development machine), then use it to +help produce a "host" Python for Android. So make sure you have all the usual +tools and libraries needed to build Python for your development machine. The easiest way to do a build is to use the `android.py` script. You can either have it perform the entire build process from start to finish in one step, or @@ -60,8 +66,8 @@ To do all steps in a single command, run: ./android.py build HOST ``` -In the end you should have a build Python in `cross-build/build`, and an Android -build in `cross-build/HOST`. +In the end you should have a build Python in `cross-build/build`, and a host +Python in `cross-build/HOST`. You can use `--` as a separator for any of the `configure`-related commands – including `build` itself – to pass arguments to the underlying `configure` @@ -73,14 +79,29 @@ call. For example, if you want a pydebug build that also caches the results from ``` +## Packaging + +After building an architecture as described in the section above, you can +package it for release with this command: + +```sh +./android.py package HOST +``` + +`HOST` is defined in the section above. + +This will generate a tarball in `cross-build/HOST/dist`, whose structure is +similar to the `Android` directory of the CPython source tree. + + ## Testing -The test suite can be run on Linux, macOS, or Windows: +The Python test suite can be run on Linux, macOS, or Windows. -* On Linux, the emulator needs access to the KVM virtualization interface, and - a DISPLAY environment variable pointing at an X server. -* On Windows, you won't be able to do the build on the same machine, so you'll - have to copy the `cross-build/HOST` directory from somewhere else. +On Linux, the emulator needs access to the KVM virtualization interface. This may +require adding your user to a group, or changing your udev rules. On GitHub +Actions, the test script will do this automatically using the commands shown +[here](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). The test suite can usually be run on a device with 2 GB of RAM, but this is borderline, so you may need to increase it to 4 GB. As of Android @@ -90,9 +111,16 @@ and find `hw.ramSize` in both config.ini and hardware-qemu.ini. Either set these manually to the same value, or use the Android Studio Device Manager, which will update both files. -Before running the test suite, follow the instructions in the previous section -to build the architecture you want to test. Then run the test script in one of -the following modes: +You can run the test suite either: + +* Within the CPython repository, after doing a build as described above. On + Windows, you won't be able to do the build on the same machine, so you'll have + to copy the `cross-build/HOST/prefix` directory from somewhere else. + +* Or by taking a release package built using the `package` command, extracting + it wherever you want, and using its own copy of `android.py`. + +The test script supports the following modes: * In `--connected` mode, it runs on a device or emulator you have already connected to the build machine. List the available devices with @@ -119,10 +147,10 @@ stderr. Add the `-v` option to also show Gradle output, and non-Python logcat messages. Any other arguments on the `android.py test` command line will be passed through -to `python -m test` – use `--` to separate them from android.py's own options. +to `python -m test` – use `--` to separate them from android.py's own options. See the [Python Developer's Guide](https://devguide.python.org/testing/run-write-tests/) for common options -– most of them will work on Android, except for those that involve subprocesses, +– most of them will work on Android, except for those that involve subprocesses, such as `-j`. Every time you run `android.py test`, changes in pure-Python files in the @@ -130,7 +158,11 @@ repository's `Lib` directory will be picked up immediately. Changes in C files, and architecture-specific files such as sysconfigdata, will not take effect until you re-run `android.py make-host` or `build`. +The testbed app can also be used to test third-party packages. For more details, +run `android.py test --help`, paying attention to the options `--site-packages`, +`--cwd`, `-c` and `-m`. + ## Using in your own app -See `Doc/using/android.rst`. +See https://docs.python.org/3/using/android.html. diff --git a/Android/android-env.sh b/Android/android-env.sh index a0f23ef8c9fc52..5859c0eac4a88f 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -1,10 +1,10 @@ # This script must be sourced with the following variables already set: -: ${ANDROID_HOME:?} # Path to Android SDK -: ${HOST:?} # GNU target triplet +: "${ANDROID_HOME:?}" # Path to Android SDK +: "${HOST:?}" # GNU target triplet # You may also override the following: -: ${api_level:=24} # Minimum Android API level the build will run on -: ${PREFIX:-} # Path in which to find required libraries +: "${ANDROID_API_LEVEL:=24}" # Minimum Android API level the build will run on +: "${PREFIX:-}" # Path in which to find required libraries # Print all messages on stderr so they're visible when running within build-wheel. @@ -24,26 +24,26 @@ fail() { # * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md # where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: # https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md -ndk_version=27.1.12297006 +ndk_version=27.3.13750724 ndk=$ANDROID_HOME/ndk/$ndk_version -if ! [ -e $ndk ]; then +if ! [ -e "$ndk" ]; then log "Installing NDK - this may take several minutes" - yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager "ndk;$ndk_version" + yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" "ndk;$ndk_version" fi -if [ $HOST = "arm-linux-androideabi" ]; then +if [ "$HOST" = "arm-linux-androideabi" ]; then clang_triplet=armv7a-linux-androideabi else - clang_triplet=$HOST + clang_triplet="$HOST" fi # These variables are based on BuildSystemMaintainers.md above, and # $ndk/build/cmake/android.toolchain.cmake. -toolchain=$(echo $ndk/toolchains/llvm/prebuilt/*) +toolchain=$(echo "$ndk"/toolchains/llvm/prebuilt/*) export AR="$toolchain/bin/llvm-ar" export AS="$toolchain/bin/llvm-as" -export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CC="$toolchain/bin/${clang_triplet}${ANDROID_API_LEVEL}-clang" export CXX="${CC}++" export LD="$toolchain/bin/ld" export NM="$toolchain/bin/llvm-nm" @@ -72,12 +72,12 @@ LDFLAGS="$LDFLAGS -lm" # -mstackrealign is included where necessary in the clang launcher scripts which are # pointed to by $CC, so we don't need to include it here. -if [ $HOST = "arm-linux-androideabi" ]; then +if [ "$HOST" = "arm-linux-androideabi" ]; then CFLAGS="$CFLAGS -march=armv7-a -mthumb" fi if [ -n "${PREFIX:-}" ]; then - abs_prefix=$(realpath $PREFIX) + abs_prefix="$(realpath "$PREFIX")" CFLAGS="$CFLAGS -I$abs_prefix/include" LDFLAGS="$LDFLAGS -L$abs_prefix/lib" @@ -87,11 +87,13 @@ fi # When compiling C++, some build systems will combine CFLAGS and CXXFLAGS, and some will # use CXXFLAGS alone. -export CXXFLAGS=$CFLAGS +export CXXFLAGS="$CFLAGS" # Use the same variable name as conda-build -if [ $(uname) = "Darwin" ]; then - export CPU_COUNT=$(sysctl -n hw.ncpu) +if [ "$(uname)" = "Darwin" ]; then + CPU_COUNT="$(sysctl -n hw.ncpu)" + export CPU_COUNT else - export CPU_COUNT=$(nproc) + CPU_COUNT="$(nproc)" + export CPU_COUNT fi diff --git a/Android/android.py b/Android/android.py index ae630aa8f4427c..d1a10be776ed16 100755 --- a/Android/android.py +++ b/Android/android.py @@ -2,8 +2,9 @@ import asyncio import argparse -from glob import glob +import json import os +import platform import re import shlex import shutil @@ -13,18 +14,26 @@ import sysconfig from asyncio import wait_for from contextlib import asynccontextmanager -from os.path import basename, relpath +from datetime import datetime, timezone +from glob import glob +from os.path import abspath, basename, relpath from pathlib import Path from subprocess import CalledProcessError from tempfile import TemporaryDirectory SCRIPT_NAME = Path(__file__).name -CHECKOUT = Path(__file__).resolve().parent.parent -ANDROID_DIR = CHECKOUT / "Android" +ANDROID_DIR = Path(__file__).resolve().parent +PYTHON_DIR = ANDROID_DIR.parent +in_source_tree = ( + ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() +) + +ENV_SCRIPT = ANDROID_DIR / "android-env.sh" TESTBED_DIR = ANDROID_DIR / "testbed" -CROSS_BUILD_DIR = CHECKOUT / "cross-build" +CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" +HOSTS = ["aarch64-linux-android", "x86_64-linux-android"] APP_ID = "org.python.testbed" DECODE_ARGS = ("UTF-8", "backslashreplace") @@ -44,7 +53,19 @@ + (".bat" if os.name == "nt" else "") ) -logcat_started = False +# Whether we've seen any output from Python yet. +python_started = False + +# Buffer for verbose output which will be displayed only if a test fails and +# there has been no output from Python. +hidden_output = [] + + +def log_verbose(context, line, stream=sys.stdout): + if context.verbose: + stream.write(line) + else: + hidden_output.append((stream, line)) def delete_glob(pattern): @@ -58,12 +79,10 @@ def delete_glob(pattern): path.unlink() -def subdir(name, *, clean=None): - path = CROSS_BUILD_DIR / name - if clean: - delete_glob(path) +def subdir(*parts, create=False): + path = CROSS_BUILD_DIR.joinpath(*parts) if not path.exists(): - if clean is None: + if not create: sys.exit( f"{path} does not exist. Create it by running the appropriate " f"`configure` subcommand of {SCRIPT_NAME}.") @@ -76,39 +95,67 @@ def run(command, *, host=None, env=None, log=True, **kwargs): kwargs.setdefault("check", True) if env is None: env = os.environ.copy() - original_env = env.copy() if host: - env_script = ANDROID_DIR / "android-env.sh" - env_output = subprocess.run( - f"set -eu; " - f"HOST={host}; " - f"PREFIX={subdir(host)}/prefix; " - f". {env_script}; " - f"export", - check=True, shell=True, text=True, stdout=subprocess.PIPE - ).stdout - - for line in env_output.splitlines(): - # We don't require every line to match, as there may be some other - # output from installing the NDK. - if match := re.search( - "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line - ): - key, value = match[2], match[3] - if env.get(key) != value: - print(line) - env[key] = value - - if env == original_env: - raise ValueError(f"Found no variables in {env_script.name} output:\n" - + env_output) + host_env = android_env(host) + print_env(host_env) + env.update(host_env) if log: - print(">", " ".join(map(str, command))) + print(">", join_command(command)) return subprocess.run(command, env=env, **kwargs) +# Format a command so it can be copied into a shell. Like shlex.join, but also +# accepts arguments which are Paths, or a single string/Path outside of a list. +def join_command(args): + if isinstance(args, (str, Path)): + return str(args) + else: + return shlex.join(map(str, args)) + + +# Format the environment so it can be pasted into a shell. +def print_env(env): + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") + + +def android_env(host): + if host: + prefix = subdir(host) / "prefix" + else: + prefix = ANDROID_DIR / "prefix" + sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") + sysconfig_filename = next(sysconfig_files).name + host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] + + env_output = subprocess.run( + f"set -eu; " + f"HOST={host}; " + f"PREFIX={prefix}; " + f". {ENV_SCRIPT}; " + f"export", + check=True, shell=True, capture_output=True, encoding='utf-8', + ).stdout + + env = {} + for line in env_output.splitlines(): + # We don't require every line to match, as there may be some other + # output from installing the NDK. + if match := re.search( + "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line + ): + key, value = match[2], match[3] + if os.environ.get(key) != value: + env[key] = value + + if not env: + raise ValueError(f"Found no variables in {ENV_SCRIPT.name} output:\n" + + env_output) + return env + + def build_python_path(): """The path to the build Python binary.""" build_dir = subdir("build") @@ -123,9 +170,11 @@ def build_python_path(): def configure_build_python(context): - os.chdir(subdir("build", clean=context.clean)) + if context.clean: + clean("build") + os.chdir(subdir("build", create=True)) - command = [relpath(CHECKOUT / "configure")] + command = [relpath(PYTHON_DIR / "configure")] if context.args: command.extend(context.args) run(command) @@ -136,38 +185,43 @@ def make_build_python(context): run(["make", "-j", str(os.cpu_count())]) -def unpack_deps(host): +# To create new builds of these dependencies, usually all that's necessary is to +# push a tag to the cpython-android-source-deps repository, and GitHub Actions +# will do the rest. +# +# If you're a member of the Python core team, and you'd like to be able to push +# these tags yourself, please contact Malcolm Smith or Russell Keith-Magee. +def unpack_deps(host, prefix_dir): + os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", - "sqlite-3.45.3-3", "xz-5.4.6-1"]: + for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.18-0", + "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") - run(["tar", "-xf", filename]) + shutil.unpack_archive(filename) os.remove(filename) def download(url, target_dir="."): out_path = f"{target_dir}/{basename(url)}" - run(["curl", "-Lf", "-o", out_path, url]) + run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url]) return out_path def configure_host_python(context): - host_dir = subdir(context.host, clean=context.clean) + if context.clean: + clean(context.host) + host_dir = subdir(context.host, create=True) prefix_dir = host_dir / "prefix" if not prefix_dir.exists(): prefix_dir.mkdir() - os.chdir(prefix_dir) - unpack_deps(context.host) - - build_dir = host_dir / "build" - build_dir.mkdir(exist_ok=True) - os.chdir(build_dir) + unpack_deps(context.host, prefix_dir) + os.chdir(host_dir) command = [ # Basic cross-compiling configuration - relpath(CHECKOUT / "configure"), + relpath(PYTHON_DIR / "configure"), f"--host={context.host}", f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", f"--with-build-python={build_python_path()}", @@ -193,13 +247,21 @@ def make_host_python(context): # the build. host_dir = subdir(context.host) prefix_dir = host_dir / "prefix" - delete_glob(f"{prefix_dir}/include/python*") - delete_glob(f"{prefix_dir}/lib/libpython*") - delete_glob(f"{prefix_dir}/lib/python*") + for pattern in ("include/python*", "lib/libpython*", "lib/python*"): + delete_glob(f"{prefix_dir}/{pattern}") + + # The Android environment variables were already captured in the Makefile by + # `configure`, and passing them again when running `make` may cause some + # flags to be duplicated. So we don't use the `host` argument here. + os.chdir(host_dir) + run(["make", "-j", str(os.cpu_count())]) - os.chdir(host_dir / "build") - run(["make", "-j", str(os.cpu_count())], host=context.host) - run(["make", "install", f"prefix={prefix_dir}"], host=context.host) + # The `make install` output is very verbose and rarely useful, so + # suppress it by default. + run( + ["make", "install", f"prefix={prefix_dir}"], + capture_output=not context.verbose, + ) def build_all(context): @@ -209,8 +271,40 @@ def build_all(context): step(context) +def clean(host): + delete_glob(CROSS_BUILD_DIR / host) + + def clean_all(context): - delete_glob(CROSS_BUILD_DIR) + for host in HOSTS + ["build"]: + clean(host) + + +def setup_ci(): + if "GITHUB_ACTIONS" in os.environ: + # Enable emulator hardware acceleration + # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). + if platform.system() == "Linux": + run( + ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"], + input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n', + text=True, + ) + run(["sudo", "udevadm", "control", "--reload-rules"]) + run(["sudo", "udevadm", "trigger", "--name-match=kvm"]) + + # Free up disk space by deleting unused versions of the NDK + # (https://github.com/freakboy3742/pyspamsum/pull/108). + for line in ENV_SCRIPT.read_text().splitlines(): + if match := re.fullmatch(r"ndk_version=(.+)", line): + ndk_version = match[1] + break + else: + raise ValueError(f"Failed to find NDK version in {ENV_SCRIPT.name}") + + for item in (android_home / "ndk").iterdir(): + if item.name[0].isdigit() and item.name != ndk_version: + delete_glob(item) def setup_sdk(): @@ -224,7 +318,12 @@ def setup_sdk(): if not all((android_home / "licenses" / path).exists() for path in [ "android-sdk-arm-dbt-license", "android-sdk-license" ]): - run([sdkmanager, "--licenses"], text=True, input="y\n" * 100) + run( + [sdkmanager, "--licenses"], + text=True, + capture_output=True, + input="y\n" * 100, + ) # Gradle may install this automatically, but we can't rely on that because # we need to run adb within the logcat task. @@ -234,31 +333,26 @@ def setup_sdk(): # To avoid distributing compiled artifacts without corresponding source code, # the Gradle wrapper is not included in the CPython repository. Instead, we -# extract it from the Gradle release. +# extract it from the Gradle GitHub repository. def setup_testbed(): - if all((TESTBED_DIR / path).exists() for path in [ - "gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar", - ]): + paths = ["gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar"] + if all((TESTBED_DIR / path).exists() for path in paths): return - ver_long = "8.7.0" - ver_short = ver_long.removesuffix(".0") - - for filename in ["gradlew", "gradlew.bat"]: - out_path = download( - f"https://raw.githubusercontent.com/gradle/gradle/v{ver_long}/{filename}", - TESTBED_DIR) + # The wrapper version isn't important, as any version of the wrapper can + # download any version of Gradle. The Gradle version actually used for the + # build is specified in testbed/gradle/wrapper/gradle-wrapper.properties. + version = "8.9.0" + + for path in paths: + out_path = TESTBED_DIR / path + out_path.parent.mkdir(exist_ok=True) + download( + f"https://raw.githubusercontent.com/gradle/gradle/v{version}/{path}", + out_path.parent, + ) os.chmod(out_path, 0o755) - with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: - bin_zip = download( - f"https://services.gradle.org/distributions/gradle-{ver_short}-bin.zip", - temp_dir) - outer_jar = f"gradle-{ver_short}/lib/plugins/gradle-wrapper-{ver_short}.jar" - run(["unzip", "-d", temp_dir, bin_zip, outer_jar]) - run(["unzip", "-o", "-d", f"{TESTBED_DIR}/gradle/wrapper", - f"{temp_dir}/{outer_jar}", "gradle-wrapper.jar"]) - # run_testbed will build the app automatically, but it's useful to have this as # a separate command to allow running the app outside of this script. @@ -412,17 +506,19 @@ async def logcat_task(context, initial_devices): # `--pid` requires API level 24 or higher. args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] - hidden_output = [] + logcat_started = False async with async_process( *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): + logcat_started = True level, message = match.groups() else: - # If the regex doesn't match, this is probably the second or - # subsequent line of a multi-line message. Python won't produce - # such messages, but other components might. + # If the regex doesn't match, this is either a logcat startup + # error, or the second or subsequent line of a multi-line + # message. Python won't produce multi-line messages, but other + # components might. level, message = None, line # Exclude high-volume messages which are rarely useful. @@ -442,25 +538,22 @@ async def logcat_task(context, initial_devices): # tag indicators from Python's stdout and stderr. for prefix in ["python.stdout: ", "python.stderr: "]: if message.startswith(prefix): - global logcat_started - logcat_started = True + global python_started + python_started = True stream.write(message.removeprefix(prefix)) break else: - if context.verbose: - # Non-Python messages add a lot of noise, but they may - # sometimes help explain a failure. - stream.write(line) - else: - hidden_output.append(line) + # Non-Python messages add a lot of noise, but they may + # sometimes help explain a failure. + log_verbose(context, line, stream) # If the device disconnects while logcat is running, which always # happens in --managed mode, some versions of adb return non-zero. # Distinguish this from a logcat startup error by checking whether we've - # received a message from Python yet. + # received any logcat messages yet. status = await wait_for(process.wait(), timeout=1) if status != 0 and not logcat_started: - raise CalledProcessError(status, args, "".join(hidden_output)) + raise CalledProcessError(status, args) def stop_app(serial): @@ -475,12 +568,38 @@ async def gradle_task(context): task_prefix = "connected" env["ANDROID_SERIAL"] = context.connected + if context.ci_mode: + context.args[0:0] = [ + # See _add_ci_python_opts in libregrtest/main.py. + "-W", "error", "-bb", "-E", + + # Randomization is disabled because order-dependent failures are + # much less likely to pass on a rerun in single-process mode. + "-m", "test", + f"--{context.ci_mode}-ci", "--single-process", "--no-randomize" + ] + + if not any(arg in context.args for arg in ["-c", "-m"]): + context.args[0:0] = ["-m", "test"] + args = [ gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", - "-Pandroid.testInstrumentationRunnerArguments.pythonArgs=" - + shlex.join(context.args), + ] + [ + f"-P{name}={value}" + for name, value in [ + ("python.sitePackages", context.site_packages), + ("python.cwd", context.cwd), + ( + "android.testInstrumentationRunnerArguments.pythonArgs", + json.dumps(context.args), + ), + ] + if value ] - hidden_output = [] + if context.verbose >= 2: + args.append("--info") + log_verbose(context, f"> {join_command(args)}\n") + try: async with async_process( *args, cwd=TESTBED_DIR, env=env, @@ -489,10 +608,10 @@ async def gradle_task(context): while line := (await process.stdout.readline()).decode(*DECODE_ARGS): # Gradle may take several minutes to install SDK packages, so # it's worth showing those messages even in non-verbose mode. - if context.verbose or line.startswith('Preparing "Install'): + if line.startswith('Preparing "Install'): sys.stdout.write(line) else: - hidden_output.append(line) + log_verbose(context, line) status = await wait_for(process.wait(), timeout=1) if status == 0: @@ -500,17 +619,13 @@ async def gradle_task(context): else: raise CalledProcessError(status, args) finally: - # If logcat never started, then something has gone badly wrong, so the - # user probably wants to see the Gradle output even in non-verbose mode. - if hidden_output and not logcat_started: - sys.stdout.write("".join(hidden_output)) - # Gradle does not stop the tests when interrupted. if context.connected: stop_app(context.connected) async def run_testbed(context): + setup_ci() setup_sdk() setup_testbed() @@ -534,10 +649,136 @@ async def run_testbed(context): except* MySystemExit as e: raise SystemExit(*e.exceptions[0].args) from None except* CalledProcessError as e: + # If Python produced no output, then the user probably wants to see the + # verbose output to explain why the test failed. + if not python_started: + for stream, line in hidden_output: + stream.write(line) + # Extract it from the ExceptionGroup so it can be handled by `main`. raise e.exceptions[0] +def package_version(prefix_dir): + patchlevel_glob = f"{prefix_dir}/include/python*/patchlevel.h" + patchlevel_paths = glob(patchlevel_glob) + if len(patchlevel_paths) != 1: + sys.exit(f"{patchlevel_glob} matched {len(patchlevel_paths)} paths.") + + for line in open(patchlevel_paths[0]): + if match := re.fullmatch(r'\s*#define\s+PY_VERSION\s+"(.+)"\s*', line): + version = match[1] + break + else: + sys.exit(f"Failed to find Python version in {patchlevel_paths[0]}.") + + # If not building against a tagged commit, add a timestamp to the version. + # Follow the PyPA version number rules, as this will make it easier to + # process with other tools. + if version.endswith("+"): + version += datetime.now(timezone.utc).strftime("%Y%m%d.%H%M%S") + + return version + + +def package(context): + prefix_dir = subdir(context.host, "prefix") + version = package_version(prefix_dir) + + with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + temp_dir = Path(temp_dir) + + # Include all tracked files from the Android directory. + for line in run( + ["git", "ls-files"], + cwd=ANDROID_DIR, capture_output=True, text=True, log=False, + ).stdout.splitlines(): + src = ANDROID_DIR / line + dst = temp_dir / line + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst, follow_symlinks=False) + + # Include anything from the prefix directory which could be useful + # either for embedding Python in an app, or building third-party + # packages against it. + for rel_dir, patterns in [ + ("include", ["openssl*", "python*", "sqlite*"]), + ("lib", ["engines-3", "libcrypto*.so", "libpython*", "libsqlite*", + "libssl*.so", "ossl-modules", "python*"]), + ("lib/pkgconfig", ["*crypto*", "*ssl*", "*python*", "*sqlite*"]), + ]: + for pattern in patterns: + for src in glob(f"{prefix_dir}/{rel_dir}/{pattern}"): + dst = temp_dir / relpath(src, prefix_dir.parent) + dst.parent.mkdir(parents=True, exist_ok=True) + if Path(src).is_dir(): + shutil.copytree( + src, dst, symlinks=True, + ignore=lambda *args: ["__pycache__"] + ) + else: + shutil.copy2(src, dst, follow_symlinks=False) + + # Strip debug information. + if not context.debug: + so_files = glob(f"{temp_dir}/**/*.so", recursive=True) + run([android_env(context.host)["STRIP"], *so_files], log=False) + + dist_dir = subdir(context.host, "dist", create=True) + package_path = shutil.make_archive( + f"{dist_dir}/python-{version}-{context.host}", "gztar", temp_dir + ) + print(f"Wrote {package_path}") + return package_path + + +def ci(context): + for step in [ + configure_build_python, + make_build_python, + configure_host_python, + make_host_python, + package, + ]: + caption = ( + step.__name__.replace("_", " ") + .capitalize() + .replace("python", "Python") + ) + print(f"::group::{caption}") + result = step(context) + if step is package: + package_path = result + print("::endgroup::") + + if ( + "GITHUB_ACTIONS" in os.environ + and (platform.system(), platform.machine()) != ("Linux", "x86_64") + ): + print( + "Skipping tests: GitHub Actions does not support the Android " + "emulator on this platform." + ) + else: + with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + print("::group::Tests") + + # Prove the package is self-contained by using it to run the tests. + shutil.unpack_archive(package_path, temp_dir) + launcher_args = [ + "--managed", "maxVersion", "-v", f"--{context.ci_mode}-ci" + ] + run( + ["./android.py", "test", *launcher_args], + cwd=temp_dir + ) + print("::endgroup::") + + +def env(context): + print_env(android_env(getattr(context, "host", None))) + + # Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated # by the buildbot worker, we'll make an attempt to clean up our subprocesses. def install_signal_handler(): @@ -549,41 +790,54 @@ def signal_handler(*args): def parse_args(): parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand") - build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser("configure-build", - help="Run `configure` for the " - "build Python") - make_build = subcommands.add_parser("make-build", - help="Run `make` for the build Python") - configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for Android") - make_host = subcommands.add_parser("make-host", - help="Run `make` for Android") - subcommands.add_parser( - "clean", help="Delete the cross-build directory") - - for subcommand in build, configure_build, configure_host: + subcommands = parser.add_subparsers(dest="subcommand", required=True) + + def add_parser(*args, **kwargs): + parser = subcommands.add_parser(*args, **kwargs) + parser.add_argument( + "-v", "--verbose", action="count", default=0, + help="Show verbose output. Use twice to be even more verbose.") + return parser + + # Subcommands + build = add_parser( + "build", help="Run configure-build, make-build, configure-host and " + "make-host") + configure_build = add_parser( + "configure-build", help="Run `configure` for the build Python") + add_parser( + "make-build", help="Run `make` for the build Python") + configure_host = add_parser( + "configure-host", help="Run `configure` for Android") + make_host = add_parser( + "make-host", help="Run `make` for Android") + + add_parser("clean", help="Delete all build directories") + add_parser("build-testbed", help="Build the testbed app") + test = add_parser("test", help="Run the testbed app") + package = add_parser("package", help="Make a release package") + ci = add_parser("ci", help="Run build, package and test") + env = add_parser("env", help="Print environment variables") + + # Common arguments + for subcommand in [build, configure_build, configure_host, ci]: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", - help="Delete any relevant directories before building") - for subcommand in build, configure_host, make_host: + help="Delete the relevant build directories first") + + host_commands = [build, configure_host, make_host, package, ci] + if in_source_tree: + host_commands.append(env) + for subcommand in host_commands: subcommand.add_argument( - "host", metavar="HOST", - choices=["aarch64-linux-android", "x86_64-linux-android"], + "host", metavar="HOST", choices=HOSTS, help="Host triplet: choices=[%(choices)s]") - for subcommand in build, configure_build, configure_host: + + for subcommand in [build, configure_build, configure_host, ci]: subcommand.add_argument("args", nargs="*", help="Extra arguments to pass to `configure`") - subcommands.add_parser( - "build-testbed", help="Build the testbed app") - test = subcommands.add_parser( - "test", help="Run the test suite") - test.add_argument( - "-v", "--verbose", action="count", default=0, - help="Show Gradle output, and non-Python logcat messages. " - "Use twice to include high-volume messages which are rarely useful.") + # Test arguments device_group = test.add_mutually_exclusive_group(required=True) device_group.add_argument( "--connected", metavar="SERIAL", help="Run on a connected device. " @@ -591,9 +845,34 @@ def parse_args(): device_group.add_argument( "--managed", metavar="NAME", help="Run on a Gradle-managed device. " "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") + + test.add_argument( + "--site-packages", metavar="DIR", type=abspath, + help="Directory to copy as the app's site-packages.") + test.add_argument( + "--cwd", metavar="DIR", type=abspath, + help="Directory to copy as the app's working directory.") test.add_argument( - "args", nargs="*", help=f"Arguments for `python -m test`. " - f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") + "args", nargs="*", help=f"Python command-line arguments. " + f"Separate them from {SCRIPT_NAME}'s own arguments with `--`. " + f"If neither -c nor -m are included, `-m test` will be prepended, " + f"which will run Python's own test suite.") + + # Package arguments. + for subcommand in [package, ci]: + subcommand.add_argument( + "-g", action="store_true", default=False, dest="debug", + help="Include debug information in package") + + # CI arguments + for subcommand in [test, ci]: + group = subcommand.add_mutually_exclusive_group(required=subcommand is ci) + group.add_argument( + "--fast-ci", action="store_const", dest="ci_mode", const="fast", + help="Add test arguments for GitHub Actions") + group.add_argument( + "--slow-ci", action="store_const", dest="ci_mode", const="slow", + help="Add test arguments for buildbots") return parser.parse_args() @@ -608,14 +887,19 @@ def main(): stream.reconfigure(line_buffering=True) context = parse_args() - dispatch = {"configure-build": configure_build_python, - "make-build": make_build_python, - "configure-host": configure_host_python, - "make-host": make_host_python, - "build": build_all, - "clean": clean_all, - "build-testbed": build_testbed, - "test": run_testbed} + dispatch = { + "configure-build": configure_build_python, + "make-build": make_build_python, + "configure-host": configure_host_python, + "make-host": make_host_python, + "build": build_all, + "clean": clean_all, + "build-testbed": build_testbed, + "test": run_testbed, + "package": package, + "ci": ci, + "env": env, + } try: result = dispatch[context.subcommand](context) @@ -629,20 +913,17 @@ def main(): def print_called_process_error(e): for stream_name in ["stdout", "stderr"]: content = getattr(e, stream_name) + if isinstance(content, bytes): + content = content.decode(*DECODE_ARGS) stream = getattr(sys, stream_name) if content: stream.write(content) if not content.endswith("\n"): stream.write("\n") - # Format the command so it can be copied into a shell. shlex uses single - # quotes, so we surround the whole command with double quotes. - args_joined = ( - e.cmd if isinstance(e.cmd, str) - else " ".join(shlex.quote(str(arg)) for arg in e.cmd) - ) + # shlex uses single quotes, so we surround the command with double quotes. print( - f'Command "{args_joined}" returned exit status {e.returncode}' + f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' ) diff --git a/Android/testbed/.gitignore b/Android/testbed/.gitignore index b9a7d611c943cf..7c57aee58c160a 100644 --- a/Android/testbed/.gitignore +++ b/Android/testbed/.gitignore @@ -1,18 +1,19 @@ -# The Gradle wrapper should be downloaded by running `../android.py setup-testbed`. +# The Gradle wrapper can be downloaded by running the `test` or `build-testbed` +# commands of android.py. /gradlew /gradlew.bat /gradle/wrapper/gradle-wrapper.jar +# The repository's top-level .gitignore file ignores all .idea directories, but +# we want to keep any files which can't be regenerated from the Gradle +# configuration. +!.idea/ +/.idea/* +!/.idea/inspectionProfiles + *.iml .gradle /local.properties -/.idea/caches -/.idea/deploymentTargetDropdown.xml -/.idea/libraries -/.idea/modules.xml -/.idea/workspace.xml -/.idea/navEditor.xml -/.idea/assetWizardSettings.xml .DS_Store /build /captures diff --git a/Android/testbed/.idea/inspectionProfiles/Project_Default.xml b/Android/testbed/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 00000000000000..220d9ed4ef20f7 --- /dev/null +++ b/Android/testbed/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,8 @@ +<component name="InspectionProjectProfileManager"> + <profile version="1.0"> + <option name="myName" value="Project Default" /> + <inspection_tool class="AndroidLintGradleDependency" enabled="true" level="WEAK WARNING" enabled_by_default="true" editorAttributes="INFO_ATTRIBUTES" /> + <inspection_tool class="AndroidLintOldTargetApi" enabled="true" level="WEAK WARNING" enabled_by_default="true" editorAttributes="INFO_ATTRIBUTES" /> + <inspection_tool class="UnstableApiUsage" enabled="true" level="WEAK WARNING" enabled_by_default="true" editorAttributes="INFO_ATTRIBUTES" /> + </profile> +</component> \ No newline at end of file diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index 211b5bbfadf64d..4f184cf42d5b5a 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -6,28 +6,72 @@ plugins { id("org.jetbrains.kotlin.android") } -val PYTHON_DIR = file("../../..").canonicalPath -val PYTHON_CROSS_DIR = "$PYTHON_DIR/cross-build" - -val ABIS = mapOf( - "arm64-v8a" to "aarch64-linux-android", - "x86_64" to "x86_64-linux-android", -).filter { file("$PYTHON_CROSS_DIR/${it.value}").exists() } -if (ABIS.isEmpty()) { +val ANDROID_DIR = file("../..") +val PYTHON_DIR = ANDROID_DIR.parentFile!! +val PYTHON_CROSS_DIR = file("$PYTHON_DIR/cross-build") +val inSourceTree = ( + ANDROID_DIR.name == "Android" && file("$PYTHON_DIR/pyconfig.h.in").exists() +) + +val KNOWN_ABIS = mapOf( + "aarch64-linux-android" to "arm64-v8a", + "x86_64-linux-android" to "x86_64", +) + +// Discover prefixes. +val prefixes = ArrayList<File>() +if (inSourceTree) { + for ((triplet, _) in KNOWN_ABIS.entries) { + val prefix = file("$PYTHON_CROSS_DIR/$triplet/prefix") + if (prefix.exists()) { + prefixes.add(prefix) + } + } +} else { + // Testbed is inside a release package. + val prefix = file("$ANDROID_DIR/prefix") + if (prefix.exists()) { + prefixes.add(prefix) + } +} +if (prefixes.isEmpty()) { throw GradleException( - "No Android ABIs found in $PYTHON_CROSS_DIR: see Android/README.md " + - "for building instructions." + "No Android prefixes found: see README.md for testing instructions" ) } -val PYTHON_VERSION = file("$PYTHON_DIR/Include/patchlevel.h").useLines { - for (line in it) { - val match = """#define PY_VERSION\s+"(\d+\.\d+)""".toRegex().find(line) - if (match != null) { - return@useLines match.groupValues[1] +// Detect Python versions and ABIs. +lateinit var pythonVersion: String +var abis = HashMap<File, String>() +for ((i, prefix) in prefixes.withIndex()) { + val libDir = file("$prefix/lib") + val version = run { + for (filename in libDir.list()!!) { + """python(\d+\.\d+[a-z]*)""".toRegex().matchEntire(filename)?.let { + return@run it.groupValues[1] + } + } + throw GradleException("Failed to find Python version in $libDir") + } + if (i == 0) { + pythonVersion = version + } else if (pythonVersion != version) { + throw GradleException( + "${prefixes[0]} is Python $pythonVersion, but $prefix is Python $version" + ) + } + + val libPythonDir = file("$libDir/python$pythonVersion") + val triplet = run { + for (filename in libPythonDir.list()!!) { + """_sysconfigdata_[a-z]*_android_(.+).py""".toRegex() + .matchEntire(filename)?.let { + return@run it.groupValues[1] + } } + throw GradleException("Failed to find Python triplet in $libPythonDir") } - throw GradleException("Failed to find Python version") + abis[prefix] = KNOWN_ABIS[triplet]!! } @@ -35,28 +79,39 @@ android { val androidEnvFile = file("../../android-env.sh").absoluteFile namespace = "org.python.testbed" - compileSdk = 34 + compileSdk = 35 defaultConfig { applicationId = "org.python.testbed" minSdk = androidEnvFile.useLines { for (line in it) { - """api_level:=(\d+)""".toRegex().find(line)?.let { + """ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let { return@useLines it.groupValues[1].toInt() } } throw GradleException("Failed to find API level in $androidEnvFile") } - targetSdk = 34 + + // This controls the API level of the maxVersion managed emulator, which is used + // by CI and cibuildwheel. 34 takes up too much disk space (#142289), 35 has + // issues connecting to the internet (#142387), and 36 and later are not + // available as aosp_atd images yet. + targetSdk = 33 versionCode = 1 versionName = "1.0" - ndk.abiFilters.addAll(ABIS.keys) + ndk.abiFilters.addAll(abis.values) externalNativeBuild.cmake.arguments( - "-DPYTHON_CROSS_DIR=$PYTHON_CROSS_DIR", - "-DPYTHON_VERSION=$PYTHON_VERSION", + "-DPYTHON_PREFIX_DIR=" + if (inSourceTree) { + // AGP uses the ${} syntax for its own purposes, so use a Jinja style + // placeholder. + "$PYTHON_CROSS_DIR/{{triplet}}/prefix" + } else { + prefixes[0] + }, + "-DPYTHON_VERSION=$pythonVersion", "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", ) @@ -133,40 +188,59 @@ dependencies { // Create some custom tasks to copy Python and its standard library from // elsewhere in the repository. androidComponents.onVariants { variant -> - val pyPlusVer = "python$PYTHON_VERSION" + val pyPlusVer = "python$pythonVersion" generateTask(variant, variant.sources.assets!!) { into("python") { + // Include files such as pyconfig.h are used by some of the tests. into("include/$pyPlusVer") { - for (triplet in ABIS.values) { - from("$PYTHON_CROSS_DIR/$triplet/prefix/include/$pyPlusVer") + for (prefix in prefixes) { + from("$prefix/include/$pyPlusVer") } duplicatesStrategy = DuplicatesStrategy.EXCLUDE } into("lib/$pyPlusVer") { - // To aid debugging, the source directory takes priority. - from("$PYTHON_DIR/Lib") - - // The cross-build directory provides ABI-specific files such as - // sysconfigdata. - for (triplet in ABIS.values) { - from("$PYTHON_CROSS_DIR/$triplet/prefix/lib/$pyPlusVer") + // To aid debugging, the source directory takes priority when + // running inside a CPython source tree. + if (inSourceTree) { + from("$PYTHON_DIR/Lib") + } + for (prefix in prefixes) { + from("$prefix/lib/$pyPlusVer") } into("site-packages") { from("$projectDir/src/main/python") + + val sitePackages = findProperty("python.sitePackages") as String? + if (!sitePackages.isNullOrEmpty()) { + if (!file(sitePackages).exists()) { + throw GradleException("$sitePackages does not exist") + } + from(sitePackages) + } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE exclude("**/__pycache__") } + + into("cwd") { + val cwd = findProperty("python.cwd") as String? + if (!cwd.isNullOrEmpty()) { + if (!file(cwd).exists()) { + throw GradleException("$cwd does not exist") + } + from(cwd) + } + } } } generateTask(variant, variant.sources.jniLibs!!) { - for ((abi, triplet) in ABIS.entries) { + for ((prefix, abi) in abis.entries) { into(abi) { - from("$PYTHON_CROSS_DIR/$triplet/prefix/lib") + from("$prefix/lib") include("libpython*.*.so") include("lib*_python.so") } diff --git a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt index 0e888ab71d87da..e57243566f91dc 100644 --- a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt +++ b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt @@ -17,11 +17,11 @@ class PythonSuite { fun testPython() { val start = System.currentTimeMillis() try { - val context = + val status = PythonTestRunner( InstrumentationRegistry.getInstrumentation().targetContext - val args = - InstrumentationRegistry.getArguments().getString("pythonArgs", "") - val status = PythonTestRunner(context).run(args) + ).run( + InstrumentationRegistry.getArguments().getString("pythonArgs")!!, + ) assertEquals(0, status) } finally { // Make sure the process lives long enough for the test script to diff --git a/Android/testbed/app/src/main/c/CMakeLists.txt b/Android/testbed/app/src/main/c/CMakeLists.txt index 1d5df9a73465b6..6d5ccd96f8ae29 100644 --- a/Android/testbed/app/src/main/c/CMakeLists.txt +++ b/Android/testbed/app/src/main/c/CMakeLists.txt @@ -1,9 +1,14 @@ cmake_minimum_required(VERSION 3.4.1) project(testbed) -set(PREFIX_DIR ${PYTHON_CROSS_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/prefix) -include_directories(${PREFIX_DIR}/include/python${PYTHON_VERSION}) -link_directories(${PREFIX_DIR}/lib) +# Resolve variables from the command line. +string( + REPLACE {{triplet}} ${CMAKE_LIBRARY_ARCHITECTURE} + PYTHON_PREFIX_DIR ${PYTHON_PREFIX_DIR} +) + +include_directories(${PYTHON_PREFIX_DIR}/include/python${PYTHON_VERSION}) +link_directories(${PYTHON_PREFIX_DIR}/lib) link_libraries(log python${PYTHON_VERSION}) add_library(main_activity SHARED main_activity.c) diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c index ec7f93a3e5ee13..7f024f0a348b61 100644 --- a/Android/testbed/app/src/main/c/main_activity.c +++ b/Android/testbed/app/src/main/c/main_activity.c @@ -3,6 +3,7 @@ #include <jni.h> #include <pthread.h> #include <Python.h> +#include <signal.h> #include <stdio.h> #include <string.h> #include <unistd.h> @@ -15,6 +16,13 @@ static void throw_runtime_exception(JNIEnv *env, const char *message) { message); } +static void throw_errno(JNIEnv *env, const char *error_prefix) { + char error_message[1024]; + snprintf(error_message, sizeof(error_message), + "%s: %s", error_prefix, strerror(errno)); + throw_runtime_exception(env, error_message); +} + // --- Stdio redirection ------------------------------------------------------ @@ -95,10 +103,7 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL for (StreamInfo *si = STREAMS; si->file; si++) { char *error_prefix; if ((error_prefix = redirect_stream(si))) { - char error_message[1024]; - snprintf(error_message, sizeof(error_message), - "%s: %s", error_prefix, strerror(errno)); - throw_runtime_exception(env, error_message); + throw_errno(env, error_prefix); return; } } @@ -107,13 +112,38 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL // --- Python initialization --------------------------------------------------- -static PyStatus set_config_string( - JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value -) { - const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL); - PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8); - (*env)->ReleaseStringUTFChars(env, value, value_utf8); - return status; +static char *init_signals() { + // Some tests use SIGUSR1, but that's blocked by default in an Android app in + // order to make it available to `sigwait` in the Signal Catcher thread. + // (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). + // That thread's functionality is only useful for debugging the JVM, so disabling + // it should not weaken the tests. + // + // There's no safe way of stopping the thread completely (#123982), but simply + // unblocking SIGUSR1 is enough to fix most tests. + // + // However, in tests that generate multiple different signals in quick + // succession, it's possible for SIGUSR1 to arrive while the main thread is busy + // running the C-level handler for a different signal. In that case, the SIGUSR1 + // may be sent to the Signal Catcher thread instead, which will generate a log + // message containing the text "reacting to signal". + // + // Such tests may need to be changed in one of the following ways: + // * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in + // test_signal.py). + // * Send the signal to a specific thread rather than the whole process (e.g. + // test_signals in test_threadsignals.py. + sigset_t set; + if (sigemptyset(&set)) { + return "sigemptyset"; + } + if (sigaddset(&set, SIGUSR1)) { + return "sigaddset"; + } + if ((errno = pthread_sigmask(SIG_UNBLOCK, &set, NULL))) { + return "pthread_sigmask"; + } + return NULL; } static void throw_status(JNIEnv *env, PyStatus status) { @@ -121,27 +151,47 @@ static void throw_status(JNIEnv *env, PyStatus status) { } JNIEXPORT int JNICALL Java_org_python_testbed_PythonTestRunner_runPython( - JNIEnv *env, jobject obj, jstring home, jstring runModule + JNIEnv *env, jobject obj, jstring home, jarray args ) { + const char *home_utf8 = (*env)->GetStringUTFChars(env, home, NULL); + char cwd[PATH_MAX]; + snprintf(cwd, sizeof(cwd), "%s/%s", home_utf8, "cwd"); + if (chdir(cwd)) { + throw_errno(env, "chdir"); + return 1; + } + + char *error_prefix; + if ((error_prefix = init_signals())) { + throw_errno(env, error_prefix); + return 1; + } + PyConfig config; PyStatus status; - PyConfig_InitIsolatedConfig(&config); + PyConfig_InitPythonConfig(&config); - status = set_config_string(env, &config, &config.home, home); - if (PyStatus_Exception(status)) { + jsize argc = (*env)->GetArrayLength(env, args); + const char *argv[argc + 1]; + for (int i = 0; i < argc; i++) { + jobject arg = (*env)->GetObjectArrayElement(env, args, i); + argv[i] = (*env)->GetStringUTFChars(env, arg, NULL); + } + argv[argc] = NULL; + + // PyConfig_SetBytesArgv "must be called before other methods, since the + // preinitialization configuration depends on command line arguments" + if (PyStatus_Exception(status = PyConfig_SetBytesArgv(&config, argc, (char**)argv))) { throw_status(env, status); return 1; } - status = set_config_string(env, &config, &config.run_module, runModule); + status = PyConfig_SetBytesString(&config, &config.home, home_utf8); if (PyStatus_Exception(status)) { throw_status(env, status); return 1; } - // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored. - config.install_signal_handlers = 1; - status = Py_InitializeFromConfig(&config); if (PyStatus_Exception(status)) { throw_status(env, status); diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt index c4bf6cbe83d8cd..5727b0fe6c30c0 100644 --- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -5,6 +5,7 @@ import android.os.* import android.system.Os import android.widget.TextView import androidx.appcompat.app.* +import org.json.JSONArray import java.io.* @@ -15,18 +16,25 @@ class MainActivity : AppCompatActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) - val status = PythonTestRunner(this).run("-W -uall") + val status = PythonTestRunner(this).run("""["-m", "test", "-W", "-uall"]""") findViewById<TextView>(R.id.tvHello).text = "Exit status $status" } } class PythonTestRunner(val context: Context) { - /** @param args Extra arguments for `python -m test`. - * @return The Python exit status: zero if the tests passed, nonzero if - * they failed. */ - fun run(args: String = "") : Int { - Os.setenv("PYTHON_ARGS", args, true) + /** Run Python. + * + * @param args Python command-line, encoded as JSON. + * @return The Python exit status: zero on success, nonzero on failure. */ + fun run(args: String) : Int { + // We leave argument 0 as an empty string, which is a placeholder for the + // executable name in embedded mode. + val argsJsonArray = JSONArray(args) + val argsStringArray = Array<String>(argsJsonArray.length() + 1) { it -> ""} + for (i in 0..<argsJsonArray.length()) { + argsStringArray[i + 1] = argsJsonArray.getString(i) + } // Python needs this variable to help it find the temporary directory, // but Android only sets it on API level 33 and later. @@ -35,9 +43,7 @@ class PythonTestRunner(val context: Context) { val pythonHome = extractAssets() System.loadLibrary("main_activity") redirectStdioToLogcat() - - // The main module is in src/main/python/main.py. - return runPython(pythonHome.toString(), "main") + return runPython(pythonHome.toString(), argsStringArray) } private fun extractAssets() : File { @@ -46,6 +52,13 @@ class PythonTestRunner(val context: Context) { throw RuntimeException("Failed to delete $pythonHome") } extractAssetDir("python", context.filesDir) + + // Empty directories are lost in the asset packing/unpacking process. + val cwd = File(pythonHome, "cwd") + if (!cwd.exists()) { + cwd.mkdir() + } + return pythonHome } @@ -75,5 +88,5 @@ class PythonTestRunner(val context: Context) { } private external fun redirectStdioToLogcat() - private external fun runPython(home: String, runModule: String) : Int + private external fun runPython(home: String, args: Array<String>) : Int } diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/main.py deleted file mode 100644 index d6941b14412fcc..00000000000000 --- a/Android/testbed/app/src/main/python/main.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -import runpy -import shlex -import signal -import sys - -# Some tests use SIGUSR1, but that's blocked by default in an Android app in -# order to make it available to `sigwait` in the Signal Catcher thread. -# (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). -# That thread's functionality is only useful for debugging the JVM, so disabling -# it should not weaken the tests. -# -# There's no safe way of stopping the thread completely (#123982), but simply -# unblocking SIGUSR1 is enough to fix most tests. -# -# However, in tests that generate multiple different signals in quick -# succession, it's possible for SIGUSR1 to arrive while the main thread is busy -# running the C-level handler for a different signal. In that case, the SIGUSR1 -# may be sent to the Signal Catcher thread instead, which will generate a log -# message containing the text "reacting to signal". -# -# Such tests may need to be changed in one of the following ways: -# * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in -# test_signal.py). -# * Send the signal to a specific thread rather than the whole process (e.g. -# test_signals in test_threadsignals.py. -signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) - -sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) - -# The test module will call sys.exit to indicate whether the tests passed. -runpy.run_module("test") diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts index 4d1d6f87594da3..451517b3f1aeab 100644 --- a/Android/testbed/build.gradle.kts +++ b/Android/testbed/build.gradle.kts @@ -1,5 +1,5 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. plugins { - id("com.android.application") version "8.6.1" apply false + id("com.android.application") version "8.10.0" apply false id("org.jetbrains.kotlin.android") version "1.9.22" apply false } diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties index 36529c896426b0..5d42fbae084da1 100644 --- a/Android/testbed/gradle/wrapper/gradle-wrapper.properties +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Mon Feb 19 20:29:06 GMT 2024 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/Apple/.ruff.toml b/Apple/.ruff.toml new file mode 100644 index 00000000000000..4cdc39ebee4be9 --- /dev/null +++ b/Apple/.ruff.toml @@ -0,0 +1,22 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] diff --git a/Apple/__main__.py b/Apple/__main__.py new file mode 100644 index 00000000000000..256966e76c2c97 --- /dev/null +++ b/Apple/__main__.py @@ -0,0 +1,1065 @@ +#!/usr/bin/env python3 +########################################################################## +# Apple XCframework build script +# +# This script simplifies the process of configuring, compiling and packaging an +# XCframework for an Apple platform. +# +# At present, it only supports iOS, but it has been constructed so that it +# could be used on any Apple platform. +# +# The simplest entry point is: +# +# $ python Apple ci iOS +# +# which will: +# * Clean any pre-existing build artefacts +# * Configure and make a Python that can be used for the build +# * Configure and make a Python for each supported iOS architecture and ABI +# * Combine the outputs of the builds from the previous step into a single +# XCframework, merging binaries into a "fat" binary if necessary +# * Clone a copy of the testbed, configured to use the XCframework +# * Construct a tarball containing the release artefacts +# * Run the test suite using the generated XCframework. +# +# This is the complete sequence that would be needed in CI to build and test +# a candidate release artefact. +# +# Each individual step can be invoked individually - there are commands to +# clean, configure-build, make-build, configure-host, make-host, package, and +# test. +# +# There is also a build command that can be used to combine the configure and +# make steps for the build Python, an individual host, all hosts, or all +# builds. +########################################################################## +from __future__ import annotations + +import argparse +import os +import platform +import re +import shlex +import shutil +import signal +import subprocess +import sys +import sysconfig +import time +from collections.abc import Callable, Sequence +from contextlib import contextmanager +from datetime import datetime, timezone +from os.path import basename, relpath +from pathlib import Path +from subprocess import CalledProcessError + +EnvironmentT = dict[str, str] +ArgsT = Sequence[str | Path] + +SCRIPT_NAME = Path(__file__).name +PYTHON_DIR = Path(__file__).resolve().parent.parent + +CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" + +HOSTS: dict[str, dict[str, dict[str, str]]] = { + # Structure of this data: + # * Platform identifier + # * an XCframework slice that must exist for that platform + # * a host triple: the multiarch spec for that host + "iOS": { + "ios-arm64": { + "arm64-apple-ios": "arm64-iphoneos", + }, + "ios-arm64_x86_64-simulator": { + "arm64-apple-ios-simulator": "arm64-iphonesimulator", + "x86_64-apple-ios-simulator": "x86_64-iphonesimulator", + }, + }, +} + + +def subdir(name: str, create: bool = False) -> Path: + """Ensure that a cross-build directory for the given name exists.""" + path = CROSS_BUILD_DIR / name + if not path.exists(): + if not create: + sys.exit( + f"{path} does not exist. Create it by running the appropriate " + f"`configure` subcommand of {SCRIPT_NAME}." + ) + else: + path.mkdir(parents=True) + return path + + +def run( + command: ArgsT, + *, + host: str | None = None, + env: EnvironmentT | None = None, + log: bool | None = True, + **kwargs, +) -> subprocess.CompletedProcess: + """Run a command in an Apple development environment. + + Optionally logs the executed command to the console. + """ + kwargs.setdefault("check", True) + if env is None: + env = os.environ.copy() + + if host: + host_env = apple_env(host) + print_env(host_env) + env.update(host_env) + + if log: + print(">", join_command(command)) + return subprocess.run(command, env=env, **kwargs) + + +def join_command(args: str | Path | ArgsT) -> str: + """Format a command so it can be copied into a shell. + + Similar to `shlex.join`, but also accepts arguments which are Paths, or a + single string/Path outside of a list. + """ + if isinstance(args, (str, Path)): + return str(args) + else: + return shlex.join(map(str, args)) + + +def print_env(env: EnvironmentT) -> None: + """Format the environment so it can be pasted into a shell.""" + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") + + +def apple_env(host: str) -> EnvironmentT: + """Construct an Apple development environment for the given host.""" + env = { + "PATH": ":".join([ + str(PYTHON_DIR / "Apple/iOS/Resources/bin"), + str(subdir(host) / "prefix"), + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "/Library/Apple/usr/bin", + ]), + } + + return env + + +def delete_path(name: str) -> None: + """Delete the named cross-build directory, if it exists.""" + path = CROSS_BUILD_DIR / name + if path.exists(): + print(f"Deleting {path} ...") + shutil.rmtree(path) + + +def all_host_triples(platform: str) -> list[str]: + """Return all host triples for the given platform. + + The host triples are the platform definitions used as input to configure + (e.g., "arm64-apple-ios-simulator"). + """ + triples = [] + for slice_name, slice_parts in HOSTS[platform].items(): + triples.extend(list(slice_parts)) + return triples + + +def clean(context: argparse.Namespace, target: str = "all") -> None: + """The implementation of the "clean" command.""" + # If we're explicitly targeting the build, there's no platform or + # distribution artefacts. If we're cleaning tests, we keep all built + # artefacts. Otherwise, the built artefacts must be dirty, so we remove + # them. + if target not in {"build", "test"}: + paths = ["dist", context.platform] + list(HOSTS[context.platform]) + else: + paths = [] + + if target in {"all", "build"}: + paths.append("build") + + if target in {"all", "hosts"}: + paths.extend(all_host_triples(context.platform)) + elif target not in {"build", "test", "package"}: + paths.append(target) + + if target in {"all", "hosts", "test"}: + paths.extend([ + path.name + for path in CROSS_BUILD_DIR.glob(f"{context.platform}-testbed.*") + ]) + + for path in paths: + delete_path(path) + + +def build_python_path() -> Path: + """The path to the build Python binary.""" + build_dir = subdir("build") + binary = build_dir / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {build_dir}" + ) + + return binary + + +@contextmanager +def group(text: str): + """A context manager that outputs a log marker around a section of a build. + + If running in a GitHub Actions environment, the GitHub syntax for + collapsible log sections is used. + """ + if "GITHUB_ACTIONS" in os.environ: + print(f"::group::{text}") + else: + print(f"===== {text} " + "=" * (70 - len(text))) + + yield + + if "GITHUB_ACTIONS" in os.environ: + print("::endgroup::") + else: + print() + + +@contextmanager +def cwd(subdir: Path): + """A context manager that sets the current working directory.""" + orig = os.getcwd() + os.chdir(subdir) + yield + os.chdir(orig) + + +def configure_build_python(context: argparse.Namespace) -> None: + """The implementation of the "configure-build" command.""" + if context.clean: + clean(context, "build") + + with ( + group("Configuring build Python"), + cwd(subdir("build", create=True)), + ): + command = [relpath(PYTHON_DIR / "configure")] + if context.args: + command.extend(context.args) + run(command) + + +def make_build_python(context: argparse.Namespace) -> None: + """The implementation of the "make-build" command.""" + with ( + group("Compiling build Python"), + cwd(subdir("build")), + ): + run(["make", "-j", str(os.cpu_count())]) + + +def apple_target(host: str) -> str: + """Return the Apple platform identifier for a given host triple.""" + for _, platform_slices in HOSTS.items(): + for slice_name, slice_parts in platform_slices.items(): + for host_triple, multiarch in slice_parts.items(): + if host == host_triple: + return ".".join(multiarch.split("-")[::-1]) + + raise KeyError(host) + + +def apple_multiarch(host: str) -> str: + """Return the multiarch descriptor for a given host triple.""" + for _, platform_slices in HOSTS.items(): + for slice_name, slice_parts in platform_slices.items(): + for host_triple, multiarch in slice_parts.items(): + if host == host_triple: + return multiarch + + raise KeyError(host) + + +def unpack_deps( + platform: str, + host: str, + prefix_dir: Path, + cache_dir: Path, +) -> None: + """Unpack binary dependencies into a provided directory. + + Downloads binaries if they aren't already present. Downloads will be stored + in provided cache directory. + + On iOS, as a safety mechanism, any dynamic libraries will be purged from + the unpacked dependencies. + """ + # To create new builds of these dependencies, usually all that's necessary + # is to push a tag to the cpython-apple-source-deps repository, and GitHub + # Actions will do the rest. + # + # If you're a member of the Python core team, and you'd like to be able to + # push these tags yourself, please contact Malcolm Smith or Russell + # Keith-Magee. + deps_url = "https://github.com/beeware/cpython-apple-source-deps/releases/download" + for name_ver in [ + "BZip2-1.0.8-2", + "libFFI-3.4.7-2", + "OpenSSL-3.0.18-1", + "XZ-5.6.4-2", + "mpdecimal-4.0.0-2", + "zstd-1.5.7-1", + ]: + filename = f"{name_ver.lower()}-{apple_target(host)}.tar.gz" + archive_path = download( + f"{deps_url}/{name_ver}/{filename}", + target_dir=cache_dir, + ) + shutil.unpack_archive(archive_path, prefix_dir) + + # Dynamic libraries will be preferentially linked over static; + # On iOS, ensure that no dylibs are available in the prefix folder. + if platform == "iOS": + for dylib in prefix_dir.glob("**/*.dylib"): + dylib.unlink() + + +def download(url: str, target_dir: Path) -> Path: + """Download the specified URL into the given directory. + + :return: The path to the downloaded archive. + """ + target_path = Path(target_dir).resolve() + target_path.mkdir(exist_ok=True, parents=True) + + out_path = target_path / basename(url) + if not Path(out_path).is_file(): + run([ + "curl", + "-Lf", + "--retry", + "5", + "--retry-all-errors", + "-o", + out_path, + url, + ]) + else: + print(f"Using cached version of {basename(url)}") + return out_path + + +def configure_host_python( + context: argparse.Namespace, + host: str | None = None, +) -> None: + """The implementation of the "configure-host" command.""" + if host is None: + host = context.host + + if context.clean: + clean(context, host) + + host_dir = subdir(host, create=True) + prefix_dir = host_dir / "prefix" + + with group(f"Downloading dependencies ({host})"): + if not prefix_dir.exists(): + prefix_dir.mkdir() + unpack_deps(context.platform, host, prefix_dir, context.cache_dir) + else: + print("Dependencies already installed") + + with ( + group(f"Configuring host Python ({host})"), + cwd(host_dir), + ): + command = [ + # Basic cross-compiling configuration + relpath(PYTHON_DIR / "configure"), + f"--host={host}", + f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", + f"--with-build-python={build_python_path()}", + "--with-system-libmpdec", + "--enable-framework", + # Dependent libraries. + f"--with-openssl={prefix_dir}", + f"LIBLZMA_CFLAGS=-I{prefix_dir}/include", + f"LIBLZMA_LIBS=-L{prefix_dir}/lib -llzma", + f"LIBFFI_CFLAGS=-I{prefix_dir}/include", + f"LIBFFI_LIBS=-L{prefix_dir}/lib -lffi", + f"LIBMPDEC_CFLAGS=-I{prefix_dir}/include", + f"LIBMPDEC_LIBS=-L{prefix_dir}/lib -lmpdec", + f"LIBZSTD_CFLAGS=-I{prefix_dir}/include", + f"LIBZSTD_LIBS=-L{prefix_dir}/lib -lzstd", + ] + + if context.args: + command.extend(context.args) + run(command, host=host) + + +def make_host_python( + context: argparse.Namespace, + host: str | None = None, +) -> None: + """The implementation of the "make-host" command.""" + if host is None: + host = context.host + + with ( + group(f"Compiling host Python ({host})"), + cwd(subdir(host)), + ): + run(["make", "-j", str(os.cpu_count())], host=host) + run(["make", "install"], host=host) + + +def framework_path(host_triple: str, multiarch: str) -> Path: + """The path to a built single-architecture framework product. + + :param host_triple: The host triple (e.g., arm64-apple-ios-simulator) + :param multiarch: The multiarch identifier (e.g., arm64-simulator) + """ + return CROSS_BUILD_DIR / f"{host_triple}/Apple/iOS/Frameworks/{multiarch}" + + +def package_version(prefix_path: Path) -> str: + """Extract the Python version being built from patchlevel.h.""" + for path in prefix_path.glob("**/patchlevel.h"): + text = path.read_text(encoding="utf-8") + if match := re.search( + r'\n\s*#define\s+PY_VERSION\s+"(.+)"\s*\n', text + ): + version = match[1] + # If not building against a tagged commit, add a timestamp to the + # version. Follow the PyPA version number rules, as this will make + # it easier to process with other tools. The version will have a + # `+` suffix once any official release has been made; a freshly + # forked main branch will have a version of 3.X.0a0. + if version.endswith("a0"): + version += "+" + if version.endswith("+"): + version += datetime.now(timezone.utc).strftime("%Y%m%d.%H%M%S") + + return version + + sys.exit("Unable to determine Python version being packaged.") + + +def lib_platform_files(dirname, names): + """A file filter that ignores platform-specific files in lib.""" + path = Path(dirname) + if ( + path.parts[-3] == "lib" + and path.parts[-2].startswith("python") + and path.parts[-1] == "lib-dynload" + ): + return names + elif path.parts[-2] == "lib" and path.parts[-1].startswith("python"): + ignored_names = { + name + for name in names + if ( + name.startswith("_sysconfigdata_") + or name.startswith("_sysconfig_vars_") + or name == "build-details.json" + ) + } + elif path.parts[-1] == "lib": + ignored_names = { + name + for name in names + if name.startswith("libpython") and name.endswith(".dylib") + } + else: + ignored_names = set() + + return ignored_names + + +def lib_non_platform_files(dirname, names): + """A file filter that ignores anything *except* platform-specific files + in the lib directory. + """ + path = Path(dirname) + if path.parts[-2] == "lib" and path.parts[-1].startswith("python"): + return ( + set(names) - lib_platform_files(dirname, names) - {"lib-dynload"} + ) + else: + return set() + + +def create_xcframework(platform: str) -> str: + """Build an XCframework from the component parts for the platform. + + :return: The version number of the Python version that was packaged. + """ + package_path = CROSS_BUILD_DIR / platform + try: + package_path.mkdir() + except FileExistsError: + raise RuntimeError( + f"{platform} XCframework already exists; do you need to run " + "with --clean?" + ) from None + + frameworks = [] + # Merge Frameworks for each component SDK. If there's only one architecture + # for the SDK, we can use the compiled Python.framework as-is. However, if + # there's more than architecture, we need to merge the individual built + # frameworks into a merged "fat" framework. + for slice_name, slice_parts in HOSTS[platform].items(): + # Some parts are the same across all slices, so we use can any of the + # host frameworks as the source for the merged version. Use the first + # one on the list, as it's as representative as any other. + first_host_triple, first_multiarch = next(iter(slice_parts.items())) + first_framework = ( + framework_path(first_host_triple, first_multiarch) + / "Python.framework" + ) + + if len(slice_parts) == 1: + # The first framework is the only framework, so copy it. + print(f"Copying framework for {slice_name}...") + frameworks.append(first_framework) + else: + print(f"Merging framework for {slice_name}...") + slice_path = CROSS_BUILD_DIR / slice_name + slice_framework = slice_path / "Python.framework" + slice_framework.mkdir(exist_ok=True, parents=True) + + # Copy the Info.plist + shutil.copy( + first_framework / "Info.plist", + slice_framework / "Info.plist", + ) + + # Copy the headers + shutil.copytree( + first_framework / "Headers", + slice_framework / "Headers", + ) + + # Create the "fat" library binary for the slice + run( + ["lipo", "-create", "-output", slice_framework / "Python"] + + [ + ( + framework_path(host_triple, multiarch) + / "Python.framework/Python" + ) + for host_triple, multiarch in slice_parts.items() + ] + ) + + # Add this merged slice to the list to be added to the XCframework + frameworks.append(slice_framework) + + print() + print("Build XCframework...") + cmd = [ + "xcodebuild", + "-create-xcframework", + "-output", + package_path / "Python.xcframework", + ] + for framework in frameworks: + cmd.extend(["-framework", framework]) + + run(cmd) + + # Extract the package version from the merged framework + version = package_version(package_path / "Python.xcframework") + version_tag = ".".join(version.split(".")[:2]) + + # On non-macOS platforms, each framework in XCframework only contains the + # headers, libPython, plus an Info.plist. Other resources like the standard + # library and binary shims aren't allowed to live in framework; they need + # to be copied in separately. + print() + print("Copy additional resources...") + has_common_stdlib = False + for slice_name, slice_parts in HOSTS[platform].items(): + # Some parts are the same across all slices, so we can any of the + # host frameworks as the source for the merged version. + first_host_triple, first_multiarch = next(iter(slice_parts.items())) + first_path = framework_path(first_host_triple, first_multiarch) + first_framework = first_path / "Python.framework" + + slice_path = package_path / f"Python.xcframework/{slice_name}" + slice_framework = slice_path / "Python.framework" + + # Copy the binary helpers + print(f" - {slice_name} binaries") + shutil.copytree(first_path / "bin", slice_path / "bin") + + # Copy the include path (a symlink to the framework headers) + print(f" - {slice_name} include files") + shutil.copytree( + first_path / "include", + slice_path / "include", + symlinks=True, + ) + + # Copy in the cross-architecture pyconfig.h + shutil.copy( + PYTHON_DIR / f"Apple/{platform}/Resources/pyconfig.h", + slice_framework / "Headers/pyconfig.h", + ) + + print(f" - {slice_name} shared library") + # Create a simlink for the fat library + shared_lib = slice_path / f"lib/libpython{version_tag}.dylib" + shared_lib.parent.mkdir() + shared_lib.symlink_to("../Python.framework/Python") + + print(f" - {slice_name} architecture-specific files") + for host_triple, multiarch in slice_parts.items(): + print(f" - {multiarch} standard library") + arch, _ = multiarch.split("-", 1) + + if not has_common_stdlib: + print(" - using this architecture as the common stdlib") + shutil.copytree( + framework_path(host_triple, multiarch) / "lib", + package_path / "Python.xcframework/lib", + ignore=lib_platform_files, + symlinks=True, + ) + has_common_stdlib = True + + shutil.copytree( + framework_path(host_triple, multiarch) / "lib", + slice_path / f"lib-{arch}", + ignore=lib_non_platform_files, + symlinks=True, + ) + + # Copy the host's pyconfig.h to an architecture-specific name. + arch = multiarch.split("-")[0] + host_path = ( + CROSS_BUILD_DIR + / host_triple + / "Apple/iOS/Frameworks" + / multiarch + ) + host_framework = host_path / "Python.framework" + shutil.copy( + host_framework / "Headers/pyconfig.h", + slice_framework / f"Headers/pyconfig-{arch}.h", + ) + + # Apple identifies certain libraries as "security risks"; if you + # statically link those libraries into a Framework, you become + # responsible for providing a privacy manifest for that framework. + xcprivacy_file = { + "OpenSSL": subdir(host_triple) + / "prefix/share/OpenSSL.xcprivacy" + } + print(f" - {multiarch} xcprivacy files") + for module, lib in [ + ("_hashlib", "OpenSSL"), + ("_ssl", "OpenSSL"), + ]: + shutil.copy( + xcprivacy_file[lib], + slice_path + / f"lib-{arch}/python{version_tag}" + / f"lib-dynload/{module}.xcprivacy", + ) + + print(" - build tools") + shutil.copytree( + PYTHON_DIR / "Apple/testbed/Python.xcframework/build", + package_path / "Python.xcframework/build", + ) + + return version + + +def package(context: argparse.Namespace) -> None: + """The implementation of the "package" command.""" + if context.clean: + clean(context, "package") + + with group("Building package"): + # Create an XCframework + version = create_xcframework(context.platform) + + # Clone testbed + print() + run([ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + CROSS_BUILD_DIR / context.platform / "Python.xcframework", + CROSS_BUILD_DIR / context.platform / "testbed", + ]) + + # Build the final archive + archive_name = ( + CROSS_BUILD_DIR + / "dist" + / f"python-{version}-{context.platform}-XCframework" + ) + + print() + print("Create package archive...") + shutil.make_archive( + str(CROSS_BUILD_DIR / archive_name), + format="gztar", + root_dir=CROSS_BUILD_DIR / context.platform, + base_dir=".", + ) + print() + print(f"{archive_name.relative_to(PYTHON_DIR)}.tar.gz created.") + + +def build(context: argparse.Namespace, host: str | None = None) -> None: + """The implementation of the "build" command.""" + if host is None: + host = context.host + + if context.clean: + clean(context, host) + + if host in {"all", "build"}: + for step in [ + configure_build_python, + make_build_python, + ]: + step(context) + + if host == "build": + hosts = [] + elif host in {"all", "hosts"}: + hosts = all_host_triples(context.platform) + else: + hosts = [host] + + for step_host in hosts: + for step in [ + configure_host_python, + make_host_python, + ]: + step(context, host=step_host) + + if host in {"all", "hosts"}: + package(context) + + +def test(context: argparse.Namespace, host: str | None = None) -> None: # noqa: PT028 + """The implementation of the "test" command.""" + if host is None: + host = context.host + + if context.clean: + clean(context, "test") + + with group(f"Test {'XCframework' if host in {'all', 'hosts'} else host}"): + timestamp = str(time.time_ns())[:-6] + testbed_dir = ( + CROSS_BUILD_DIR / f"{context.platform}-testbed.{timestamp}" + ) + if host in {"all", "hosts"}: + framework_path = ( + CROSS_BUILD_DIR / context.platform / "Python.xcframework" + ) + else: + build_arch = platform.machine() + host_arch = host.split("-")[0] + + if not host.endswith("-simulator"): + print("Skipping test suite non-simulator build.") + return + elif build_arch != host_arch: + print( + f"Skipping test suite for an {host_arch} build " + f"on an {build_arch} machine." + ) + return + else: + framework_path = ( + CROSS_BUILD_DIR + / host + / f"Apple/{context.platform}" + / f"Frameworks/{apple_multiarch(host)}" + ) + + run([ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + framework_path, + testbed_dir, + ]) + + run( + [ + sys.executable, + testbed_dir, + "run", + "--verbose", + ] + + ( + ["--simulator", str(context.simulator)] + if context.simulator + else [] + ) + + [ + "--", + "test", + f"--{context.ci_mode}-ci", + "--single-process", + "--no-randomize", + # Timeout handling requires subprocesses; explicitly setting + # the timeout to -1 disables the faulthandler. + "--timeout=-1", + # Adding Python options requires the use of a subprocess to + # start a new Python interpreter. + "--dont-add-python-opts", + ] + ) + + +def apple_sim_host(platform_name: str) -> str: + """Determine the native simulator target for this platform.""" + for _, slice_parts in HOSTS[platform_name].items(): + for host_triple in slice_parts: + parts = host_triple.split("-") + if parts[0] == platform.machine() and parts[-1] == "simulator": + return host_triple + + raise KeyError(platform_name) + + +def ci(context: argparse.Namespace) -> None: + """The implementation of the "ci" command. + + In "Fast" mode, this compiles the build python, and the simulator for the + build machine's architecture; and runs the test suite with `--fast-ci` + configuration. + + In "Slow" mode, it compiles the build python, plus all candidate + architectures (both device and simulator); then runs the test suite with + `--slow-ci` configuration. + """ + clean(context, "all") + if context.ci_mode == "slow": + # In slow mode, build and test the full XCframework + build(context, host="all") + test(context, host="all") + else: + # In fast mode, just build the simulator platform. + sim_host = apple_sim_host(context.platform) + build(context, host="build") + build(context, host=sim_host) + test(context, host=sim_host) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "A tool for managing the build, package and test process of " + "CPython on Apple platforms." + ), + ) + parser.suggest_on_error = True + subcommands = parser.add_subparsers(dest="subcommand", required=True) + + clean = subcommands.add_parser( + "clean", + help="Delete all build directories", + ) + + configure_build = subcommands.add_parser( + "configure-build", help="Run `configure` for the build Python" + ) + subcommands.add_parser( + "make-build", help="Run `make` for the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for a specific platform and target", + ) + make_host = subcommands.add_parser( + "make-host", + help="Run `make` for a specific platform and target", + ) + package = subcommands.add_parser( + "package", + help="Create a release package for the platform", + ) + build = subcommands.add_parser( + "build", + help="Build all platform targets and create the XCframework", + ) + test = subcommands.add_parser( + "test", + help="Run the testbed for a specific platform", + ) + ci = subcommands.add_parser( + "ci", + help="Run build, package, and test", + ) + + # platform argument + for cmd in [clean, configure_host, make_host, package, build, test, ci]: + cmd.add_argument( + "platform", + choices=HOSTS.keys(), + help="The target platform to build", + ) + + # host triple argument + for cmd in [configure_host, make_host]: + cmd.add_argument( + "host", + help="The host triple to build (e.g., arm64-apple-ios-simulator)", + ) + # optional host triple argument + for cmd in [clean, build, test]: + cmd.add_argument( + "host", + nargs="?", + default="all", + help=( + "The host triple to build (e.g., arm64-apple-ios-simulator), " + "or 'build' for just the build platform, or 'hosts' for all " + "host platforms, or 'all' for the build platform and all " + "hosts. Defaults to 'all'" + ), + ) + + # --clean option + for cmd in [configure_build, configure_host, build, package, test, ci]: + cmd.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete the relevant build directories first", + ) + + # --cache-dir option + for cmd in [configure_host, build, ci]: + cmd.add_argument( + "--cache-dir", + default="./cross-build/downloads", + help="The directory to store cached downloads.", + ) + + # --simulator option + for cmd in [test, ci]: + cmd.add_argument( + "--simulator", + help=( + "The name of the simulator to use (eg: 'iPhone 16e'). " + "Defaults to the most recently released 'entry level' " + "iPhone device. Device architecture and OS version can also " + "be specified; e.g., " + "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would " + "run on an ARM64 iPhone 16 Pro simulator running iOS 26.0." + ), + ) + group = cmd.add_mutually_exclusive_group() + group.add_argument( + "--fast-ci", + action="store_const", + dest="ci_mode", + const="fast", + help="Add test arguments for GitHub Actions", + ) + group.add_argument( + "--slow-ci", + action="store_const", + dest="ci_mode", + const="slow", + help="Add test arguments for buildbots", + ) + + for subcommand in [configure_build, configure_host, build, ci]: + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) + + return parser.parse_args() + + +def print_called_process_error(e: subprocess.CalledProcessError) -> None: + for stream_name in ["stdout", "stderr"]: + content = getattr(e, stream_name) + stream = getattr(sys, stream_name) + if content: + stream.write(content) + if not content.endswith("\n"): + stream.write("\n") + + # shlex uses single quotes, so we surround the command with double quotes. + print( + f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' + ) + + +def main() -> None: + # Handle SIGTERM the same way as SIGINT. This ensures that if we're + # terminated by the buildbot worker, we'll make an attempt to clean up our + # subprocesses. + def signal_handler(*args): + os.kill(os.getpid(), signal.SIGINT) + + signal.signal(signal.SIGTERM, signal_handler) + + # Process command line arguments + context = parse_args() + dispatch: dict[str, Callable] = { + "clean": clean, + "configure-build": configure_build_python, + "make-build": make_build_python, + "configure-host": configure_host_python, + "make-host": make_host_python, + "package": package, + "build": build, + "test": test, + "ci": ci, + } + + try: + dispatch[context.subcommand](context) + except CalledProcessError as e: + print() + print_called_process_error(e) + sys.exit(1) + except RuntimeError as e: + print() + print(e) + sys.exit(2) + + +if __name__ == "__main__": + # Under the buildbot, stdout is not a TTY, but we must still flush after + # every line to make sure our output appears in the correct order relative + # to the output of our subprocesses. + for stream in [sys.stdout, sys.stderr]: + stream.reconfigure(line_buffering=True) + + main() diff --git a/Apple/iOS/README.md b/Apple/iOS/README.md new file mode 100644 index 00000000000000..7ee257b5d648f4 --- /dev/null +++ b/Apple/iOS/README.md @@ -0,0 +1,339 @@ +# Python on iOS README + +**iOS support is [tier 3](https://peps.python.org/pep-0011/#tier-3).** + +This document provides a quick overview of some iOS specific features in the +Python distribution. + +These instructions are only needed if you're planning to compile Python for iOS +yourself. Most users should *not* need to do this. If you're looking to +experiment with writing an iOS app in Python, tools such as [BeeWare's +Briefcase](https://briefcase.readthedocs.io) and [Kivy's +Buildozer](https://buildozer.readthedocs.io) will provide a much more +approachable user experience. + +## Compilers for building on iOS + +Building for iOS requires the use of Apple's Xcode tooling. It is strongly +recommended that you use the most recent stable release of Xcode. This will +require the use of the most (or second-most) recently released macOS version, +as Apple does not maintain Xcode for older macOS versions. The Xcode Command +Line Tools are not sufficient for iOS development; you need a *full* Xcode +install. + +If you want to run your code on the iOS simulator, you'll also need to install +an iOS Simulator Platform. You should be prompted to select an iOS Simulator +Platform when you first run Xcode. Alternatively, you can add an iOS Simulator +Platform by selecting an open the Platforms tab of the Xcode Settings panel. + +## Building Python on iOS + +### ABIs and Architectures + +iOS apps can be deployed on physical devices, and on the iOS simulator. Although +the API used on these devices is identical, the ABI is different - you need to +link against different libraries for an iOS device build (`iphoneos`) or an +iOS simulator build (`iphonesimulator`). + +Apple uses the `XCframework` format to allow specifying a single dependency +that supports multiple ABIs. An `XCframework` is a wrapper around multiple +ABI-specific frameworks that share a common API. + +iOS can also support different CPU architectures within each ABI. At present, +there is only a single supported architecture on physical devices - ARM64. +However, the *simulator* supports 2 architectures - ARM64 (for running on Apple +Silicon machines), and x86_64 (for running on older Intel-based machines). + +To support multiple CPU architectures on a single platform, Apple uses a "fat +binary" format - a single physical file that contains support for multiple +architectures. It is possible to compile and use a "thin" single architecture +version of a binary for testing purposes; however, the "thin" binary will not be +portable to machines using other architectures. + +### Building a multi-architecture iOS XCframework + +The `Apple` subfolder of the Python repository acts as a build script that +can be used to coordinate the compilation of a complete iOS XCframework. To use +it, run:: + + python Apple build iOS + +This will: + +* Configure and compile a version of Python to run on the build machine +* Download pre-compiled binary dependencies for each platform +* Configure and build a `Python.framework` for each required architecture and + iOS SDK +* Merge the multiple `Python.framework` folders into a single `Python.xcframework` +* Produce a `.tar.gz` archive in the `cross-build/dist` folder containing + the `Python.xcframework`, plus a copy of the Testbed app pre-configured to + use the XCframework. + +The `Apple` build script has other entry points that will perform the +individual parts of the overall `build` target, plus targets to test the +build, clean the `cross-build` folder of iOS build products, and perform a +complete "build and test" CI run. The `--clean` flag can also be used on +individual commands to ensure that a stale build product are removed before +building. + +### Building a single-architecture framework + +If you're using the `Apple` build script, you won't need to build +individual frameworks. However, if you do need to manually configure an iOS +Python build for a single framework, the following options are available. + +#### iOS specific arguments to configure + +* `--enable-framework[=DIR]` + + This argument specifies the location where the Python.framework will be + installed. If `DIR` is not specified, the framework will be installed into + a subdirectory of the `iOS/Frameworks` folder. + + This argument *must* be provided when configuring iOS builds. iOS does not + support non-framework builds. + +* `--with-framework-name=NAME` + + Specify the name for the Python framework; defaults to `Python`. + + > [!NOTE] + > Unless you know what you're doing, changing the name of the Python + > framework on iOS is not advised. If you use this option, you won't be able + > to run the `Apple` build script without making significant manual + > alterations, and you won't be able to use any binary packages unless you + > compile them yourself using your own framework name. + +#### Building Python for iOS + +The Python build system will create a `Python.framework` that supports a +*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a +framework to contain non-library content, so the iOS build will produce a +`bin` and `lib` folder in the same output folder as `Python.framework`. +The `lib` folder will be needed at runtime to support the Python library. + +If you want to use Python in a real iOS project, you need to produce multiple +`Python.framework` builds, one for each ABI and architecture. iOS builds of +Python *must* be constructed as framework builds. To support this, you must +provide the `--enable-framework` flag when configuring the build. The build +also requires the use of cross-compilation. The minimal commands for building +Python for the ARM64 iOS simulator will look something like: +``` +export PATH="$(pwd)/Apple/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" +./configure \ + --enable-framework \ + --host=arm64-apple-ios-simulator \ + --build=arm64-apple-darwin \ + --with-build-python=/path/to/python.exe +make +make install +``` + +In this invocation: + +* `Apple/iOS/Resources/bin` has been added to the path, providing some shims for the + compilers and linkers needed by the build. Xcode requires the use of `xcrun` + to invoke compiler tooling. However, if `xcrun` is pre-evaluated and the + result passed to `configure`, these results can embed user- and + version-specific paths into the sysconfig data, which limits the portability + of the compiled Python. Alternatively, if `xcrun` is used *as* the compiler, + it requires that compiler variables like `CC` include spaces, which can + cause significant problems with many C configuration systems which assume that + `CC` will be a single executable. + + To work around this problem, the `Apple/iOS/Resources/bin` folder contains some + wrapper scripts that present as simple compilers and linkers, but wrap + underlying calls to `xcrun`. This allows configure to use a `CC` + definition without spaces, and without user- or version-specific paths, while + retaining the ability to adapt to the local Xcode install. These scripts are + included in the `bin` directory of an iOS install. + + These scripts will, by default, use the currently active Xcode installation. + If you want to use a different Xcode installation, you can use + `xcode-select` to set a new default Xcode globally, or you can use the + `DEVELOPER_DIR` environment variable to specify an Xcode install. The + scripts will use the default `iphoneos`/`iphonesimulator` SDK version for + the select Xcode install; if you want to use a different SDK, you can set the + `IOS_SDK_VERSION` environment variable. (e.g, setting + `IOS_SDK_VERSION=17.1` would cause the scripts to use the `iphoneos17.1` + and `iphonesimulator17.1` SDKs, regardless of the Xcode default.) + + The path has also been cleared of any user customizations. A common source of + bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS + build. Resetting the path to a known "bare bones" value is the easiest way to + avoid these problems. + +* `--host` is the architecture and ABI that you want to build, in GNU compiler + triple format. This will be one of: + + - `arm64-apple-ios` for ARM64 iOS devices. + - `arm64-apple-ios-simulator` for the iOS simulator running on Apple + Silicon devices. + - `x86_64-apple-ios-simulator` for the iOS simulator running on Intel + devices. + +* `--build` is the GNU compiler triple for the machine that will be running + the compiler. This is one of: + + - `arm64-apple-darwin` for Apple Silicon devices. + - `x86_64-apple-darwin` for Intel devices. + +* `/path/to/python.exe` is the path to a Python binary on the machine that + will be running the compiler. This is needed because the Python compilation + process involves running some Python code. On a normal desktop build of + Python, you can compile a python interpreter and then use that interpreter to + run Python code. However, the binaries produced for iOS won't run on macOS, so + you need to provide an external Python interpreter. This interpreter must be + the same version as the Python that is being compiled. To be completely safe, + this should be the *exact* same commit hash. However, the longer a Python + release has been stable, the more likely it is that this constraint can be + relaxed - the same micro version will often be sufficient. + +* The `install` target for iOS builds is slightly different to other + platforms. On most platforms, `make install` will install the build into + the final runtime location. This won't be the case for iOS, as the final + runtime location will be on a physical device. + + However, you still need to run the `install` target for iOS builds, as it + performs some final framework assembly steps. The location specified with + `--enable-framework` will be the location where `make install` will + assemble the complete iOS framework. This completed framework can then + be copied and relocated as required. + +For a full CPython build, you also need to specify the paths to iOS builds of +the binary libraries that CPython depends on (such as XZ, LibFFI and OpenSSL). +This can be done by defining library specific environment variables (such as +`LIBLZMA_CFLAGS`, `LIBLZMA_LIBS`), and the `--with-openssl` configure +option. Versions of these libraries pre-compiled for iOS can be found in [this +repository](https://github.com/beeware/cpython-apple-source-deps/releases). +LibFFI is especially important, as many parts of the standard library +(including the `platform`, `sysconfig` and `webbrowser` modules) require +the use of the `ctypes` module at runtime. + +By default, Python will be compiled with an iOS deployment target (i.e., the +minimum supported iOS version) of 13.0. To specify a different deployment +target, provide the version number as part of the `--host` argument - for +example, `--host=arm64-apple-ios15.4-simulator` would compile an ARM64 +simulator build with a deployment target of 15.4. + +## Testing Python on iOS + +### Testing a multi-architecture framework + +Once you have a built an XCframework, you can test that framework by running: + + $ python Apple test iOS + +This test will attempt to find an "SE-class" simulator (i.e., an iPhone SE, or +iPhone 16e, or similar), and run the test suite on the most recent version of +iOS that is available. You can specify a simulator using the `--simulator` +command line argument, providing the name of the simulator (e.g., `--simulator +'iPhone 16 Pro'`). You can also use this argument to control the OS version used +for testing; `--simulator 'iPhone 16 Pro,OS=18.2'` would attempt to run the +tests on an iPhone 16 Pro running iOS 18.2. + +If the test runner is executed on GitHub Actions, the `GITHUB_ACTIONS` +environment variable will be exposed to the iOS process at runtime. + +### Testing a single-architecture framework + +The `Apple/testbed` folder that contains an Xcode project that is able to run +the Python test suite on Apple platforms. This project converts the Python test +suite into a single test case in Xcode's XCTest framework. The single XCTest +passes if the test suite passes. + +To run the test suite, configure a Python build for an iOS simulator (i.e., +`--host=arm64-apple-ios-simulator` or `--host=x86_64-apple-ios-simulator` +), specifying a framework build (i.e. `--enable-framework`). Ensure that your +`PATH` has been configured to include the `Apple/iOS/Resources/bin` folder and +exclude any non-iOS tools, then run: +``` +make all +make install +make testios +``` + +This will: + +* Build an iOS framework for your chosen architecture; +* Finalize the single-platform framework; +* Make a clean copy of the testbed project; +* Install the Python iOS framework into the copy of the testbed project; and +* Run the test suite on an "entry-level device" simulator (i.e., an iPhone SE, + iPhone 16e, or a similar). + +On success, the test suite will exit and report successful completion of the +test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 15 +minutes to run; a couple of extra minutes is required to compile the testbed +project, and then boot and prepare the iOS simulator. + +### Debugging test failures + +Running `python Apple test iOS` generates a standalone version of the +`Apple/testbed` project, and runs the full test suite. It does this using +`Apple/testbed` itself - the folder is an executable module that can be used +to create and run a clone of the testbed project. The standalone version of the +testbed will be created in a directory named +`cross-build/iOS-testbed.<timestamp>`. + +You can generate your own standalone testbed instance by running: +``` +python cross-build/iOS/testbed clone my-testbed +``` + +In this invocation, `my-testbed` is the name of the folder for the new +testbed clone. + +If you've built your own XCframework, or you only want to test a single architecture, +you can construct a standalone testbed instance by running: +``` +python Apple/testbed clone --platform iOS --framework <path/to/framework> my-testbed +``` + +The framework path can be the path path to a `Python.xcframework`, or the +path to a folder that contains a single-platform `Python.framework`. + +You can then use the `my-testbed` folder to run the Python test suite, +passing in any command line arguments you may require. For example, if you're +trying to diagnose a failure in the `os` module, you might run: +``` +python my-testbed run -- test -W test_os +``` + +This is the equivalent of running `python -m test -W test_os` on a desktop +Python build. Any arguments after the `--` will be passed to testbed as if +they were arguments to `python -m` on a desktop machine. + +### Testing in Xcode + +You can also open the testbed project in Xcode by running: +``` +open my-testbed/iOSTestbed.xcodeproj +``` + +This will allow you to use the full Xcode suite of tools for debugging. + +The arguments used to run the test suite are defined as part of the test plan. +To modify the test plan, select the test plan node of the project tree (it +should be the first child of the root node), and select the "Configurations" +tab. Modify the "Arguments Passed On Launch" value to change the testing +arguments. + +The test plan also disables parallel testing, and specifies the use of the +`Testbed.lldbinit` file for providing configuration of the debugger. The +default debugger configuration disables automatic breakpoints on the +`SIGINT`, `SIGUSR1`, `SIGUSR2`, and `SIGXFSZ` signals. + +### Testing on an iOS device + +To test on an iOS device, the app needs to be signed with known developer +credentials. To obtain these credentials, you must have an iOS Developer +account, and your Xcode install will need to be logged into your account (see +the Accounts tab of the Preferences dialog). + +Once the project is open, and you're signed into your Apple Developer account, +select the root node of the project tree (labeled "iOSTestbed"), then the +"Signing & Capabilities" tab in the details page. Select a development team +(this will likely be your own name), and plug in a physical device to your +macOS machine with a USB cable. You should then be able to select your physical +device from the list of targets in the pulldown in the Xcode titlebar. diff --git a/iOS/Resources/Info.plist.in b/Apple/iOS/Resources/Info.plist.in similarity index 100% rename from iOS/Resources/Info.plist.in rename to Apple/iOS/Resources/Info.plist.in diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-ar b/Apple/iOS/Resources/bin/arm64-apple-ios-ar new file mode 100755 index 00000000000000..3cf3eb218741fa --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} ar "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-clang b/Apple/iOS/Resources/bin/arm64-apple-ios-clang new file mode 100755 index 00000000000000..f50d5b5142fc76 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ b/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ new file mode 100755 index 00000000000000..0794731d7dcbda --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-cpp b/Apple/iOS/Resources/bin/arm64-apple-ios-cpp new file mode 100755 index 00000000000000..24fa1506bab827 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} -E "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar new file mode 100755 index 00000000000000..b836b6db9025bb --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang new file mode 100755 index 00000000000000..4891a00876e0bd --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ new file mode 100755 index 00000000000000..58b2a5f6f18c2b --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp new file mode 100755 index 00000000000000..c9df94e8b7c837 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip new file mode 100755 index 00000000000000..fd59d309b73a20 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-strip b/Apple/iOS/Resources/bin/arm64-apple-ios-strip new file mode 100755 index 00000000000000..75e823a3d02d61 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar new file mode 100755 index 00000000000000..b836b6db9025bb --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang new file mode 100755 index 00000000000000..f4739a7b945d01 --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ new file mode 100755 index 00000000000000..c348ae4c10395b --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp new file mode 100755 index 00000000000000..6d7f8084c9fdcc --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip new file mode 100755 index 00000000000000..c5cfb28929195a --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch x86_64 "$@" diff --git a/iOS/Resources/pyconfig.h b/Apple/iOS/Resources/pyconfig.h similarity index 100% rename from iOS/Resources/pyconfig.h rename to Apple/iOS/Resources/pyconfig.h diff --git a/iOS/testbed/Python.xcframework/Info.plist b/Apple/testbed/Python.xcframework/Info.plist similarity index 100% rename from iOS/testbed/Python.xcframework/Info.plist rename to Apple/testbed/Python.xcframework/Info.plist diff --git a/iOS/Resources/dylib-Info-template.plist b/Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist similarity index 96% rename from iOS/Resources/dylib-Info-template.plist rename to Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist index f652e272f71c88..d6caa01c1e44b9 100644 --- a/iOS/Resources/dylib-Info-template.plist +++ b/Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist @@ -19,7 +19,7 @@ <string>iPhoneOS</string> </array> <key>MinimumOSVersion</key> - <string>12.0</string> + <string>13.0</string> <key>CFBundleVersion</key> <string>1</string> </dict> diff --git a/Apple/testbed/Python.xcframework/build/utils.sh b/Apple/testbed/Python.xcframework/build/utils.sh new file mode 100755 index 00000000000000..e7155d8b30e213 --- /dev/null +++ b/Apple/testbed/Python.xcframework/build/utils.sh @@ -0,0 +1,151 @@ +# Utility methods for use in an Xcode project. +# +# An iOS XCframework cannot include any content other than the library binary +# and relevant metadata. However, Python requires a standard library at runtime. +# Therefore, it is necessary to add a build step to an Xcode app target that +# processes the standard library and puts the content into the final app. +# +# In general, these tools will be invoked after bundle resources have been +# copied into the app, but before framework embedding (and signing). +# +# The following is an example script, assuming that: +# * Python.xcframework is in the root of the project +# * There is an `app` folder that contains the app code +# * There is an `app_packages` folder that contains installed Python packages. +# ----- +# set -e +# source $PROJECT_DIR/Python.xcframework/build/build_utils.sh +# install_python Python.xcframework app app_packages +# ----- + +# Copy the standard library from the XCframework into the app bundle. +# +# Accepts one argument: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. +install_stdlib() { + PYTHON_XCFRAMEWORK_PATH=$1 + + mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib" + if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then + echo "Installing Python modules for iOS Simulator" + if [ -d "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/ios-arm64-simulator" ]; then + SLICE_FOLDER="ios-arm64-simulator" + else + SLICE_FOLDER="ios-arm64_x86_64-simulator" + fi + else + echo "Installing Python modules for iOS Device" + SLICE_FOLDER="ios-arm64" + fi + + # If the XCframework has a shared lib folder, then it's a full framework. + # Copy both the common and slice-specific part of the lib directory. + # Otherwise, it's a single-arch framework; use the "full" lib folder. + if [ -d "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib" ]; then + rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + rsync -au "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib-$ARCHS/" "$CODESIGNING_FOLDER_PATH/python/lib/" + else + # A single-arch framework will have a libpython symlink; that can't be included at runtime + rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" --exclude 'libpython*.dylib' + fi +} + +# Convert a single .so library into a framework that iOS can load. +# +# Accepts three arguments: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. +# 2. The base path, relative to the installed location in the app bundle, that +# needs to be processed. Any .so file found in this path (or a subdirectory +# of it) will be processed. +# 2. The full path to a single .so file to process. This path should include +# the base path. +install_dylib () { + PYTHON_XCFRAMEWORK_PATH=$1 + INSTALL_BASE=$2 + FULL_EXT=$3 + + # The name of the extension file + EXT=$(basename "$FULL_EXT") + # The name and location of the module + MODULE_PATH=$(dirname "$FULL_EXT") + MODULE_NAME=$(echo $EXT | cut -d "." -f 1) + # The location of the extension file, relative to the bundle + RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} + # The path to the extension file, relative to the install base + PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/} + # The full dotted name of the extension module, constructed from the file path. + FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" "."); + # A bundle identifier; not actually used, but required by Xcode framework packaging + FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-") + # The name of the framework folder. + FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework" + + # If the framework folder doesn't exist, create it. + if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then + echo "Creating framework for $RELATIVE_EXT" + mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" + cp "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/build/$PLATFORM_FAMILY_NAME-dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + fi + + echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + # Create a placeholder .fwork file where the .so was + echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork + # Create a back reference to the .so file location in the framework + echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin" + + # If the framework provides an xcprivacy file, install it. + if [ -e "$MODULE_PATH/$MODULE_NAME.xcprivacy" ]; then + echo "Installing XCPrivacy file for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + XCPRIVACY_FILE="$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/PrivacyInfo.xcprivacy" + if [ -e "$XCPRIVACY_FILE" ]; then + rm -rf "$XCPRIVACY_FILE" + fi + mv "$MODULE_PATH/$MODULE_NAME.xcprivacy" "$XCPRIVACY_FILE" + fi + + echo "Signing framework as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..." + /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" +} + +# Process all the dynamic libraries in a path into Framework format. +# +# Accepts two arguments: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. +# 2. The base path, relative to the installed location in the app bundle, that +# needs to be processed. Any .so file found in this path (or a subdirectory +# of it) will be processed. +process_dylibs () { + PYTHON_XCFRAMEWORK_PATH=$1 + LIB_PATH=$2 + find "$CODESIGNING_FOLDER_PATH/$LIB_PATH" -name "*.so" | while read FULL_EXT; do + install_dylib $PYTHON_XCFRAMEWORK_PATH "$LIB_PATH/" "$FULL_EXT" + done +} + +# The entry point for post-processing a Python XCframework. +# +# Accepts 1 or more arguments: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. If the XCframework is in the root of the project, +# 2+. The path of a package, relative to the root of the packaged app, that contains +# library content that should be processed for binary libraries. +install_python() { + PYTHON_XCFRAMEWORK_PATH=$1 + shift + + install_stdlib $PYTHON_XCFRAMEWORK_PATH + PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib") + echo "Install Python $PYTHON_VER standard library extension modules..." + process_dylibs $PYTHON_XCFRAMEWORK_PATH python/lib/$PYTHON_VER/lib-dynload + + for package_path in $@; do + echo "Installing $package_path extension modules ..." + process_dylibs $PYTHON_XCFRAMEWORK_PATH $package_path + done +} diff --git a/iOS/testbed/Python.xcframework/ios-arm64/README b/Apple/testbed/Python.xcframework/ios-arm64/README similarity index 100% rename from iOS/testbed/Python.xcframework/ios-arm64/README rename to Apple/testbed/Python.xcframework/ios-arm64/README diff --git a/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README b/Apple/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README similarity index 100% rename from iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README rename to Apple/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README diff --git a/Apple/testbed/Testbed.lldbinit b/Apple/testbed/Testbed.lldbinit new file mode 100644 index 00000000000000..4cf00dd0f9de1d --- /dev/null +++ b/Apple/testbed/Testbed.lldbinit @@ -0,0 +1,4 @@ +process handle SIGINT -n true -p true -s false +process handle SIGUSR1 -n true -p true -s false +process handle SIGUSR2 -n true -p true -s false +process handle SIGXFSZ -n true -p true -s false diff --git a/Apple/testbed/TestbedTests/TestbedTests.m b/Apple/testbed/TestbedTests/TestbedTests.m new file mode 100644 index 00000000000000..f7788c47f2c229 --- /dev/null +++ b/Apple/testbed/TestbedTests/TestbedTests.m @@ -0,0 +1,200 @@ +#import <XCTest/XCTest.h> +#import <Python/Python.h> + +@interface TestbedTests : XCTestCase + +@end + +@implementation TestbedTests + + +- (void)testPython { + const char **argv; + int exit_code; + int failed; + PyStatus status; + PyPreConfig preconfig; + PyConfig config; + PyObject *app_packages_path; + PyObject *method_args; + PyObject *result; + PyObject *site_module; + PyObject *site_addsitedir_attr; + PyObject *sys_module; + PyObject *sys_path_attr; + NSArray *test_args; + NSString *python_home; + NSString *path; + wchar_t *wtmp_str; + + NSString *resourcePath = [[NSBundle mainBundle] resourcePath]; + + // Set some other common environment indicators to disable color, as the + // Xcode log can't display color. Stdout will report that it is *not* a + // TTY. + setenv("NO_COLOR", "1", true); + setenv("PYTHON_COLORS", "0", true); + + if (getenv("GITHUB_ACTIONS")) { + NSLog(@"Running in a GitHub Actions environment"); + } + // Arguments to pass into the test suite runner. + // argv[0] must identify the process; any subsequent arg + // will be handled as if it were an argument to `python -m test` + // The processInfo arguments contain the binary that is running, + // followed by the arguments defined in the test plan. This means: + // run_module = test_args[1] + // argv = ["Testbed"] + test_args[2:] + test_args = [[NSProcessInfo processInfo] arguments]; + if (test_args == NULL) { + NSLog(@"Unable to identify test arguments."); + } + NSLog(@"Test arguments: %@", test_args); + argv = malloc(sizeof(char *) * ([test_args count] - 1)); + argv[0] = "Testbed"; + for (int i = 1; i < [test_args count] - 1; i++) { + argv[i] = [[test_args objectAtIndex:i+1] UTF8String]; + } + + // Generate an isolated Python configuration. + NSLog(@"Configuring isolated Python..."); + PyPreConfig_InitIsolatedConfig(&preconfig); + PyConfig_InitIsolatedConfig(&config); + + // Configure the Python interpreter: + // Enforce UTF-8 encoding for stderr, stdout, file-system encoding and locale. + // See https://docs.python.org/3/library/os.html#python-utf-8-mode. + preconfig.utf8_mode = 1; + // Use the system logger for stdout/err + config.use_system_logger = 1; + // Don't buffer stdio. We want output to appears in the log immediately + config.buffered_stdio = 0; + // Don't write bytecode; we can't modify the app bundle + // after it has been signed. + config.write_bytecode = 0; + // Ensure that signal handlers are installed + config.install_signal_handlers = 1; + // Run the test module. + config.run_module = Py_DecodeLocale([[test_args objectAtIndex:1] UTF8String], NULL); + // For debugging - enable verbose mode. + // config.verbose = 1; + + NSLog(@"Pre-initializing Python runtime..."); + status = Py_PreInitialize(&preconfig); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to pre-initialize Python interpreter: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + // Set the home for the Python interpreter + python_home = [NSString stringWithFormat:@"%@/python", resourcePath, nil]; + NSLog(@"PythonHome: %@", python_home); + wtmp_str = Py_DecodeLocale([python_home UTF8String], NULL); + status = PyConfig_SetString(&config, &config.home, wtmp_str); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to set PYTHONHOME: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + PyMem_RawFree(wtmp_str); + + // Read the site config + status = PyConfig_Read(&config); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to read site config: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + NSLog(@"Configure argc/argv..."); + status = PyConfig_SetBytesArgv(&config, [test_args count] - 1, (char**) argv); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to configure argc/argv: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + NSLog(@"Initializing Python runtime..."); + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + XCTFail(@"Unable to initialize Python interpreter: %s", status.err_msg); + PyConfig_Clear(&config); + return; + } + + // Add app_packages as a site directory. This both adds to sys.path, + // and ensures that any .pth files in that directory will be executed. + site_module = PyImport_ImportModule("site"); + if (site_module == NULL) { + XCTFail(@"Could not import site module"); + return; + } + + site_addsitedir_attr = PyObject_GetAttrString(site_module, "addsitedir"); + if (site_addsitedir_attr == NULL || !PyCallable_Check(site_addsitedir_attr)) { + XCTFail(@"Could not access site.addsitedir"); + return; + } + + path = [NSString stringWithFormat:@"%@/app_packages", resourcePath, nil]; + NSLog(@"App packages path: %@", path); + wtmp_str = Py_DecodeLocale([path UTF8String], NULL); + app_packages_path = PyUnicode_FromWideChar(wtmp_str, wcslen(wtmp_str)); + if (app_packages_path == NULL) { + XCTFail(@"Could not convert app_packages path to unicode"); + return; + } + PyMem_RawFree(wtmp_str); + + method_args = Py_BuildValue("(O)", app_packages_path); + if (method_args == NULL) { + XCTFail(@"Could not create arguments for site.addsitedir"); + return; + } + + result = PyObject_CallObject(site_addsitedir_attr, method_args); + if (result == NULL) { + XCTFail(@"Could not add app_packages directory using site.addsitedir"); + return; + } + + // Add test code to sys.path + sys_module = PyImport_ImportModule("sys"); + if (sys_module == NULL) { + XCTFail(@"Could not import sys module"); + return; + } + + sys_path_attr = PyObject_GetAttrString(sys_module, "path"); + if (sys_path_attr == NULL) { + XCTFail(@"Could not access sys.path"); + return; + } + + path = [NSString stringWithFormat:@"%@/app", resourcePath, nil]; + NSLog(@"App path: %@", path); + wtmp_str = Py_DecodeLocale([path UTF8String], NULL); + failed = PyList_Insert(sys_path_attr, 0, PyUnicode_FromString([path UTF8String])); + if (failed) { + XCTFail(@"Unable to add app to sys.path"); + return; + } + PyMem_RawFree(wtmp_str); + + // Ensure the working directory is the app folder. + chdir([path UTF8String]); + + // Start the test suite. Print a separator to differentiate Python startup logs from app logs + NSLog(@"---------------------------------------------------------------------------"); + + exit_code = Py_RunMain(); + XCTAssertEqual(exit_code, 0, @"Test suite did not pass"); + + NSLog(@"---------------------------------------------------------------------------"); + + Py_Finalize(); +} + + +@end diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py new file mode 100644 index 00000000000000..0dd77ab8b82797 --- /dev/null +++ b/Apple/testbed/__main__.py @@ -0,0 +1,435 @@ +import argparse +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +TEST_SLICES = { + "iOS": "ios-arm64_x86_64-simulator", +} + +DECODE_ARGS = ("UTF-8", "backslashreplace") + +# The system log prefixes each line: +# 2025-01-17 16:14:29.093742+0800 iOSTestbed[23987:1fd393b4] ... +# 2025-01-17 16:14:29.093742+0800 iOSTestbed[23987:1fd393b4] ... + +LOG_PREFIX_REGEX = re.compile( + r"^\d{4}-\d{2}-\d{2}" # YYYY-MM-DD + r"\s+\d+:\d{2}:\d{2}\.\d+\+\d{4}" # HH:MM:SS.ssssss+ZZZZ + r"\s+iOSTestbed\[\d+:\w+\]" # Process/thread ID +) + + +# Select a simulator device to use. +def select_simulator_device(platform): + # List the testing simulators, in JSON format + raw_json = subprocess.check_output(["xcrun", "simctl", "list", "-j"]) + json_data = json.loads(raw_json) + + if platform == "iOS": + # Any iOS device will do; we'll look for "SE" devices - but the name + # isn't consistent over time. Older Xcode versions will use "iPhone SE + # (Nth generation)"; As of 2025, they've started using "iPhone 16e". + # + # When Xcode is updated after a new release, new devices will be + # available and old ones will be dropped from the set available on the + # latest iOS version. Select the one with the highest minimum runtime + # version - this is an indicator of the "newest" released device, which + # should always be supported on the "most recent" iOS version. + se_simulators = sorted( + (devicetype["minRuntimeVersion"], devicetype["name"]) + for devicetype in json_data["devicetypes"] + if devicetype["productFamily"] == "iPhone" + and ( + ( + "iPhone " in devicetype["name"] + and devicetype["name"].endswith("e") + ) + or "iPhone SE " in devicetype["name"] + ) + ) + simulator = se_simulators[-1][1] + else: + raise ValueError(f"Unknown platform {platform}") + + return simulator + + +def xcode_test(location: Path, platform: str, simulator: str, verbose: bool): + # Build and run the test suite on the named simulator. + args = [ + "-project", + str(location / f"{platform}Testbed.xcodeproj"), + "-scheme", + f"{platform}Testbed", + "-destination", + f"platform={platform} Simulator,name={simulator}", + "-derivedDataPath", + str(location / "DerivedData"), + ] + verbosity_args = [] if verbose else ["-quiet"] + + print("Building test project...") + subprocess.run( + ["xcodebuild", "build-for-testing"] + args + verbosity_args, + check=True, + ) + + # Any environment variable prefixed with TEST_RUNNER_ is exposed into the + # test runner environment. There are some variables (like those identifying + # CI platforms) that can be useful to have access to. + test_env = os.environ.copy() + if "GITHUB_ACTIONS" in os.environ: + test_env["TEST_RUNNER_GITHUB_ACTIONS"] = os.environ["GITHUB_ACTIONS"] + + print("Running test project...") + # Test execution *can't* be run -quiet; verbose mode + # is how we see the output of the test output. + process = subprocess.Popen( + ["xcodebuild", "test-without-building"] + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=test_env, + ) + while line := (process.stdout.readline()).decode(*DECODE_ARGS): + # Strip the timestamp/process prefix from each log line + line = LOG_PREFIX_REGEX.sub("", line) + sys.stdout.write(line) + sys.stdout.flush() + + status = process.wait(timeout=5) + exit(status) + + +def copy(src, tgt): + """An all-purpose copy. + + If src is a file, it is copied. If src is a symlink, it is copied *as a + symlink*. If src is a directory, the full tree is duplicated, with symlinks + being preserved. + """ + if src.is_file() or src.is_symlink(): + shutil.copyfile(src, tgt, follow_symlinks=False) + else: + shutil.copytree(src, tgt, symlinks=True) + + +def clone_testbed( + source: Path, + target: Path, + framework: Path, + platform: str, + apps: list[Path], +) -> None: + if target.exists(): + print(f"{target} already exists; aborting without creating project.") + sys.exit(10) + + if framework is None: + if not ( + source / "Python.xcframework" / TEST_SLICES[platform] / "bin" + ).is_dir(): + print( + f"The testbed being cloned ({source}) does not contain " + "a framework with slices. Re-run with --framework" + ) + sys.exit(11) + else: + if not framework.is_dir(): + print(f"{framework} does not exist.") + sys.exit(12) + elif not ( + framework.suffix == ".xcframework" + or (framework / "Python.framework").is_dir() + ): + print( + f"{framework} is not an XCframework, " + f"or a simulator slice of a framework build." + ) + sys.exit(13) + + print("Cloning testbed project:") + print(f" Cloning {source}...", end="") + # Only copy the files for the platform being cloned plus the files common + # to all platforms. The XCframework will be copied later, if needed. + target.mkdir(parents=True) + + for name in [ + "__main__.py", + "TestbedTests", + "Testbed.lldbinit", + f"{platform}Testbed", + f"{platform}Testbed.xcodeproj", + f"{platform}Testbed.xctestplan", + ]: + copy(source / name, target / name) + + print(" done") + + orig_xc_framework_path = source / "Python.xcframework" + xc_framework_path = target / "Python.xcframework" + test_framework_path = xc_framework_path / TEST_SLICES[platform] + if framework is not None: + if framework.suffix == ".xcframework": + print(" Installing XCFramework...", end="") + xc_framework_path.symlink_to( + framework.relative_to(xc_framework_path.parent, walk_up=True) + ) + print(" done") + else: + print(" Installing simulator framework...", end="") + # We're only installing a slice of a framework; we need + # to do a full tree copy to make sure we don't damage + # symlinked content. + shutil.copytree(orig_xc_framework_path, xc_framework_path) + if test_framework_path.is_dir(): + shutil.rmtree(test_framework_path) + else: + test_framework_path.unlink(missing_ok=True) + test_framework_path.symlink_to( + framework.relative_to(test_framework_path.parent, walk_up=True) + ) + print(" done") + else: + copy(orig_xc_framework_path, xc_framework_path) + + if ( + xc_framework_path.is_symlink() + and not xc_framework_path.readlink().is_absolute() + ): + # XCFramework is a relative symlink. Rewrite the symlink relative + # to the new location. + print(" Rewriting symlink to XCframework...", end="") + resolved_xc_framework_path = ( + source / xc_framework_path.readlink() + ).resolve() + xc_framework_path.unlink() + xc_framework_path.symlink_to( + resolved_xc_framework_path.relative_to( + xc_framework_path.parent, walk_up=True + ) + ) + print(" done") + elif ( + test_framework_path.is_symlink() + and not test_framework_path.readlink().is_absolute() + ): + print(" Rewriting symlink to simulator framework...", end="") + # Simulator framework is a relative symlink. Rewrite the symlink + # relative to the new location. + orig_test_framework_path = ( + source / "Python.XCframework" / test_framework_path.readlink() + ).resolve() + test_framework_path.unlink() + test_framework_path.symlink_to( + orig_test_framework_path.relative_to( + test_framework_path.parent, walk_up=True + ) + ) + print(" done") + else: + print(" Using pre-existing Python framework.") + + for app_src in apps: + print(f" Installing app {app_src.name!r}...", end="") + app_target = target / f"Testbed/app/{app_src.name}" + if app_target.is_dir(): + shutil.rmtree(app_target) + shutil.copytree(app_src, app_target) + print(" done") + + print(f"Successfully cloned testbed: {target.resolve()}") + + +def update_test_plan(testbed_path, platform, args): + # Modify the test plan to use the requested test arguments. + test_plan_path = testbed_path / f"{platform}Testbed.xctestplan" + with test_plan_path.open("r", encoding="utf-8") as f: + test_plan = json.load(f) + + test_plan["defaultOptions"]["commandLineArgumentEntries"] = [ + {"argument": shlex.quote(arg)} for arg in args + ] + + with test_plan_path.open("w", encoding="utf-8") as f: + json.dump(test_plan, f, indent=2) + + +def run_testbed( + platform: str, + simulator: str | None, + args: list[str], + verbose: bool = False, +): + location = Path(__file__).parent + print("Updating test plan...", end="") + update_test_plan(location, platform, args) + print(" done.") + + if simulator is None: + simulator = select_simulator_device(platform) + print(f"Running test on {simulator}") + + xcode_test( + location, + platform=platform, + simulator=simulator, + verbose=verbose, + ) + + +def main(): + # Look for directories like `iOSTestbed` as an indicator of the platforms + # that the testbed folder supports. The original source testbed can support + # many platforms, but when cloned, only one platform is preserved. + available_platforms = [ + platform + for platform in ["iOS"] + if (Path(__file__).parent / f"{platform}Testbed").is_dir() + ] + + parser = argparse.ArgumentParser( + description=( + "Manages the process of testing an Apple Python project " + "through Xcode." + ), + ) + + subcommands = parser.add_subparsers(dest="subcommand") + clone = subcommands.add_parser( + "clone", + description=( + "Clone the testbed project, copying in a Python framework and" + "any specified application code." + ), + help="Clone a testbed project to a new location.", + ) + clone.add_argument( + "--framework", + help=( + "The location of the XCFramework (or simulator-only slice of an " + "XCFramework) to use when running the testbed" + ), + ) + clone.add_argument( + "--platform", + dest="platform", + choices=available_platforms, + default=available_platforms[0], + help=f"The platform to target (default: {available_platforms[0]})", + ) + clone.add_argument( + "--app", + dest="apps", + action="append", + default=[], + help="The location of any code to include in the testbed project", + ) + clone.add_argument( + "location", + help="The path where the testbed will be cloned.", + ) + + run = subcommands.add_parser( + "run", + usage=( + "%(prog)s [-h] [--simulator SIMULATOR] -- " + "<test arg> [<test arg> ...]" + ), + description=( + "Run a testbed project. The arguments provided after `--` will be " + "passed to the running iOS process as if they were arguments to " + "`python -m`." + ), + help="Run a testbed project", + ) + run.add_argument( + "--platform", + dest="platform", + choices=available_platforms, + default=available_platforms[0], + help=f"The platform to target (default: {available_platforms[0]})", + ) + run.add_argument( + "--simulator", + help=( + "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to " + "the most recently released 'entry level' iPhone device. Device " + "architecture and OS version can also be specified; e.g., " + "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would run on " + "an ARM64 iPhone 16 Pro simulator running iOS 26.0." + ), + ) + run.add_argument( + "-v", + "--verbose", + action="store_true", + help="Enable verbose output", + ) + + try: + pos = sys.argv.index("--") + testbed_args = sys.argv[1:pos] + test_args = sys.argv[pos + 1 :] + except ValueError: + testbed_args = sys.argv[1:] + test_args = [] + + context = parser.parse_args(testbed_args) + + if context.subcommand == "clone": + clone_testbed( + source=Path(__file__).parent.resolve(), + target=Path(context.location).resolve(), + framework=Path(context.framework).resolve() + if context.framework + else None, + platform=context.platform, + apps=[Path(app) for app in context.apps], + ) + elif context.subcommand == "run": + if test_args: + if not ( + Path(__file__).parent + / "Python.xcframework" + / TEST_SLICES[context.platform] + / "bin" + ).is_dir(): + print( + "Testbed does not contain a compiled Python framework. " + f"Use `python {sys.argv[0]} clone ...` to create a " + "runnable clone of this testbed." + ) + sys.exit(20) + + run_testbed( + platform=context.platform, + simulator=context.simulator, + verbose=context.verbose, + args=test_args, + ) + else: + print( + "Must specify test arguments " + f"(e.g., {sys.argv[0]} run -- test)" + ) + print() + parser.print_help(sys.stderr) + sys.exit(21) + else: + parser.print_help(sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + # Under the buildbot, stdout is not a TTY, but we must still flush after + # every line to make sure our output appears in the correct order relative + # to the output of our subprocesses. + for stream in [sys.stdout, sys.stderr]: + stream.reconfigure(line_buffering=True) + main() diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj similarity index 79% rename from iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj rename to Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj index d57cfc3dbe0304..f8835a3bc587df 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -11,12 +11,13 @@ 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 607A66212B0EFA390010BFC8 /* Assets.xcassets */; }; 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */; }; 607A66282B0EFA390010BFC8 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66272B0EFA390010BFC8 /* main.m */; }; - 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */; }; + 607A66322B0EFA3A0010BFC8 /* TestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */; }; 607A664C2B0EFC080010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; 607A664D2B0EFC080010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; 607A66502B0EFFE00010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; 607A66512B0EFFE00010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; - 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */ = {isa = PBXBuildFile; fileRef = 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */; }; + 608619542CB77BA900F46182 /* app_packages in Resources */ = {isa = PBXBuildFile; fileRef = 608619532CB77BA900F46182 /* app_packages */; }; + 608619562CB7819B00F46182 /* app in Resources */ = {isa = PBXBuildFile; fileRef = 608619552CB7819B00F46182 /* app */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -62,10 +63,12 @@ 607A66242B0EFA390010BFC8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; }; 607A66272B0EFA390010BFC8 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = "<group>"; }; 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = iOSTestbedTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; - 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = iOSTestbedTests.m; sourceTree = "<group>"; }; + 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = TestbedTests.m; sourceTree = "<group>"; }; 607A664A2B0EFB310010BFC8 /* Python.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = Python.xcframework; sourceTree = "<group>"; }; - 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "dylib-Info-template.plist"; sourceTree = "<group>"; }; 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "iOSTestbed-Info.plist"; sourceTree = "<group>"; }; + 608619532CB77BA900F46182 /* app_packages */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app_packages; sourceTree = "<group>"; }; + 608619552CB7819B00F46182 /* app */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app; sourceTree = "<group>"; }; + 60FE0EFB2E56BB6D00524F87 /* iOSTestbed.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = iOSTestbed.xctestplan; sourceTree = "<group>"; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -91,9 +94,10 @@ 607A66092B0EFA380010BFC8 = { isa = PBXGroup; children = ( + 60FE0EFB2E56BB6D00524F87 /* iOSTestbed.xctestplan */, 607A664A2B0EFB310010BFC8 /* Python.xcframework */, 607A66142B0EFA380010BFC8 /* iOSTestbed */, - 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */, + 607A66302B0EFA3A0010BFC8 /* TestbedTests */, 607A66132B0EFA380010BFC8 /* Products */, 607A664F2B0EFFE00010BFC8 /* Frameworks */, ); @@ -111,8 +115,9 @@ 607A66142B0EFA380010BFC8 /* iOSTestbed */ = { isa = PBXGroup; children = ( + 608619552CB7819B00F46182 /* app */, + 608619532CB77BA900F46182 /* app_packages */, 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */, - 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */, 607A66152B0EFA380010BFC8 /* AppDelegate.h */, 607A66162B0EFA380010BFC8 /* AppDelegate.m */, 607A66212B0EFA390010BFC8 /* Assets.xcassets */, @@ -122,12 +127,12 @@ path = iOSTestbed; sourceTree = "<group>"; }; - 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */ = { + 607A66302B0EFA3A0010BFC8 /* TestbedTests */ = { isa = PBXGroup; children = ( - 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */, + 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */, ); - path = iOSTestbedTests; + path = TestbedTests; sourceTree = "<group>"; }; 607A664F2B0EFFE00010BFC8 /* Frameworks */ = { @@ -147,8 +152,7 @@ 607A660E2B0EFA380010BFC8 /* Sources */, 607A660F2B0EFA380010BFC8 /* Frameworks */, 607A66102B0EFA380010BFC8 /* Resources */, - 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */, - 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */, + 607A66552B0F061D0010BFC8 /* Process Python libraries */, 607A664E2B0EFC080010BFC8 /* Embed Frameworks */, ); buildRules = ( @@ -222,8 +226,9 @@ buildActionMask = 2147483647; files = ( 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */, - 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */, + 608619562CB7819B00F46182 /* app in Resources */, 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */, + 608619542CB77BA900F46182 /* app_packages in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -237,7 +242,7 @@ /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ - 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */ = { + 607A66552B0F061D0010BFC8 /* Process Python libraries */ = { isa = PBXShellScriptBuildPhase; alwaysOutOfDate = 1; buildActionMask = 2147483647; @@ -247,33 +252,15 @@ ); inputPaths = ( ); - name = "Install Target Specific Python Standard Library"; + name = "Process Python libraries"; outputFileListPaths = ( ); outputPaths = ( ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\nmkdir -p \"$CODESIGNING_FOLDER_PATH/python/lib\"\nif [ \"$EFFECTIVE_PLATFORM_NAME\" = \"-iphonesimulator\" ]; then\n echo \"Installing Python modules for iOS Simulator\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nelse\n echo \"Installing Python modules for iOS Device\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nfi\n"; - }; - 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */ = { - isa = PBXShellScriptBuildPhase; - alwaysOutOfDate = 1; - buildActionMask = 2147483647; - files = ( - ); - inputFileListPaths = ( - ); - inputPaths = ( - ); - name = "Prepare Python Binary Modules"; - outputFileListPaths = ( - ); - outputPaths = ( - ); - runOnlyForDeploymentPostprocessing = 0; - shellPath = /bin/sh; - shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + shellScript = "set -e\nsource $PROJECT_DIR/Python.xcframework/build/utils.sh\ninstall_python Python.xcframework app app_packages\n"; + showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ @@ -291,7 +278,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */, + 607A66322B0EFA3A0010BFC8 /* TestbedTests.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -369,7 +356,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; @@ -424,7 +411,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; @@ -450,7 +437,7 @@ INFOPLIST_KEY_UIMainStoryboardFile = Main; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -481,7 +468,7 @@ INFOPLIST_KEY_UIMainStoryboardFile = Main; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -504,7 +491,7 @@ DEVELOPMENT_TEAM = 3HEZE76D99; GENERATE_INFOPLIST_FILE = YES; HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -524,7 +511,7 @@ DEVELOPMENT_TEAM = 3HEZE76D99; GENERATE_INFOPLIST_FILE = YES; HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests; PRODUCT_NAME = "$(TARGET_NAME)"; diff --git a/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme b/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme new file mode 100644 index 00000000000000..3c330a4152bf92 --- /dev/null +++ b/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme @@ -0,0 +1,97 @@ +<?xml version="1.0" encoding="UTF-8"?> +<Scheme + LastUpgradeVersion = "1640" + version = "1.7"> + <BuildAction + parallelizeBuildables = "YES" + buildImplicitDependencies = "YES" + buildArchitectures = "Automatic"> + <BuildActionEntries> + <BuildActionEntry + buildForTesting = "YES" + buildForRunning = "YES" + buildForProfiling = "YES" + buildForArchiving = "YES" + buildForAnalyzing = "YES"> + <BuildableReference + BuildableIdentifier = "primary" + BlueprintIdentifier = "607A66112B0EFA380010BFC8" + BuildableName = "iOSTestbed.app" + BlueprintName = "iOSTestbed" + ReferencedContainer = "container:iOSTestbed.xcodeproj"> + </BuildableReference> + </BuildActionEntry> + </BuildActionEntries> + </BuildAction> + <TestAction + buildConfiguration = "Debug" + selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" + selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" + customLLDBInitFile = "$(SOURCE_ROOT)/Testbed.lldbinit" + shouldUseLaunchSchemeArgsEnv = "YES"> + <TestPlans> + <TestPlanReference + reference = "container:iOSTestbed.xctestplan" + default = "YES"> + </TestPlanReference> + </TestPlans> + <Testables> + <TestableReference + skipped = "NO" + parallelizable = "NO"> + <BuildableReference + BuildableIdentifier = "primary" + BlueprintIdentifier = "607A662C2B0EFA3A0010BFC8" + BuildableName = "iOSTestbedTests.xctest" + BlueprintName = "iOSTestbedTests" + ReferencedContainer = "container:iOSTestbed.xcodeproj"> + </BuildableReference> + </TestableReference> + </Testables> + </TestAction> + <LaunchAction + buildConfiguration = "Debug" + selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" + selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" + launchStyle = "0" + useCustomWorkingDirectory = "NO" + ignoresPersistentStateOnLaunch = "NO" + debugDocumentVersioning = "YES" + debugServiceExtension = "internal" + allowLocationSimulation = "YES"> + <BuildableProductRunnable + runnableDebuggingMode = "0"> + <BuildableReference + BuildableIdentifier = "primary" + BlueprintIdentifier = "607A66112B0EFA380010BFC8" + BuildableName = "iOSTestbed.app" + BlueprintName = "iOSTestbed" + ReferencedContainer = "container:iOSTestbed.xcodeproj"> + </BuildableReference> + </BuildableProductRunnable> + </LaunchAction> + <ProfileAction + buildConfiguration = "Release" + shouldUseLaunchSchemeArgsEnv = "YES" + savedToolIdentifier = "" + useCustomWorkingDirectory = "NO" + debugDocumentVersioning = "YES"> + <BuildableProductRunnable + runnableDebuggingMode = "0"> + <BuildableReference + BuildableIdentifier = "primary" + BlueprintIdentifier = "607A66112B0EFA380010BFC8" + BuildableName = "iOSTestbed.app" + BlueprintName = "iOSTestbed" + ReferencedContainer = "container:iOSTestbed.xcodeproj"> + </BuildableReference> + </BuildableProductRunnable> + </ProfileAction> + <AnalyzeAction + buildConfiguration = "Debug"> + </AnalyzeAction> + <ArchiveAction + buildConfiguration = "Release" + revealArchiveInOrganizer = "YES"> + </ArchiveAction> +</Scheme> diff --git a/Apple/testbed/iOSTestbed.xctestplan b/Apple/testbed/iOSTestbed.xctestplan new file mode 100644 index 00000000000000..0c4ab9eb2bad30 --- /dev/null +++ b/Apple/testbed/iOSTestbed.xctestplan @@ -0,0 +1,46 @@ +{ + "configurations" : [ + { + "id" : "F5A95CE4-1ADE-4A6E-A0E1-CDBAE26DF0C5", + "name" : "Test Scheme Action", + "options" : { + + } + } + ], + "defaultOptions" : { + "commandLineArgumentEntries" : [ + { + "argument" : "test" + }, + { + "argument" : "-uall" + }, + { + "argument" : "--single-process" + }, + { + "argument" : "--rerun" + }, + { + "argument" : "-W" + } + ], + "targetForVariableExpansion" : { + "containerPath" : "container:iOSTestbed.xcodeproj", + "identifier" : "607A66112B0EFA380010BFC8", + "name" : "iOSTestbed" + } + }, + "testTargets" : [ + { + "parallelizable" : false, + "target" : { + "containerPath" : "container:iOSTestbed.xcodeproj", + "identifier" : "607A662C2B0EFA3A0010BFC8", + "name" : "iOSTestbedTests" + } + } + ], + "version" : 1 +} diff --git a/iOS/testbed/iOSTestbed/AppDelegate.h b/Apple/testbed/iOSTestbed/AppDelegate.h similarity index 100% rename from iOS/testbed/iOSTestbed/AppDelegate.h rename to Apple/testbed/iOSTestbed/AppDelegate.h diff --git a/iOS/testbed/iOSTestbed/AppDelegate.m b/Apple/testbed/iOSTestbed/AppDelegate.m similarity index 100% rename from iOS/testbed/iOSTestbed/AppDelegate.m rename to Apple/testbed/iOSTestbed/AppDelegate.m diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json b/Apple/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json similarity index 100% rename from iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json rename to Apple/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json b/Apple/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json similarity index 100% rename from iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json rename to Apple/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json b/Apple/testbed/iOSTestbed/Assets.xcassets/Contents.json similarity index 100% rename from iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json rename to Apple/testbed/iOSTestbed/Assets.xcassets/Contents.json diff --git a/iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard b/Apple/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard similarity index 100% rename from iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard rename to Apple/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard diff --git a/Apple/testbed/iOSTestbed/app/README b/Apple/testbed/iOSTestbed/app/README new file mode 100644 index 00000000000000..46c0e8e2a29a1c --- /dev/null +++ b/Apple/testbed/iOSTestbed/app/README @@ -0,0 +1,7 @@ +This folder can contain any Python application code. + +During the build, any binary modules found in this folder will be processed into +Framework form. + +When the test suite runs, this folder will be on the PYTHONPATH, and will be the +working directory for the test suite. diff --git a/Apple/testbed/iOSTestbed/app_packages/README b/Apple/testbed/iOSTestbed/app_packages/README new file mode 100644 index 00000000000000..02c2beccfbdaed --- /dev/null +++ b/Apple/testbed/iOSTestbed/app_packages/README @@ -0,0 +1,7 @@ +This folder can be a target for installing any Python dependencies needed by the +test suite. + +During the build, any binary modules found in this folder will be processed into +Framework form. + +When the test suite runs, this folder will be on the PYTHONPATH. diff --git a/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist b/Apple/testbed/iOSTestbed/iOSTestbed-Info.plist similarity index 97% rename from iOS/testbed/iOSTestbed/iOSTestbed-Info.plist rename to Apple/testbed/iOSTestbed/iOSTestbed-Info.plist index e2aa460b6fd5ee..fea45e1fad6f6f 100644 --- a/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist +++ b/Apple/testbed/iOSTestbed/iOSTestbed-Info.plist @@ -41,8 +41,6 @@ <string>UIInterfaceOrientationLandscapeLeft</string> <string>UIInterfaceOrientationLandscapeRight</string> </array> - <key>MainModule</key> - <string>ios</string> <key>UIApplicationSceneManifest</key> <dict> <key>UIApplicationSupportsMultipleScenes</key> diff --git a/iOS/testbed/iOSTestbed/main.m b/Apple/testbed/iOSTestbed/main.m similarity index 100% rename from iOS/testbed/iOSTestbed/main.m rename to Apple/testbed/iOSTestbed/main.m diff --git a/Doc/.ruff.toml b/Doc/.ruff.toml index 111ce03b91df38..3e676e13c3f41a 100644 --- a/Doc/.ruff.toml +++ b/Doc/.ruff.toml @@ -1,7 +1,6 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + target-version = "py312" # Align with the version in oldest_supported_sphinx -fix = true -output-format = "full" -line-length = 79 extend-exclude = [ "includes/*", # Temporary exclusions: diff --git a/Doc/Makefile b/Doc/Makefile index a090ee5ba92705..4d605980a62904 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -14,15 +14,15 @@ PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) REQUIREMENTS = requirements.txt -SPHINXERRORHANDLING = -W +SPHINXERRORHANDLING = --fail-on-warning # Internal variables. -PAPEROPT_a4 = -D latex_elements.papersize=a4paper -PAPEROPT_letter = -D latex_elements.papersize=letterpaper +PAPEROPT_a4 = --define latex_elements.papersize=a4paper +PAPEROPT_letter = --define latex_elements.papersize=letterpaper -ALLSPHINXOPTS = -b $(BUILDER) \ - -d build/doctrees \ - -j $(JOBS) \ +ALLSPHINXOPTS = --builder $(BUILDER) \ + --doctree-dir build/doctrees \ + --jobs $(JOBS) \ $(PAPEROPT_$(PAPER)) \ $(SPHINXOPTS) $(SPHINXERRORHANDLING) \ . build/$(BUILDER) $(SOURCES) @@ -140,11 +140,12 @@ doctest: pydoc-topics: BUILDER = pydoc-topics pydoc-topics: build @echo "Building finished; now run this:" \ - "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" + "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" \ + "&& cp build/pydoc-topics/module_docs.py ../Lib/pydoc_data/module_docs.py" .PHONY: gettext gettext: BUILDER = gettext -gettext: SPHINXOPTS += -d build/doctrees-gettext +gettext: override SPHINXOPTS := --doctree-dir build/doctrees-gettext $(SPHINXOPTS) gettext: build .PHONY: htmlview @@ -170,9 +171,10 @@ venv: echo "venv already exists."; \ echo "To recreate it, remove it first with \`make clean-venv'."; \ else \ + set -e; \ echo "Creating venv in $(VENVDIR)"; \ if $(UV) --version >/dev/null 2>&1; then \ - $(UV) venv $(VENVDIR); \ + $(UV) venv --python=$(PYTHON) $(VENVDIR); \ VIRTUAL_ENV=$(VENVDIR) $(UV) pip install -r $(REQUIREMENTS); \ else \ $(PYTHON) -m venv $(VENVDIR); \ @@ -183,7 +185,7 @@ venv: fi .PHONY: dist-no-html -dist-no-html: dist-text dist-pdf dist-epub dist-texinfo +dist-no-html: dist-text dist-epub dist-texinfo .PHONY: dist dist: @@ -204,6 +206,7 @@ dist-html: find dist -name 'python-$(DISTVERSION)-docs-html*' -exec rm -rf {} \; $(MAKE) html cp -pPR build/html dist/python-$(DISTVERSION)-docs-html + rm -rf dist/python-$(DISTVERSION)-docs-html/_images/social_previews/ tar -C dist -cf dist/python-$(DISTVERSION)-docs-html.tar python-$(DISTVERSION)-docs-html bzip2 -9 -k dist/python-$(DISTVERSION)-docs-html.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-html.zip python-$(DISTVERSION)-docs-html) @@ -239,7 +242,8 @@ dist-pdf: # as otherwise the full latexmk process is run twice. # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile - (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) + if [ -n "$(filter output-sync,$(value .FEATURES))" ]; then OUTPUTSYNC=--output-sync; else OUTPUTSYNC=; fi && \ + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`getconf _NPROCESSORS_ONLN`+1)) $$OUTPUTSYNC LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 @echo "Build finished and archived!" @@ -294,26 +298,26 @@ check: _ensure-pre-commit .PHONY: serve serve: - @echo "The serve target was removed, use htmlview instead (see bpo-36329)" + @echo "The serve target was removed, use htmllive instead (see gh-80510)" # Targets for daily automated doc build # By default, Sphinx only rebuilds pages where the page content has changed. # This means it doesn't always pick up changes to preferred link targets, etc # To ensure such changes are picked up, we build the published docs with -# `-E` (to ignore the cached environment) and `-a` (to ignore already existing -# output files) +# ``--fresh-env`` (to ignore the cached environment) and ``--write-all`` +# (to ignore already existing output files) # for development releases: always build .PHONY: autobuild-dev autobuild-dev: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) autobuild-dev: - $(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION) + $(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) --fresh-env --write-all --html-define daily=1' DISTVERSION=$(DISTVERSION) # for HTML-only rebuilds .PHONY: autobuild-dev-html autobuild-dev-html: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) autobuild-dev-html: - $(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION) + $(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) --fresh-env --write-all --html-define daily=1' DISTVERSION=$(DISTVERSION) # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage diff --git a/Doc/README.rst b/Doc/README.rst index efcee0db428908..2d1148753e0c6b 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -133,8 +133,5 @@ Bugs in the content should be reported to the Bugs in the toolset should be reported to the tools themselves. -You can also send a mail to the Python Documentation Team at docs@python.org, -and we will process your request as soon as possible. - -If you want to help the Documentation Team, you are always welcome. Just send -a mail to docs@python.org. +To help with the documentation, or report any problems, please leave a message +on `discuss.python.org <https://discuss.python.org/c/documentation>`_. diff --git a/Doc/_static/tachyon-example-flamegraph.html b/Doc/_static/tachyon-example-flamegraph.html new file mode 100644 index 00000000000000..701c959bda3ccc --- /dev/null +++ b/Doc/_static/tachyon-example-flamegraph.html @@ -0,0 +1,3260 @@ +<!doctype html> +<html lang="en" data-theme="light"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <title>Tachyon Profiler - Flamegraph Report + + + + + + + +
+ +
+
+ + Tachyon + + Flamegraph Report +
+
+ + +
+
+ + + + + + + + +
+
+ + +
+ + + + +
+
+
+
+ + +
+ + Tachyon Profiler + + + Python Sampling Profiler + + + + + +
+
+ + + + diff --git a/Doc/_static/tachyon-example-heatmap.html b/Doc/_static/tachyon-example-heatmap.html new file mode 100644 index 00000000000000..f725e9475009fe --- /dev/null +++ b/Doc/_static/tachyon-example-heatmap.html @@ -0,0 +1,3804 @@ + + + + + + /tmp/tachyon_selfcontained.py - Heatmap + + + +
+ +
+
+ + Tachyon + + /tmp/tachyon_selfcontained.py +
+
+ + + + + + + + + + + +
+
+ + +
+
+
+
1,054
+
Self Samples
+
+
+
4,236
+
Cumulative
+
+
+
24
+
Lines Hit
+
+
+
100.00%
+
% of Total
+
+
+
206
+
Max Self
+
+
+
1054
+
Max Total
+
+
+
+ + +
+
+ Intensity: +
+
+ Cold + + Hot +
+ +
+
+ Self Time +
+ Total Time +
+
+ Show All +
+ Hot Only +
+
+ Heat +
+ Specialization +
+ + +
+
+
+ + +
+
+
Line
+
Self
+
Total
+
Code
+
+
+
1
+
+
+
+
"""
+ +
+
+
2
+
+
+
+
Tachyon Demo - Self-contained profiling example.
+ +
+
+
3
+
+
+
+
Pure Python with no external imports for clean heatmap output.
+ +
+
+
4
+
+
+
+
"""
+ +
+
+
5
+
+
+
+
+ +
+
+
6
+
+
+
+
+ +
+
+
7
+
1
+
1
+ +
def fibonacci(n):
+ +
+ +
+
8
+
+
+
+
"""Recursive fibonacci - creates deep call stacks."""
+ +
+
+
9
+
3
+
3
+ +
if n <= 1:
+ +
+ +
+
10
+
17
+
17
+ +
return n
+
+
+ +
+
11
+
206
+
227
+ +
return fibonacci(n - 1) + fibonacci(n - 2)
+
+
+ +
+
12
+
+
+
+
+ +
+
+
13
+
+
+
+
+ +
+
+
14
+
+
+
+
def bubble_sort(arr):
+ +
+
+
15
+
+
+
+
"""Classic bubble sort - O(n^2) comparison sorting."""
+ +
+
+
16
+
+
+
+
n = len(arr)
+ +
+
+
17
+
+
+
+
for i in range(n):
+ +
+
+
18
+
2
+
2
+ +
for j in range(0, n - i - 1):
+ +
+ +
+
19
+
8
+
8
+ +
if arr[j] > arr[j + 1]:
+
+
+ +
+
20
+
2
+
2
+ +
arr[j], arr[j + 1] = arr[j + 1], arr[j]
+ +
+ +
+
21
+
+
+
+
return arr
+ +
+
+
22
+
+
+
+
+ +
+
+
23
+
+
+
+
+ +
+
+
24
+
+
+
+
def matrix_multiply(a, b):
+ +
+
+
25
+
+
+
+
"""Matrix multiplication using nested loops."""
+ +
+
+
26
+
+
+
+
rows_a, cols_a = len(a), len(a[0])
+ +
+
+
27
+
+
+
+
rows_b, cols_b = len(b), len(b[0])
+ +
+
+
28
+
+
+
+
result = [[0] * cols_b for _ in range(rows_a)]
+ +
+
+
29
+
+
+
+
+ +
+
+
30
+
+
+
+
for i in range(rows_a):
+ +
+
+
31
+
+
+
+
for j in range(cols_b):
+ +
+
+
32
+
8
+
8
+ +
for k in range(cols_a):
+ +
+ +
+
33
+
62
+
62
+ +
result[i][j] += a[i][k] * b[k][j]
+
+
+ +
+
34
+
+
+
+
return result
+ +
+
+
35
+
+
+
+
+ +
+
+
36
+
+
+
+
+ +
+
+
37
+
+
+
+
def prime_sieve(limit):
+ +
+
+
38
+
+
+
+
"""Sieve of Eratosthenes - find all primes up to limit."""
+ +
+
+
39
+
+
+
+
is_prime = [True] * (limit + 1)
+ +
+
+
40
+
+
+
+
is_prime[0] = is_prime[1] = False
+ +
+
+
41
+
+
+
+
+ +
+
+
42
+
+
+
+
for num in range(2, int(limit ** 0.5) + 1):
+ +
+
+
43
+
+
+
+
if is_prime[num]:
+ +
+
+
44
+
1
+
1
+ +
for multiple in range(num * num, limit + 1, num):
+
+
+ +
+
45
+
+
+
+
is_prime[multiple] = False
+ +
+
+
46
+
+
+
+
+ +
+
+
47
+
2
+
2
+ +
return [num for num, prime in enumerate(is_prime) if prime]
+ +
+ +
+
48
+
+
+
+
+ +
+
+
49
+
+
+
+
+ +
+
+
50
+
+
+
+
def string_processing(iterations):
+ +
+
+
51
+
+
+
+
"""String operations - concatenation and formatting."""
+ +
+
+
52
+
+
+
+
for _ in range(iterations):
+ +
+
+
53
+
+
+
+
result = ""
+ +
+
+
54
+
+
+
+
for i in range(50):
+ +
+
+
55
+
91
+
91
+ +
result += f"item_{i}_"
+
+
+ +
+
56
+
22
+
22
+ +
parts = result.split("_")
+ +
+ +
+
57
+
10
+
10
+ +
joined = "-".join(parts)
+ +
+ +
+
58
+
+
+
+
+ +
+
+
59
+
+
+
+
+ +
+
+
60
+
+
+
+
def list_operations(iterations):
+ +
+
+
61
+
+
+
+
"""List comprehensions and operations."""
+ +
+
+
62
+
+
+
+
for _ in range(iterations):
+ +
+
+
63
+
118
+
118
+ +
squares = [x ** 2 for x in range(500)]
+ +
+ +
+
64
+
119
+
119
+ +
evens = [x for x in squares if x % 2 == 0]
+
+
+ +
+
65
+
12
+
12
+ +
total = sum(evens)
+ +
+ +
+
66
+
+
+
+
+ +
+
+
67
+
+
+
+
+ +
+
+
68
+
+
+
+
def dict_operations(iterations):
+ +
+
+
69
+
+
+
+
"""Dictionary creation and lookups."""
+ +
+
+
70
+
+
+
+
for _ in range(iterations):
+ +
+
+
71
+
206
+
206
+ +
data = {f"key_{i}": i ** 2 for i in range(200)}
+
+
+ +
+
72
+
158
+
158
+ +
values = [data[f"key_{i}"] for i in range(200)]
+ +
+ +
+
73
+
5
+
5
+ +
total = sum(values)
+ +
+ +
+
74
+
+
+
+
+ +
+
+
75
+
+
+
+
+ +
+
+
76
+
+
+
+
def compute_heavy():
+ +
+
+
77
+
+
+
+
"""CPU-intensive computation section."""
+ +
+
+
78
+
1
+
301
+ +
fibonacci(30)
+
+
+ +
+
79
+
+
+
+
+ +
+
+
80
+
+
+
+
size = 60
+ +
+
+
81
+
+
+
+
a = [[i + j for j in range(size)] for i in range(size)]
+ +
+
+
82
+
+
+
+
b = [[i * j for j in range(size)] for i in range(size)]
+ +
+
+
83
+
+
+
+
matrix_multiply(a, b)
+ +
+
+
84
+
+
+
+
+ +
+
+
85
+
+
+
+
prime_sieve(10000)
+ +
+
+
86
+
+
+
+
+ +
+
+
87
+
+
+
+
+ +
+
+
88
+
+
+
+
def data_processing():
+ +
+
+
89
+
+
+
+
"""Data structure operations."""
+ +
+
+
90
+
+
753
+ +
string_processing(2000)
+
+
+ +
+
91
+
+
+
+
list_operations(1500)
+ +
+
+
92
+
+
+
+
dict_operations(1000)
+ +
+
+
93
+
+
+
+
+ +
+
+
94
+
+
+
+
data = list(range(800))
+ +
+
+
95
+
+
+
+
for _ in range(3):
+ +
+
+
96
+
+
+
+
data = data[::-1]
+ +
+
+
97
+
+
+
+
bubble_sort(data[:200])
+ +
+
+
98
+
+
+
+
+ +
+
+
99
+
+
+
+
+ +
+
+
100
+
+
+
+
def main():
+ +
+
+
101
+
+
+
+
"""Main entry point."""
+ +
+
+
102
+
+
1,054
+ +
compute_heavy()
+
+
+ +
+
103
+
+
+
+
data_processing()
+ +
+
+
104
+
+
+
+
+ +
+
+
105
+
+
+
+
+ +
+
+
106
+
+
+
+
if __name__ == "__main__":
+ +
+
+
107
+
+
1,054
+ +
main()
+
+
+ + +
+
+ + + + diff --git a/Doc/about.rst b/Doc/about.rst index 5e6160ff2700ed..5c1b497ca6bcea 100644 --- a/Doc/about.rst +++ b/Doc/about.rst @@ -1,10 +1,11 @@ -===================== -About these documents -===================== +======================== +About this documentation +======================== -These documents are generated from `reStructuredText`_ sources by `Sphinx`_, a -document processor specifically written for the Python documentation. +Python's documentation is generated from `reStructuredText`_ sources +using `Sphinx`_, a documentation generator originally created for Python +and now maintained as an independent project. .. _reStructuredText: https://docutils.sourceforge.io/rst.html .. _Sphinx: https://www.sphinx-doc.org/ @@ -20,19 +21,20 @@ volunteers are always welcome! Many thanks go to: * Fred L. Drake, Jr., the creator of the original Python documentation toolset - and writer of much of the content; + and author of much of the content; * the `Docutils `_ project for creating reStructuredText and the Docutils suite; * Fredrik Lundh for his Alternative Python Reference project from which Sphinx got many good ideas. -Contributors to the Python Documentation +Contributors to the Python documentation ---------------------------------------- Many people have contributed to the Python language, the Python standard -library, and the Python documentation. See :source:`Misc/ACKS` in the Python -source distribution for a partial list of contributors. +library, and the Python documentation. See the `CPython +GitHub repository `__ +for a partial list of contributors. It is only with the input and contributions of the Python community that Python has such wonderful documentation -- Thank You! diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 5d0f68ca69675e..0683eebbaf677b 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -19,6 +19,12 @@ If you find a bug in this documentation or would like to propose an improvement, please submit a bug report on the :ref:`issue tracker `. If you have a suggestion on how to fix it, include that as well. +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, submit the report to the + `translation’s repository `_ instead. + You can also open a discussion item on our `Documentation Discourse forum `_. @@ -37,8 +43,8 @@ tracker `_. `Helping with Documentation `_ Comprehensive guide for individuals that are interested in contributing to Python documentation. - `Documentation Translations `_ - A list of GitHub pages for documentation translation and their primary contacts. + `Documentation Translations `_ + A list of GitHub pages for documentation translation and their coordination teams. .. _using-the-tracker: diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 0d53b18ea87d5e..59044d2d88cc16 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -15,10 +15,21 @@ Allocating Objects on the Heap .. c:function:: PyObject* PyObject_Init(PyObject *op, PyTypeObject *type) Initialize a newly allocated object *op* with its type and initial - reference. Returns the initialized object. If *type* indicates that the - object participates in the cyclic garbage detector, it is added to the - detector's set of observed objects. Other fields of the object are not - affected. + reference. Returns the initialized object. Other fields of the object are + not initialized. Despite its name, this function is unrelated to the + object's :meth:`~object.__init__` method (:c:member:`~PyTypeObject.tp_init` + slot). Specifically, this function does **not** call the object's + :meth:`!__init__` method. + + In general, consider this function to be a low-level routine. Use + :c:member:`~PyTypeObject.tp_alloc` where possible. + For implementing :c:member:`!tp_alloc` for your type, prefer + :c:func:`PyType_GenericAlloc` or :c:func:`PyObject_New`. + + .. note:: + + This function only initializes the object's memory corresponding to the + initial :c:type:`PyObject` structure. It does not zero the rest. .. c:function:: PyVarObject* PyObject_InitVar(PyVarObject *op, PyTypeObject *type, Py_ssize_t size) @@ -26,35 +37,108 @@ Allocating Objects on the Heap This does everything :c:func:`PyObject_Init` does, and also initializes the length information for a variable-size object. + .. note:: + + This function only initializes some of the object's memory. It does not + zero the rest. + .. c:macro:: PyObject_New(TYPE, typeobj) - Allocate a new Python object using the C structure type *TYPE* - and the Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header are not initialized. - The caller will own the only reference to the object - (i.e. its reference count will be one). - The size of the memory allocation is determined from the - :c:member:`~PyTypeObject.tp_basicsize` field of the type object. + Allocates a new Python object using the C structure type *TYPE* and the + Python type object *typeobj* (``PyTypeObject*``) by calling + :c:func:`PyObject_Malloc` to allocate memory and initializing it like + :c:func:`PyObject_Init`. The caller will own the only reference to the + object (i.e. its reference count will be one). + + Avoid calling this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + This macro does not call :c:member:`~PyTypeObject.tp_alloc`, + :c:member:`~PyTypeObject.tp_new` (:meth:`~object.__new__`), or + :c:member:`~PyTypeObject.tp_init` (:meth:`~object.__init__`). + + This cannot be used for objects with :c:macro:`Py_TPFLAGS_HAVE_GC` set in + :c:member:`~PyTypeObject.tp_flags`; use :c:macro:`PyObject_GC_New` instead. + + Memory allocated by this macro must be freed with :c:func:`PyObject_Free` + (usually called via the object's :c:member:`~PyTypeObject.tp_free` slot). + + .. note:: + + The returned memory is not guaranteed to have been completely zeroed + before it was initialized. + + .. note:: + + This macro does not construct a fully initialized object of the given + type; it merely allocates memory and prepares it for further + initialization by :c:member:`~PyTypeObject.tp_init`. To construct a + fully initialized object, call *typeobj* instead. For example:: + + PyObject *foo = PyObject_CallNoArgs((PyObject *)&PyFoo_Type); + + .. seealso:: + + * :c:func:`PyObject_Free` + * :c:macro:`PyObject_GC_New` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` .. c:macro:: PyObject_NewVar(TYPE, typeobj, size) - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header - are not initialized. The allocated memory allows for the *TYPE* structure - plus *size* (``Py_ssize_t``) fields of the size - given by the :c:member:`~PyTypeObject.tp_itemsize` field of - *typeobj*. This is useful for implementing objects like tuples, which are - able to determine their size at construction time. Embedding the array of - fields into the same allocation decreases the number of allocations, - improving the memory management efficiency. + Like :c:macro:`PyObject_New` except: + + * It allocates enough memory for the *TYPE* structure plus *size* + (``Py_ssize_t``) fields of the size given by the + :c:member:`~PyTypeObject.tp_itemsize` field of *typeobj*. + * The memory is initialized like :c:func:`PyObject_InitVar`. + + This is useful for implementing objects like tuples, which are able to + determine their size at construction time. Embedding the array of fields + into the same allocation decreases the number of allocations, improving the + memory management efficiency. + + Avoid calling this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + This cannot be used for objects with :c:macro:`Py_TPFLAGS_HAVE_GC` set in + :c:member:`~PyTypeObject.tp_flags`; use :c:macro:`PyObject_GC_NewVar` + instead. + Memory allocated by this function must be freed with :c:func:`PyObject_Free` + (usually called via the object's :c:member:`~PyTypeObject.tp_free` slot). -.. c:function:: void PyObject_Del(void *op) + .. note:: + + The returned memory is not guaranteed to have been completely zeroed + before it was initialized. + + .. note:: + + This macro does not construct a fully initialized object of the given + type; it merely allocates memory and prepares it for further + initialization by :c:member:`~PyTypeObject.tp_init`. To construct a + fully initialized object, call *typeobj* instead. For example:: + + PyObject *list_instance = PyObject_CallNoArgs((PyObject *)&PyList_Type); + + .. seealso:: + + * :c:func:`PyObject_Free` + * :c:macro:`PyObject_GC_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` - Same as :c:func:`PyObject_Free`. .. c:var:: PyObject _Py_NoneStruct @@ -65,6 +149,38 @@ Allocating Objects on the Heap .. seealso:: - :c:func:`PyModule_Create` + :ref:`moduleobjects` To allocate and create extension modules. + +Deprecated aliases +^^^^^^^^^^^^^^^^^^ + +These are :term:`soft deprecated` aliases to existing functions and macros. +They exist solely for backwards compatibility. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Deprecated alias + * Function + * * .. c:macro:: PyObject_NEW(type, typeobj) + * :c:macro:`PyObject_New` + * * .. c:macro:: PyObject_NEW_VAR(type, typeobj, n) + * :c:macro:`PyObject_NewVar` + * * .. c:macro:: PyObject_INIT(op, typeobj) + * :c:func:`PyObject_Init` + * * .. c:macro:: PyObject_INIT_VAR(op, typeobj, n) + * :c:func:`PyObject_InitVar` + * * .. c:macro:: PyObject_MALLOC(n) + * :c:func:`PyObject_Malloc` + * * .. c:macro:: PyObject_REALLOC(p, n) + * :c:func:`PyObject_Realloc` + * * .. c:macro:: PyObject_FREE(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_DEL(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_Del(p) + * :c:func:`PyObject_Free` diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index f6c8284daeacb0..cb98d4307ee522 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -6,9 +6,13 @@ API and ABI Versioning *********************** + +Build-time version constants +---------------------------- + CPython exposes its version number in the following macros. -Note that these correspond to the version code is **built** with, -not necessarily the version used at **run time**. +Note that these correspond to the version code is **built** with. +See :c:var:`Py_Version` for the version used at **run time**. See :ref:`stable` for a discussion of API and ABI stability across versions. @@ -30,6 +34,23 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. This can be ``0xA`` for alpha, ``0xB`` for beta, ``0xC`` for release candidate or ``0xF`` for final. + + .. c:namespace:: NULL + .. c:macro:: PY_RELEASE_LEVEL_ALPHA + :no-typesetting: + .. c:macro:: PY_RELEASE_LEVEL_BETA + :no-typesetting: + .. c:macro:: PY_RELEASE_LEVEL_GAMMA + :no-typesetting: + .. c:macro:: PY_RELEASE_LEVEL_FINAL + :no-typesetting: + + For completeness, the values are available as macros: + :c:macro:`!PY_RELEASE_LEVEL_ALPHA` (``0xA``), + :c:macro:`!PY_RELEASE_LEVEL_BETA` (``0xB``), + :c:macro:`!PY_RELEASE_LEVEL_GAMMA` (``0xC``), and + :c:macro:`!PY_RELEASE_LEVEL_FINAL` (``0xF``). + .. c:macro:: PY_RELEASE_SERIAL The ``2`` in ``3.4.1a2``. Zero for final releases. @@ -37,37 +58,89 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. .. c:macro:: PY_VERSION_HEX The Python version number encoded in a single integer. + See :c:func:`Py_PACK_FULL_VERSION` for the encoding details. + + Use this for numeric comparisons, for example, + ``#if PY_VERSION_HEX >= ...``. - The underlying version information can be found by treating it as a 32 bit - number in the following manner: +.. c:macro:: PY_VERSION - +-------+-------------------------+-------------------------+--------------------------+ - | Bytes | Bits (big endian order) | Meaning | Value for ``3.4.1a2`` | - +=======+=========================+=========================+==========================+ - | 1 | 1-8 | ``PY_MAJOR_VERSION`` | ``0x03`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 2 | 9-16 | ``PY_MINOR_VERSION`` | ``0x04`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 3 | 17-24 | ``PY_MICRO_VERSION`` | ``0x01`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 4 | 25-28 | ``PY_RELEASE_LEVEL`` | ``0xA`` | - + +-------------------------+-------------------------+--------------------------+ - | | 29-32 | ``PY_RELEASE_SERIAL`` | ``0x2`` | - +-------+-------------------------+-------------------------+--------------------------+ + The Python version as a string, for example, ``"3.4.1a2"``. - Thus ``3.4.1a2`` is hexversion ``0x030401a2`` and ``3.10.0`` is - hexversion ``0x030a00f0``. +These macros are defined in :source:`Include/patchlevel.h`. - Use this for numeric comparisons, e.g. ``#if PY_VERSION_HEX >= ...``. - This version is also available via the symbol :c:var:`Py_Version`. +Run-time version +---------------- .. c:var:: const unsigned long Py_Version - The Python runtime version number encoded in a single constant integer, with - the same format as the :c:macro:`PY_VERSION_HEX` macro. + The Python runtime version number encoded in a single constant integer. + See :c:func:`Py_PACK_FULL_VERSION` for the encoding details. This contains the Python version used at run time. + Use this for numeric comparisons, for example, ``if (Py_Version >= ...)``. + .. versionadded:: 3.11 -All the given macros are defined in :source:`Include/patchlevel.h`. + +Bit-packing macros +------------------ + +.. c:function:: uint32_t Py_PACK_FULL_VERSION(int major, int minor, int micro, int release_level, int release_serial) + + Return the given version, encoded as a single 32-bit integer with + the following structure: + + +------------------+-------+----------------+-----------+--------------------------+ + | | No. | | | Example values | + | | of | | +-------------+------------+ + | Argument | bits | Bit mask | Bit shift | ``3.4.1a2`` | ``3.10.0`` | + +==================+=======+================+===========+=============+============+ + | *major* | 8 | ``0xFF000000`` | 24 | ``0x03`` | ``0x03`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *minor* | 8 | ``0x00FF0000`` | 16 | ``0x04`` | ``0x0A`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *micro* | 8 | ``0x0000FF00`` | 8 | ``0x01`` | ``0x00`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *release_level* | 4 | ``0x000000F0`` | 4 | ``0xA`` | ``0xF`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *release_serial* | 4 | ``0x0000000F`` | 0 | ``0x2`` | ``0x0`` | + +------------------+-------+----------------+-----------+-------------+------------+ + + For example: + + +-------------+------------------------------------+-----------------+ + | Version | ``Py_PACK_FULL_VERSION`` arguments | Encoded version | + +=============+====================================+=================+ + | ``3.4.1a2`` | ``(3, 4, 1, 0xA, 2)`` | ``0x030401a2`` | + +-------------+------------------------------------+-----------------+ + | ``3.10.0`` | ``(3, 10, 0, 0xF, 0)`` | ``0x030a00f0`` | + +-------------+------------------------------------+-----------------+ + + Out-of range bits in the arguments are ignored. + That is, the macro can be defined as: + + .. code-block:: c + + #ifndef Py_PACK_FULL_VERSION + #define Py_PACK_FULL_VERSION(X, Y, Z, LEVEL, SERIAL) ( \ + (((X) & 0xff) << 24) | \ + (((Y) & 0xff) << 16) | \ + (((Z) & 0xff) << 8) | \ + (((LEVEL) & 0xf) << 4) | \ + (((SERIAL) & 0xf) << 0)) + #endif + + ``Py_PACK_FULL_VERSION`` is primarily a macro, intended for use in + ``#if`` directives, but it is also available as an exported function. + + .. versionadded:: 3.14 + +.. c:function:: uint32_t Py_PACK_VERSION(int major, int minor) + + Equivalent to ``Py_PACK_FULL_VERSION(major, minor, 0, 0, 0)``. + The result does not correspond to any Python release, but is useful + in numeric comparisons. + + .. versionadded:: 3.14 diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 3201bdc82691f4..fd6be6a9b67a03 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -5,7 +5,7 @@ Parsing arguments and building values ===================================== -These functions are useful when creating your own extensions functions and +These functions are useful when creating your own extension functions and methods. Additional information and examples are available in :ref:`extending-index`. @@ -160,7 +160,7 @@ There are three ways strings and buffers can be converted to C: ``w*`` (read-write :term:`bytes-like object`) [Py_buffer] This format accepts any object which implements the read-write buffer interface. It fills a :c:type:`Py_buffer` structure provided by the caller. - The buffer may contain embedded null bytes. The caller have to call + The buffer may contain embedded null bytes. The caller has to call :c:func:`PyBuffer_Release` when it is done with the buffer. ``es`` (:class:`str`) [const char \*encoding, char \*\*buffer] @@ -229,41 +229,58 @@ There are three ways strings and buffers can be converted to C: Numbers ------- +These formats allow representing Python numbers or single characters as C numbers. +Formats that require :class:`int`, :class:`float` or :class:`complex` can +also use the corresponding special methods :meth:`~object.__index__`, +:meth:`~object.__float__` or :meth:`~object.__complex__` to convert +the Python object to the required type. + +For signed integer formats, :exc:`OverflowError` is raised if the value +is out of range for the C type. +For unsigned integer formats, the +most significant bits are silently truncated when the receiving field is too +small to receive the value, and :exc:`DeprecationWarning` is emitted when +the value is larger than the maximal value for the C type or less than +the minimal value for the corresponding signed integer type of the same size. + ``b`` (:class:`int`) [unsigned char] - Convert a nonnegative Python integer to an unsigned tiny int, stored in a C + Convert a nonnegative Python integer to an unsigned tiny integer, stored in a C :c:expr:`unsigned char`. ``B`` (:class:`int`) [unsigned char] - Convert a Python integer to a tiny int without overflow checking, stored in a C + Convert a Python integer to a tiny integer without overflow checking, stored in a C :c:expr:`unsigned char`. + Convert a Python integer to a C :c:expr:`unsigned char`. ``h`` (:class:`int`) [short int] Convert a Python integer to a C :c:expr:`short int`. ``H`` (:class:`int`) [unsigned short int] - Convert a Python integer to a C :c:expr:`unsigned short int`, without overflow - checking. + Convert a Python integer to a C :c:expr:`unsigned short int`. ``i`` (:class:`int`) [int] Convert a Python integer to a plain C :c:expr:`int`. ``I`` (:class:`int`) [unsigned int] - Convert a Python integer to a C :c:expr:`unsigned int`, without overflow - checking. + Convert a Python integer to a C :c:expr:`unsigned int`. ``l`` (:class:`int`) [long int] Convert a Python integer to a C :c:expr:`long int`. ``k`` (:class:`int`) [unsigned long] - Convert a Python integer to a C :c:expr:`unsigned long` without - overflow checking. + Convert a Python integer to a C :c:expr:`unsigned long`. + + .. versionchanged:: 3.14 + Use :meth:`~object.__index__` if available. ``L`` (:class:`int`) [long long] Convert a Python integer to a C :c:expr:`long long`. ``K`` (:class:`int`) [unsigned long long] - Convert a Python integer to a C :c:expr:`unsigned long long` - without overflow checking. + Convert a Python integer to a C :c:expr:`unsigned long long`. + + .. versionchanged:: 3.14 + Use :meth:`~object.__index__` if available. ``n`` (:class:`int`) [:c:type:`Py_ssize_t`] Convert a Python integer to a C :c:type:`Py_ssize_t`. @@ -288,6 +305,14 @@ Numbers ``D`` (:class:`complex`) [Py_complex] Convert a Python complex number to a C :c:type:`Py_complex` structure. +.. deprecated:: 3.15 + + For unsigned integer formats ``B``, ``H``, ``I``, ``k`` and ``K``, + :exc:`DeprecationWarning` is emitted when the value is larger than + the maximal value for the C type or less than the minimal value for + the corresponding signed integer type of the same size. + + Other objects ------------- @@ -307,7 +332,7 @@ Other objects .. _o_ampersand: -``O&`` (object) [*converter*, *anything*] +``O&`` (object) [*converter*, *address*] Convert a Python object to a C variable through a *converter* function. This takes two arguments: the first is a function, the second is the address of a C variable (of arbitrary type), converted to :c:expr:`void *`. The *converter* @@ -321,14 +346,20 @@ Other objects the conversion has failed. When the conversion fails, the *converter* function should raise an exception and leave the content of *address* unmodified. - If the *converter* returns ``Py_CLEANUP_SUPPORTED``, it may get called a + .. c:macro:: Py_CLEANUP_SUPPORTED + :no-typesetting: + + If the *converter* returns :c:macro:`!Py_CLEANUP_SUPPORTED`, it may get called a second time if the argument parsing eventually fails, giving the converter a chance to release any memory that it had already allocated. In this second call, the *object* parameter will be ``NULL``; *address* will have the same value as in the original call. + Examples of converters: :c:func:`PyUnicode_FSConverter` and + :c:func:`PyUnicode_FSDecoder`. + .. versionchanged:: 3.1 - ``Py_CLEANUP_SUPPORTED`` was added. + :c:macro:`!Py_CLEANUP_SUPPORTED` was added. ``p`` (:class:`bool`) [int] Tests the value passed in for truth (a boolean **p**\ redicate) and converts @@ -339,16 +370,25 @@ Other objects .. versionadded:: 3.3 -``(items)`` (:class:`tuple`) [*matching-items*] - The object must be a Python sequence whose length is the number of format units +``(items)`` (sequence) [*matching-items*] + The object must be a Python sequence (except :class:`str`, :class:`bytes` + or :class:`bytearray`) whose length is the number of format units in *items*. The C arguments must correspond to the individual format units in *items*. Format units for sequences may be nested. -It is possible to pass "long" integers (integers whose value exceeds the -platform's :c:macro:`LONG_MAX`) however no proper range checking is done --- the -most significant bits are silently truncated when the receiving field is too -small to receive the value (actually, the semantics are inherited from downcasts -in C --- your mileage may vary). + If *items* contains format units which store a :ref:`borrowed buffer + ` (``s``, ``s#``, ``z``, ``z#``, ``y``, or ``y#``) + or a :term:`borrowed reference` (``S``, ``Y``, ``U``, ``O``, or ``O!``), + the object must be a Python tuple. + The *converter* for the ``O&`` format unit in *items* must not store + a borrowed buffer or a borrowed reference. + + .. versionchanged:: 3.14 + :class:`str` and :class:`bytearray` objects no longer accepted as a sequence. + + .. deprecated:: 3.14 + Non-tuple sequences are deprecated if *items* contains format units + which store a borrowed buffer or a borrowed reference. A few other characters have a meaning in a format string. These may not occur inside nested parentheses. They are: @@ -627,12 +667,25 @@ Building values ``L`` (:class:`int`) [long long] Convert a C :c:expr:`long long` to a Python integer object. + .. _capi-py-buildvalue-format-K: + ``K`` (:class:`int`) [unsigned long long] Convert a C :c:expr:`unsigned long long` to a Python integer object. ``n`` (:class:`int`) [:c:type:`Py_ssize_t`] Convert a C :c:type:`Py_ssize_t` to a Python integer. + ``p`` (:class:`bool`) [int] + Convert a C :c:expr:`int` to a Python :class:`bool` object. + + Be aware that this format requires an ``int`` argument. + Unlike most other contexts in C, variadic arguments are not coerced to + a suitable type automatically. + You can convert another type (for example, a pointer or a float) to a + suitable ``int`` value using ``(x) ? 1 : 0`` or ``!!x``. + + .. versionadded:: 3.14 + ``c`` (:class:`bytes` of length 1) [char] Convert a C :c:expr:`int` representing a byte to a Python :class:`bytes` object of length 1. diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index dc43a3d5fcb094..6bb72a2312be3b 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -26,17 +26,19 @@ characteristic of being backed by a possibly large memory buffer. It is then desirable, in some situations, to access that buffer directly and without intermediate copying. -Python provides such a facility at the C level in the form of the :ref:`buffer -protocol `. This protocol has two sides: +Python provides such a facility at the C and Python level in the form of the +:ref:`buffer protocol `. This protocol has two sides: .. index:: single: PyBufferProcs (C type) - on the producer side, a type can export a "buffer interface" which allows objects of that type to expose information about their underlying buffer. - This interface is described in the section :ref:`buffer-structs`; + This interface is described in the section :ref:`buffer-structs`; for + Python see :ref:`python-buffer-protocol`. - on the consumer side, several means are available to obtain a pointer to - the raw underlying data of an object (for example a method parameter). + the raw underlying data of an object (for example a method parameter). For + Python see :class:`memoryview`. Simple objects such as :class:`bytes` and :class:`bytearray` expose their underlying buffer in byte-oriented form. Other forms are possible; for example, @@ -62,6 +64,10 @@ In both cases, :c:func:`PyBuffer_Release` must be called when the buffer isn't needed anymore. Failure to do so could lead to various issues such as resource leaks. +.. versionadded:: 3.12 + + The buffer protocol is now accessible in Python, see + :ref:`python-buffer-protocol` and :class:`memoryview`. .. _buffer-structure: @@ -255,6 +261,10 @@ readonly, format MUST be consistent for all consumers. For example, :c:expr:`PyBUF_SIMPLE | PyBUF_WRITABLE` can be used to request a simple writable buffer. + .. c:macro:: PyBUF_WRITEABLE + + This is a :term:`soft deprecated` alias to :c:macro:`PyBUF_WRITABLE`. + .. c:macro:: PyBUF_FORMAT Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 9045689a6be567..e2b22ec3c794ae 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -74,6 +74,11 @@ Direct API functions .. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) Resize the internal buffer of *bytearray* to *len*. + Failure is a ``-1`` return with an exception set. + + .. versionchanged:: 3.14 + A negative *len* will now result in an exception being set and -1 returned. + Macros ^^^^^^ diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d47beee68eaa33..82c2557368371f 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -47,6 +47,10 @@ called with a non-bytes parameter. *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of the bytes object are uninitialized. + .. deprecated:: 3.15 + ``PyBytes_FromStringAndSize(NULL, len)`` is :term:`soft deprecated`, + use the :c:type:`PyBytesWriter` API instead. + .. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) @@ -219,3 +223,209 @@ called with a non-bytes parameter. reallocation fails, the original bytes object at *\*bytes* is deallocated, *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. + + .. deprecated:: 3.15 + The function is :term:`soft deprecated`, + use the :c:type:`PyBytesWriter` API instead. + + +.. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) + + Get the string representation of *bytes*. This function is currently used to + implement :meth:`!bytes.__repr__` in Python. + + This function does not do type checking; it is undefined behavior to pass + *bytes* as a non-bytes object or ``NULL``. + + If *smartquotes* is true, the representation will use a double-quoted string + instead of single-quoted string when single-quotes are present in *bytes*. + For example, the byte string ``'Python'`` would be represented as + ``b"'Python'"`` when *smartquotes* is true, or ``b'\'Python\''`` when it is + false. + + On success, this function returns a :term:`strong reference` to a + :class:`str` object containing the representation. On failure, this + returns ``NULL`` with an exception set. + + +.. c:function:: PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t len, const char *errors, Py_ssize_t unicode, const char *recode_encoding) + + Unescape a backslash-escaped string *s*. *s* must not be ``NULL``. + *len* must be the size of *s*. + + *errors* must be one of ``"strict"``, ``"replace"``, or ``"ignore"``. If + *errors* is ``NULL``, then ``"strict"`` is used by default. + + On success, this function returns a :term:`strong reference` to a Python + :class:`bytes` object containing the unescaped string. On failure, this + function returns ``NULL`` with an exception set. + + .. versionchanged:: 3.9 + *unicode* and *recode_encoding* are now unused. + + +.. _pybyteswriter: + +PyBytesWriter +------------- + +The :c:type:`PyBytesWriter` API can be used to create a Python :class:`bytes` +object. + +.. versionadded:: 3.15 + +.. c:type:: PyBytesWriter + + A bytes writer instance. + + The API is **not thread safe**: a writer should only be used by a single + thread at the same time. + + The instance must be destroyed by :c:func:`PyBytesWriter_Finish` on + success, or :c:func:`PyBytesWriter_Discard` on error. + + +Create, Finish, Discard +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: PyBytesWriter* PyBytesWriter_Create(Py_ssize_t size) + + Create a :c:type:`PyBytesWriter` to write *size* bytes. + + If *size* is greater than zero, allocate *size* bytes, and set the + writer size to *size*. The caller is responsible to write *size* + bytes using :c:func:`PyBytesWriter_GetData`. + This function does not overallocate. + + On error, set an exception and return ``NULL``. + + *size* must be positive or zero. + +.. c:function:: PyObject* PyBytesWriter_Finish(PyBytesWriter *writer) + + Finish a :c:type:`PyBytesWriter` created by + :c:func:`PyBytesWriter_Create`. + + On success, return a Python :class:`bytes` object. + On error, set an exception and return ``NULL``. + + The writer instance is invalid after the call in any case. + No API can be called on the writer after :c:func:`PyBytesWriter_Finish`. + +.. c:function:: PyObject* PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) + + Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer + to *size* bytes before creating the :class:`bytes` object. + +.. c:function:: PyObject* PyBytesWriter_FinishWithPointer(PyBytesWriter *writer, void *buf) + + Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer + using *buf* pointer before creating the :class:`bytes` object. + + Set an exception and return ``NULL`` if *buf* pointer is outside the + internal buffer bounds. + + Function pseudo-code:: + + Py_ssize_t size = (char*)buf - (char*)PyBytesWriter_GetData(writer); + return PyBytesWriter_FinishWithSize(writer, size); + +.. c:function:: void PyBytesWriter_Discard(PyBytesWriter *writer) + + Discard a :c:type:`PyBytesWriter` created by :c:func:`PyBytesWriter_Create`. + + Do nothing if *writer* is ``NULL``. + + The writer instance is invalid after the call. + No API can be called on the writer after :c:func:`PyBytesWriter_Discard`. + +High-level API +^^^^^^^^^^^^^^ + +.. c:function:: int PyBytesWriter_WriteBytes(PyBytesWriter *writer, const void *bytes, Py_ssize_t size) + + Grow the *writer* internal buffer by *size* bytes, + write *size* bytes of *bytes* at the *writer* end, + and add *size* to the *writer* size. + + If *size* is equal to ``-1``, call ``strlen(bytes)`` to get the + string length. + + On success, return ``0``. + On error, set an exception and return ``-1``. + +.. c:function:: int PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) + + Similar to :c:func:`PyBytes_FromFormat`, but write the output directly at + the writer end. Grow the writer internal buffer on demand. Then add the + written size to the writer size. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + +Getters +^^^^^^^ + +.. c:function:: Py_ssize_t PyBytesWriter_GetSize(PyBytesWriter *writer) + + Get the writer size. + +.. c:function:: void* PyBytesWriter_GetData(PyBytesWriter *writer) + + Get the writer data: start of the internal buffer. + + The pointer is valid until :c:func:`PyBytesWriter_Finish` or + :c:func:`PyBytesWriter_Discard` is called on *writer*. + + +Low-level API +^^^^^^^^^^^^^ + +.. c:function:: int PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) + + Resize the writer to *size* bytes. It can be used to enlarge or to + shrink the writer. + This function typically overallocates to achieve amortized performance when + resizing multiple times. + + Newly allocated bytes are left uninitialized. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + *size* must be positive or zero. + +.. c:function:: int PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t grow) + + Resize the writer by adding *grow* bytes to the current writer size. + This function typically overallocates to achieve amortized performance when + resizing multiple times. + + Newly allocated bytes are left uninitialized. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + *size* can be negative to shrink the writer. + +.. c:function:: void* PyBytesWriter_GrowAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, void *buf) + + Similar to :c:func:`PyBytesWriter_Grow`, but update also the *buf* + pointer. + + The *buf* pointer is moved if the internal buffer is moved in memory. + The *buf* relative position within the internal buffer is left + unchanged. + + On error, set an exception and return ``NULL``. + + *buf* must not be ``NULL``. + + Function pseudo-code:: + + Py_ssize_t pos = (char*)buf - (char*)PyBytesWriter_GetData(writer); + if (PyBytesWriter_Grow(writer, size) < 0) { + return NULL; + } + return (char*)PyBytesWriter_GetData(writer) + pos; diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index 7198d6bc056eb4..9838879a528934 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -347,6 +347,8 @@ please see individual documentation for details. .. versionadded:: 3.9 +.. c:function:: PyObject* _PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + :no-typesetting: .. c:function:: PyObject* PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) @@ -358,7 +360,12 @@ please see individual documentation for details. Return the result of the call on success, or raise an exception and return *NULL* on failure. - .. versionadded:: 3.9 + .. versionadded:: 3.8 as ``_PyObject_Vectorcall`` + + .. versionchanged:: 3.9 + + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. c:function:: PyObject* PyObject_VectorcallDict(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwdict) diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index cdb8aa33e9fd32..03a848d68ed7ab 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -15,13 +15,19 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:type:: PyCapsule This subtype of :c:type:`PyObject` represents an opaque value, useful for C - extension modules who need to pass an opaque value (as a :c:expr:`void*` + extension modules which need to pass an opaque value (as a :c:expr:`void*` pointer) through Python code to other C code. It is often used to make a C function pointer defined in one module available to other modules, so the regular import mechanism can be used to access C APIs defined in dynamically loaded modules. +.. c:var:: PyTypeObject PyCapsule_Type + + The type object corresponding to capsule objects. This is the same object + as :class:`types.CapsuleType` in the Python layer. + + .. c:type:: PyCapsule_Destructor The type of a destructor callback for a capsule. Defined as:: @@ -105,9 +111,19 @@ Refer to :ref:`using-capsules` for more information on using these objects. ``module.attribute``. The *name* stored in the capsule must match this string exactly. + This function splits *name* on the ``.`` character, and imports the first + element. It then processes further elements using attribute lookups. + Return the capsule's internal *pointer* on success. On failure, set an exception and return ``NULL``. + .. note:: + + If *name* points to an attribute of some submodule or subpackage, this + submodule or subpackage must be previously imported using other means + (for example, by using :c:func:`PyImport_ImportModule`) for the + attribute lookups to succeed. + .. versionchanged:: 3.3 *no_block* has no effect anymore. diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index 61eb994c370946..2501ed9580df89 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -7,7 +7,7 @@ Cell Objects "Cell" objects are used to implement variables referenced by multiple scopes. For each such variable, a cell object is created to store the value; the local -variables of each stack frame that references the value contains a reference to +variables of each stack frame that references the value contain a reference to the cells from outer scopes which also use that variable. When the value is accessed, the value contained in the cell is used instead of the cell object itself. This de-referencing of the cell object requires support from the diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 6eae24b38fae48..048bc2c2154e77 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -69,13 +69,14 @@ bound into a function. The old name is deprecated, but will remain available until the signature changes again. +.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(...) + :no-typesetting: + .. c:function:: PyCodeObject* PyUnstable_Code_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) Similar to :c:func:`PyUnstable_Code_New`, but with an extra "posonlyargcount" for positional-only arguments. The same caveats that apply to ``PyUnstable_Code_New`` also apply to this function. - .. index:: single: PyCode_NewWithPosOnlyArgs (C function) - .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs`` .. versionchanged:: 3.11 @@ -182,7 +183,7 @@ bound into a function. Type of a code object watcher callback function. If *event* is ``PY_CODE_EVENT_CREATE``, then the callback is invoked - after `co` has been fully initialized. Otherwise, the callback is invoked + after *co* has been fully initialized. Otherwise, the callback is invoked before the destruction of *co* takes place, so the prior state of *co* can be inspected. @@ -211,6 +212,82 @@ bound into a function. .. versionadded:: 3.12 +.. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) + + This is a :term:`soft deprecated` function that does nothing. + + Prior to Python 3.10, this function would perform basic optimizations to a + code object. + + .. versionchanged:: 3.10 + This function now does nothing. + + +.. _c_codeobject_flags: + +Code Object Flags +----------------- + +Code objects contain a bit-field of flags, which can be retrieved as the +:attr:`~codeobject.co_flags` Python attribute (for example using +:c:func:`PyObject_GetAttrString`), and set using a *flags* argument to +:c:func:`PyUnstable_Code_New` and similar functions. + +Flags whose names start with ``CO_FUTURE_`` correspond to features normally +selectable by :ref:`future statements `. These flags can be used in +:c:member:`PyCompilerFlags.cf_flags`. +Note that many ``CO_FUTURE_`` flags are mandatory in current versions of +Python, and setting them has no effect. + +The following flags are available. +For their meaning, see the linked documentation of their Python equivalents. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Flag + * Meaning + * * .. c:macro:: CO_OPTIMIZED + * :py:data:`inspect.CO_OPTIMIZED` + * * .. c:macro:: CO_NEWLOCALS + * :py:data:`inspect.CO_NEWLOCALS` + * * .. c:macro:: CO_VARARGS + * :py:data:`inspect.CO_VARARGS` + * * .. c:macro:: CO_VARKEYWORDS + * :py:data:`inspect.CO_VARKEYWORDS` + * * .. c:macro:: CO_NESTED + * :py:data:`inspect.CO_NESTED` + * * .. c:macro:: CO_GENERATOR + * :py:data:`inspect.CO_GENERATOR` + * * .. c:macro:: CO_COROUTINE + * :py:data:`inspect.CO_COROUTINE` + * * .. c:macro:: CO_ITERABLE_COROUTINE + * :py:data:`inspect.CO_ITERABLE_COROUTINE` + * * .. c:macro:: CO_ASYNC_GENERATOR + * :py:data:`inspect.CO_ASYNC_GENERATOR` + * * .. c:macro:: CO_HAS_DOCSTRING + * :py:data:`inspect.CO_HAS_DOCSTRING` + * * .. c:macro:: CO_METHOD + * :py:data:`inspect.CO_METHOD` + + * * .. c:macro:: CO_FUTURE_DIVISION + * no effect (:py:data:`__future__.division`) + * * .. c:macro:: CO_FUTURE_ABSOLUTE_IMPORT + * no effect (:py:data:`__future__.absolute_import`) + * * .. c:macro:: CO_FUTURE_WITH_STATEMENT + * no effect (:py:data:`__future__.with_statement`) + * * .. c:macro:: CO_FUTURE_PRINT_FUNCTION + * no effect (:py:data:`__future__.print_function`) + * * .. c:macro:: CO_FUTURE_UNICODE_LITERALS + * no effect (:py:data:`__future__.unicode_literals`) + * * .. c:macro:: CO_FUTURE_GENERATOR_STOP + * no effect (:py:data:`__future__.generator_stop`) + * * .. c:macro:: CO_FUTURE_ANNOTATIONS + * :py:data:`__future__.annotations` + + Extra information ----------------- @@ -222,9 +299,12 @@ These functions are part of the unstable C API tier: this functionality is a CPython implementation detail, and the API may change without deprecation warnings. +.. c:function:: Py_ssize_t _PyEval_RequestCodeExtraIndex(freefunc free) + :no-typesetting: + .. c:function:: Py_ssize_t PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) - Return a new an opaque index value used to adding data to code objects. + Return a new opaque index value used to adding data to code objects. You generally call this function once (per interpreter) and use the result with ``PyCode_GetExtra`` and ``PyCode_SetExtra`` to manipulate @@ -234,8 +314,6 @@ may change without deprecation warnings. *free* will be called on non-``NULL`` data stored under the new index. Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`. - .. index:: single: _PyEval_RequestCodeExtraIndex (C function) - .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex`` .. versionchanged:: 3.12 @@ -244,6 +322,9 @@ may change without deprecation warnings. The old private name is deprecated, but will be available until the API changes. +.. c:function:: int _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) + :no-typesetting: + .. c:function:: int PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra) Set *extra* to the extra data stored under the given index. @@ -252,8 +333,6 @@ may change without deprecation warnings. If no data was set under the index, set *extra* to ``NULL`` and return 0 without setting an exception. - .. index:: single: _PyCode_GetExtra (C function) - .. versionadded:: 3.6 as ``_PyCode_GetExtra`` .. versionchanged:: 3.12 @@ -262,13 +341,14 @@ may change without deprecation warnings. The old private name is deprecated, but will be available until the API changes. +.. c:function:: int _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) + :no-typesetting: + .. c:function:: int PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra) Set the extra data stored under the given index to *extra*. Return 0 on success. Set an exception and return -1 on failure. - .. index:: single: _PyCode_SetExtra (C function) - .. versionadded:: 3.6 as ``_PyCode_SetExtra`` .. versionchanged:: 3.12 diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst index 8ae5c4fecd6248..35ee048bd5fa9f 100644 --- a/Doc/c-api/codec.rst +++ b/Doc/c-api/codec.rst @@ -7,7 +7,7 @@ Codec registry and support functions Register a new codec search function. - As side effect, this tries to load the :mod:`!encodings` package, if not yet + As a side effect, this tries to load the :mod:`!encodings` package, if not yet done, to make sure that it is always first in the list of search functions. .. c:function:: int PyCodec_Unregister(PyObject *search_function) @@ -39,7 +39,7 @@ Codec registry and support functions *object* is passed through the decoder function found for the given *encoding* using the error handling method defined by *errors*. *errors* may be ``NULL`` to use the default method defined for the codec. Raises a - :exc:`LookupError` if no encoder can be found. + :exc:`LookupError` if no decoder can be found. Codec lookup API @@ -129,3 +129,13 @@ Registry API for Unicode encoding error handlers Replace the unicode encode error with ``\N{...}`` escapes. .. versionadded:: 3.5 + + +Codec utility variables +----------------------- + +.. c:var:: const char *Py_hexdigits + + A string constant containing the lowercase hexadecimal digits: ``"0123456789abcdef"``. + + .. versionadded:: 3.3 diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 16bd79475dc1e6..629312bd771beb 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -3,92 +3,24 @@ .. _complexobjects: Complex Number Objects ----------------------- +====================== .. index:: pair: object; complex number -Python's complex number objects are implemented as two distinct types when -viewed from the C API: one is the Python object exposed to Python programs, and -the other is a C structure which represents the actual complex number value. -The API provides functions for working with both. - - -Complex Numbers as C Structures -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Note that the functions which accept these structures as parameters and return -them as results do so *by value* rather than dereferencing them through -pointers. This is consistent throughout the API. - - -.. c:type:: Py_complex - - The C structure which corresponds to the value portion of a Python complex - number object. Most of the functions for dealing with complex number objects - use structures of this type as input or output values, as appropriate. - - .. c:member:: double real - double imag - - The structure is defined as:: - - typedef struct { - double real; - double imag; - } Py_complex; - - -.. c:function:: Py_complex _Py_c_sum(Py_complex left, Py_complex right) - - Return the sum of two complex numbers, using the C :c:type:`Py_complex` - representation. - - -.. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) - - Return the difference between two complex numbers, using the C - :c:type:`Py_complex` representation. - - -.. c:function:: Py_complex _Py_c_neg(Py_complex num) - - Return the negation of the complex number *num*, using the C - :c:type:`Py_complex` representation. - - -.. c:function:: Py_complex _Py_c_prod(Py_complex left, Py_complex right) - - Return the product of two complex numbers, using the C :c:type:`Py_complex` - representation. - - -.. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) - - Return the quotient of two complex numbers, using the C :c:type:`Py_complex` - representation. - - If *divisor* is null, this method returns zero and sets - :c:data:`errno` to :c:macro:`!EDOM`. - - -.. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) - - Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` - representation. - - If *num* is null and *exp* is not a positive real number, - this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. - - Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. +.. c:type:: PyComplexObject -Complex Numbers as Python Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + This subtype of :c:type:`PyObject` represents a Python complex number object. + .. c:member:: Py_complex cval -.. c:type:: PyComplexObject + The complex number value, using the C :c:type:`Py_complex` representation. - This subtype of :c:type:`PyObject` represents a Python complex number object. + .. deprecated-removed:: 3.15 3.20 + Use :c:func:`PyComplex_AsCComplex` and + :c:func:`PyComplex_FromCComplex` to convert a + Python complex number to/from the C :c:type:`Py_complex` + representation. .. c:var:: PyTypeObject PyComplex_Type @@ -109,12 +41,6 @@ Complex Numbers as Python Objects :c:type:`PyComplexObject`. This function always succeeds. -.. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) - - Create a new Python complex number object from a C :c:type:`Py_complex` value. - Return ``NULL`` with an exception set on error. - - .. c:function:: PyObject* PyComplex_FromDoubles(double real, double imag) Return a new :c:type:`PyComplexObject` object from *real* and *imag*. @@ -153,6 +79,29 @@ Complex Numbers as Python Objects .. versionchanged:: 3.13 Use :meth:`~object.__complex__` if available. + +.. c:type:: Py_complex + + This C structure defines an export format for a Python complex + number object. + + .. c:member:: double real + double imag + + The structure is defined as:: + + typedef struct { + double real; + double imag; + } Py_complex; + + +.. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) + + Create a new Python complex number object from a C :c:type:`Py_complex` value. + Return ``NULL`` with an exception set on error. + + .. c:function:: Py_complex PyComplex_AsCComplex(PyObject *op) Return the :c:type:`Py_complex` value of the complex number *op*. @@ -169,3 +118,82 @@ Complex Numbers as Python Objects .. versionchanged:: 3.8 Use :meth:`~object.__index__` if available. + + +Complex Numbers as C Structures +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The API also provides functions for working with complex numbers, using the +:c:type:`Py_complex` representation. Note that the functions which accept +these structures as parameters and return them as results do so *by value* +rather than dereferencing them through pointers. + +Please note, that these functions are :term:`soft deprecated` since Python +3.15. Avoid using this API in a new code to do complex arithmetic: either use +the `Number Protocol `_ API or use native complex types, like +:c:expr:`double complex`. + + +.. c:function:: Py_complex _Py_c_sum(Py_complex left, Py_complex right) + + Return the sum of two complex numbers, using the C :c:type:`Py_complex` + representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) + + Return the difference between two complex numbers, using the C + :c:type:`Py_complex` representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_neg(Py_complex num) + + Return the negation of the complex number *num*, using the C + :c:type:`Py_complex` representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_prod(Py_complex left, Py_complex right) + + Return the product of two complex numbers, using the C :c:type:`Py_complex` + representation. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) + + Return the quotient of two complex numbers, using the C :c:type:`Py_complex` + representation. + + If *divisor* is null, this method returns zero and sets + :c:data:`errno` to :c:macro:`!EDOM`. + + .. deprecated:: 3.15 + + +.. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) + + Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` + representation. + + If *num* is null and *exp* is not a positive real number, + this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. + + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + + .. deprecated:: 3.15 + + +.. c:function:: double _Py_c_abs(Py_complex num) + + Return the absolute value of the complex number *num*. + + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + + .. deprecated:: 3.15 diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 880f7b15ce68e8..1746fe95eaaca9 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -109,11 +109,20 @@ Other Objects descriptor.rst slice.rst memoryview.rst + picklebuffer.rst weakref.rst capsule.rst frame.rst gen.rst coro.rst contextvars.rst - datetime.rst typehints.rst + + +C API for extension modules +=========================== + +.. toctree:: + + curses.rst + datetime.rst diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index 4aaf3905e81c8a..f7c8ef8b22b955 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -41,7 +41,7 @@ The return value (*rv*) for these functions should be interpreted as follows: ``rv + 1`` bytes would have been needed to succeed. ``str[size-1]`` is ``'\0'`` in this case. -* When ``rv < 0``, "something bad happened." ``str[size-1]`` is ``'\0'`` in +* When ``rv < 0``, the output conversion failed and ``str[size-1]`` is ``'\0'`` in this case too, but the rest of *str* is undefined. The exact cause of the error depends on the underlying platform. @@ -105,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_HUGE_VAL`` (with + if ``overflow_exception`` is ``NULL`` return :c:macro:`!INFINITY` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set @@ -128,22 +128,46 @@ The following functions provide locale-independent string to number conversions. must be 0 and is ignored. The ``'r'`` format code specifies the standard :func:`repr` format. - *flags* can be zero or more of the values ``Py_DTSF_SIGN``, - ``Py_DTSF_ADD_DOT_0``, or ``Py_DTSF_ALT``, or-ed together: + *flags* can be zero or more of the following values or-ed together: - * ``Py_DTSF_SIGN`` means to always precede the returned string with a sign - character, even if *val* is non-negative. + .. c:namespace:: NULL - * ``Py_DTSF_ADD_DOT_0`` means to ensure that the returned string will not look - like an integer. + .. c:macro:: Py_DTSF_SIGN - * ``Py_DTSF_ALT`` means to apply "alternate" formatting rules. See the - documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for - details. + Always precede the returned string with a sign + character, even if *val* is non-negative. - If *ptype* is non-``NULL``, then the value it points to will be set to one of - ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that - *val* is a finite number, an infinite number, or not a number, respectively. + .. c:macro:: Py_DTSF_ADD_DOT_0 + + Ensure that the returned string will not look like an integer. + + .. c:macro:: Py_DTSF_ALT + + Apply "alternate" formatting rules. + See the documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for + details. + + .. c:macro:: Py_DTSF_NO_NEG_0 + + Negative zero is converted to positive zero. + + .. versionadded:: 3.11 + + If *ptype* is non-``NULL``, then the value it points to will be set to one + of the following constants depending on the type of *val*: + + .. list-table:: + :header-rows: 1 + :align: left + + * - *\*ptype* + - type of *val* + * - .. c:macro:: Py_DTST_FINITE + - finite number + * - .. c:macro:: Py_DTST_INFINITE + - infinite number + * - .. c:macro:: Py_DTST_NAN + - not a number The return value is a pointer to *buffer* with the converted string or ``NULL`` if the conversion failed. The caller is responsible for freeing the @@ -152,13 +176,85 @@ The following functions provide locale-independent string to number conversions. .. versionadded:: 3.1 -.. c:function:: int PyOS_stricmp(const char *s1, const char *s2) +.. c:function:: int PyOS_mystricmp(const char *str1, const char *str2) + int PyOS_mystrnicmp(const char *str1, const char *str2, Py_ssize_t size) + + Case insensitive comparison of strings. These functions work almost + identically to :c:func:`!strcmp` and :c:func:`!strncmp` (respectively), + except that they ignore the case of ASCII characters. + + Return ``0`` if the strings are equal, a negative value if *str1* sorts + lexicographically before *str2*, or a positive value if it sorts after. + + In the *str1* or *str2* arguments, a NUL byte marks the end of the string. + For :c:func:`!PyOS_mystrnicmp`, the *size* argument gives the maximum size + of the string, as if NUL was present at the index given by *size*. + + These functions do not use the locale. + + +.. c:function:: int PyOS_stricmp(const char *str1, const char *str2) + int PyOS_strnicmp(const char *str1, const char *str2, Py_ssize_t size) + + Case insensitive comparison of strings. + + On Windows, these are aliases of :c:func:`!stricmp` and :c:func:`!strnicmp`, + respectively. + + On other platforms, they are aliases of :c:func:`PyOS_mystricmp` and + :c:func:`PyOS_mystrnicmp`, respectively. + + +Character classification and conversion +======================================= + +The following macros provide locale-independent (unlike the C standard library +``ctype.h``) character classification and conversion. +The argument must be a signed or unsigned :c:expr:`char`. + + +.. c:macro:: Py_ISALNUM(c) + + Return true if the character *c* is an alphanumeric character. + + +.. c:macro:: Py_ISALPHA(c) + + Return true if the character *c* is an alphabetic character (``a-z`` and ``A-Z``). + + +.. c:macro:: Py_ISDIGIT(c) + + Return true if the character *c* is a decimal digit (``0-9``). + + +.. c:macro:: Py_ISLOWER(c) + + Return true if the character *c* is a lowercase ASCII letter (``a-z``). + + +.. c:macro:: Py_ISUPPER(c) + + Return true if the character *c* is an uppercase ASCII letter (``A-Z``). + + +.. c:macro:: Py_ISSPACE(c) + + Return true if the character *c* is a whitespace character (space, tab, + carriage return, newline, vertical tab, or form feed). + + +.. c:macro:: Py_ISXDIGIT(c) + + Return true if the character *c* is a hexadecimal digit (``0-9``, ``a-f``, and + ``A-F``). + + +.. c:macro:: Py_TOLOWER(c) - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strcmp` except that it ignores the case. + Return the lowercase equivalent of the character *c*. -.. c:function:: int PyOS_strnicmp(const char *s1, const char *s2, Py_ssize_t size) +.. c:macro:: Py_TOUPPER(c) - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strncmp` except that it ignores the case. + Return the uppercase equivalent of the character *c*. diff --git a/Doc/c-api/curses.rst b/Doc/c-api/curses.rst new file mode 100644 index 00000000000000..5a1697c43cc969 --- /dev/null +++ b/Doc/c-api/curses.rst @@ -0,0 +1,138 @@ +.. highlight:: c + +Curses C API +------------ + +:mod:`curses` exposes a small C interface for extension modules. +Consumers must include the header file :file:`py_curses.h` (which is not +included by default by :file:`Python.h`) and :c:func:`import_curses` must +be invoked, usually as part of the module initialisation function, to populate +:c:var:`PyCurses_API`. + +.. warning:: + + Neither the C API nor the pure Python :mod:`curses` module are compatible + with subinterpreters. + +.. c:macro:: import_curses() + + Import the curses C API. The macro does not need a semi-colon to be called. + + On success, populate the :c:var:`PyCurses_API` pointer. + + On failure, set :c:var:`PyCurses_API` to NULL and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + import_curses(); // semi-colon is optional but recommended + if (PyErr_Occurred()) { /* cleanup */ } + + +.. c:var:: void **PyCurses_API + + Dynamically allocated object containing the curses C API. + This variable is only available once :c:macro:`import_curses` succeeds. + + ``PyCurses_API[0]`` corresponds to :c:data:`PyCursesWindow_Type`. + + ``PyCurses_API[1]``, ``PyCurses_API[2]``, and ``PyCurses_API[3]`` + are pointers to predicate functions of type ``int (*)(void)``. + + When called, these predicates return whether :func:`curses.setupterm`, + :func:`curses.initscr`, and :func:`curses.start_color` have been called + respectively. + + See also the convenience macros :c:macro:`PyCursesSetupTermCalled`, + :c:macro:`PyCursesInitialised`, and :c:macro:`PyCursesInitialisedColor`. + + .. note:: + + The number of entries in this structure is subject to changes. + Consider using :c:macro:`PyCurses_API_pointers` to check if + new fields are available or not. + + +.. c:macro:: PyCurses_API_pointers + + The number of accessible fields (``4``) in :c:var:`PyCurses_API`. + This number is incremented whenever new fields are added. + + +.. c:var:: PyTypeObject PyCursesWindow_Type + + The :ref:`heap type ` corresponding to :class:`curses.window`. + + +.. c:function:: int PyCursesWindow_Check(PyObject *op) + + Return true if *op* is a :class:`curses.window` instance, false otherwise. + + +The following macros are convenience macros expanding into C statements. +In particular, they can only be used as ``macro;`` or ``macro``, but not +``macro()`` or ``macro();``. + +.. c:macro:: PyCursesSetupTermCalled + + Macro checking if :func:`curses.setupterm` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_setupterm_called = (predicate_t)PyCurses_API[1]; + if (!was_setupterm_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialised + + Macro checking if :func:`curses.initscr` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_initscr_called = (predicate_t)PyCurses_API[2]; + if (!was_initscr_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialisedColor + + Macro checking if :func:`curses.start_color` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_start_color_called = (predicate_t)PyCurses_API[3]; + if (!was_start_color_called()) { + return NULL; + } + } + + +Internal data +------------- + +The following objects are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyCurses_CAPSULE_NAME + + Name of the curses capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`import_curses` instead. + diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index d2d4d5309c7098..127d7c9c91a3d5 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -8,11 +8,42 @@ DateTime Objects Various date and time objects are supplied by the :mod:`datetime` module. Before using any of these functions, the header file :file:`datetime.h` must be included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of +and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following +into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following macros. +.. c:macro:: PyDateTime_IMPORT() + + Import the datetime C API. + + On success, populate the :c:var:`PyDateTimeAPI` pointer. + On failure, set :c:var:`PyDateTimeAPI` to ``NULL`` and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + PyDateTime_IMPORT; + if (PyErr_Occurred()) { /* cleanup */ } + + .. warning:: + + This is not compatible with subinterpreters. + +.. c:type:: PyDateTime_CAPI + + Structure containing the fields for the datetime C API. + + The fields of this structure are private and subject to change. + + Do not use this directly; prefer ``PyDateTime_*`` APIs instead. + +.. c:var:: PyDateTime_CAPI *PyDateTimeAPI + + Dynamically allocated object containing the datetime C API. + + This variable is only available once :c:macro:`PyDateTime_IMPORT` succeeds. + .. c:type:: PyDateTime_Date This subtype of :c:type:`PyObject` represents a Python date object. @@ -46,7 +77,7 @@ macros. .. c:var:: PyTypeObject PyDateTime_DeltaType - This instance of :c:type:`PyTypeObject` represents Python type for + This instance of :c:type:`PyTypeObject` represents the Python type for the difference between two datetime values; it is the same object as :class:`datetime.timedelta` in the Python layer. @@ -325,3 +356,16 @@ Macros for the convenience of modules implementing the DB API: Create and return a new :class:`datetime.date` object given an argument tuple suitable for passing to :meth:`datetime.date.fromtimestamp`. + + +Internal data +------------- + +The following symbols are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyDateTime_CAPSULE_NAME + + Name of the datetime capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`PyDateTime_IMPORT` instead. diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index b32c113e5f0457..e23288c6a58590 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -10,31 +10,127 @@ found in the dictionary of type objects. .. XXX document these! -.. c:var:: PyTypeObject PyProperty_Type +.. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) + - The type object for the built-in descriptor types. +.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) -.. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) +.. c:var:: PyTypeObject PyMemberDescr_Type + The type object for member descriptor objects created from + :c:type:`PyMemberDef` structures. These descriptors expose fields of a + C struct as attributes on a type, and correspond + to :class:`types.MemberDescriptorType` objects in Python. -.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) + + +.. c:var:: PyTypeObject PyGetSetDescr_Type + + The type object for get/set descriptor objects created from + :c:type:`PyGetSetDef` structures. These descriptors implement attributes + whose value is computed by C getter and setter functions, and are used + for many built-in type attributes. .. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth) +.. c:var:: PyTypeObject PyMethodDescr_Type + + The type object for method descriptor objects created from + :c:type:`PyMethodDef` structures. These descriptors expose C functions as + methods on a type, and correspond to :class:`types.MemberDescriptorType` + objects in Python. + + .. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped) +.. c:var:: PyTypeObject PyWrapperDescr_Type + + The type object for wrapper descriptor objects created by + :c:func:`PyDescr_NewWrapper` and :c:func:`PyWrapper_New`. Wrapper + descriptors are used internally to expose special methods implemented + via wrapper structures, and appear in Python as + :class:`types.WrapperDescriptorType` objects. + + .. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) .. c:function:: int PyDescr_IsData(PyObject *descr) - Return non-zero if the descriptor objects *descr* describes a data attribute, or + Return non-zero if the descriptor object *descr* describes a data attribute, or ``0`` if it describes a method. *descr* must be a descriptor object; there is no error checking. .. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) + + +.. c:macro:: PyDescr_COMMON + + This is a :term:`soft deprecated` macro including the common fields for a + descriptor object. + + This was included in Python's C API by mistake; do not use it in extensions. + For creating custom descriptor objects, create a class implementing the + descriptor protocol (:c:member:`~PyTypeObject.tp_descr_get` and + :c:member:`~PyTypeObject.tp_descr_set`). + + +Built-in descriptors +^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyProperty_Type + + The type object for property objects. This is the same object as + :class:`property` in the Python layer. + + +.. c:var:: PyTypeObject PySuper_Type + + The type object for super objects. This is the same object as + :class:`super` in the Python layer. + + +.. c:var:: PyTypeObject PyClassMethod_Type + + The type of class method objects. This is the same object as + :class:`classmethod` in the Python layer. + + +.. c:var:: PyTypeObject PyClassMethodDescr_Type + + The type object for C-level class method descriptor objects. + This is the type of the descriptors created for :func:`classmethod` defined in + C extension types, and is the same object as :class:`classmethod` + in Python. + + +.. c:function:: PyObject *PyClassMethod_New(PyObject *callable) + + Create a new :class:`classmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new class + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + + +.. c:var:: PyTypeObject PyStaticMethod_Type + + The type of static method objects. This is the same object as + :class:`staticmethod` in the Python layer. + + +.. c:function:: PyObject *PyStaticMethod_New(PyObject *callable) + + Create a new :class:`staticmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new static + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index ce73fa0cc60ebb..9c4428ced41b5a 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -43,6 +43,17 @@ Dictionary Objects prevent modification of the dictionary for non-dynamic class types. +.. c:var:: PyTypeObject PyDictProxy_Type + + The type object for mapping proxy objects created by + :c:func:`PyDictProxy_New` and for the read-only ``__dict__`` attribute + of many built-in types. A :c:type:`PyDictProxy_Type` instance provides a + dynamic, read-only view of an underlying dictionary: changes to the + underlying dictionary are reflected in the proxy, but the proxy itself + does not support mutation operations. This corresponds to + :class:`types.MappingProxyType` in Python. + + .. c:function:: void PyDict_Clear(PyObject *p) Empty an existing dictionary of all key-value pairs. @@ -50,7 +61,7 @@ Dictionary Objects .. c:function:: int PyDict_Contains(PyObject *p, PyObject *key) - Determine if dictionary *p* contains *key*. If an item in *p* is matches + Determine if dictionary *p* contains *key*. If an item in *p* matches *key*, return ``1``, otherwise return ``0``. On error, return ``-1``. This is equivalent to the Python expression ``key in p``. @@ -127,7 +138,7 @@ Dictionary Objects Prefer the :c:func:`PyDict_GetItemWithError` function instead. .. versionchanged:: 3.10 - Calling this API without :term:`GIL` held had been allowed for historical + Calling this API without an :term:`attached thread state` had been allowed for historical reason. It is no longer allowed. @@ -198,7 +209,7 @@ Dictionary Objects .. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) Remove *key* from dictionary *p* and optionally return the removed value. - Do not raise :exc:`KeyError` if the key missing. + Do not raise :exc:`KeyError` if the key is missing. - If the key is present, set *\*result* to a new reference to the removed value if *result* is not ``NULL``, and return ``1``. @@ -207,7 +218,7 @@ Dictionary Objects - On error, raise an exception and return ``-1``. Similar to :meth:`dict.pop`, but without the default value and - not raising :exc:`KeyError` if the key missing. + not raising :exc:`KeyError` if the key is missing. .. versionadded:: 3.13 @@ -245,6 +256,11 @@ Dictionary Objects ``len(p)`` on a dictionary. +.. c:function:: Py_ssize_t PyDict_GET_SIZE(PyObject *p) + + Similar to :c:func:`PyDict_Size`, but without error checking. + + .. c:function:: int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) Iterate over all key-value pairs in the dictionary *p*. The @@ -301,6 +317,15 @@ Dictionary Objects } Py_END_CRITICAL_SECTION(); + .. note:: + + On the free-threaded build, this function can be used safely inside a + critical section. However, the references returned for *pkey* and *pvalue* + are :term:`borrowed ` and are only valid while the + critical section is held. If you need to use these objects outside the + critical section or when the critical section can be suspended, create a + :term:`strong reference ` (for example, using + :c:func:`Py_NewRef`). .. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) @@ -417,3 +442,138 @@ Dictionary Objects it before returning. .. versionadded:: 3.12 + + +Dictionary View Objects +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: int PyDictViewSet_Check(PyObject *op) + + Return true if *op* is a view of a set inside a dictionary. This is currently + equivalent to :c:expr:`PyDictKeys_Check(op) || PyDictItems_Check(op)`. This + function always succeeds. + + +.. c:var:: PyTypeObject PyDictKeys_Type + + Type object for a view of dictionary keys. In Python, this is the type of + the object returned by :meth:`dict.keys`. + + +.. c:function:: int PyDictKeys_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary keys view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictValues_Type + + Type object for a view of dictionary values. In Python, this is the type of + the object returned by :meth:`dict.values`. + + +.. c:function:: int PyDictValues_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary values view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictItems_Type + + Type object for a view of dictionary items. In Python, this is the type of + the object returned by :meth:`dict.items`. + + +.. c:function:: int PyDictItems_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary items view. This function + always succeeds. + + +Ordered Dictionaries +^^^^^^^^^^^^^^^^^^^^ + +Python's C API provides interface for :class:`collections.OrderedDict` from C. +Since Python 3.7, dictionaries are ordered by default, so there is usually +little need for these functions; prefer ``PyDict*`` where possible. + + +.. c:var:: PyTypeObject PyODict_Type + + Type object for ordered dictionaries. This is the same object as + :class:`collections.OrderedDict` in the Python layer. + + +.. c:function:: int PyODict_Check(PyObject *od) + + Return true if *od* is an ordered dictionary object or an instance of a + subtype of the :class:`~collections.OrderedDict` type. This function + always succeeds. + + +.. c:function:: int PyODict_CheckExact(PyObject *od) + + Return true if *od* is an ordered dictionary object, but not an instance of + a subtype of the :class:`~collections.OrderedDict` type. + This function always succeeds. + + +.. c:var:: PyTypeObject PyODictKeys_Type + + Analogous to :c:type:`PyDictKeys_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictValues_Type + + Analogous to :c:type:`PyDictValues_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictItems_Type + + Analogous to :c:type:`PyDictItems_Type` for ordered dictionaries. + + +.. c:function:: PyObject *PyODict_New(void) + + Return a new empty ordered dictionary, or ``NULL`` on failure. + + This is analogous to :c:func:`PyDict_New`. + + +.. c:function:: int PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) + + Insert *value* into the ordered dictionary *od* with a key of *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_SetItem`. + + +.. c:function:: int PyODict_DelItem(PyObject *od, PyObject *key) + + Remove the entry in the ordered dictionary *od* with key *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_DelItem`. + + +These are :term:`soft deprecated` aliases to ``PyDict`` APIs: + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * ``PyODict`` + * ``PyDict`` + * * .. c:macro:: PyODict_GetItem(od, key) + * :c:func:`PyDict_GetItem` + * * .. c:macro:: PyODict_GetItemWithError(od, key) + * :c:func:`PyDict_GetItemWithError` + * * .. c:macro:: PyODict_GetItemString(od, key) + * :c:func:`PyDict_GetItemString` + * * .. c:macro:: PyODict_Contains(od, key) + * :c:func:`PyDict_Contains` + * * .. c:macro:: PyODict_Size(od) + * :c:func:`PyDict_Size` + * * .. c:macro:: PyODict_SIZE(od) + * :c:func:`PyDict_GET_SIZE` diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index fc2336d120c259..59af470f59ff34 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -309,6 +309,14 @@ For convenience, some of these functions will always return a .. versionadded:: 3.4 +.. c:function:: void PyErr_RangedSyntaxLocationObject(PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset) + + Similar to :c:func:`PyErr_SyntaxLocationObject`, but also sets the + *end_lineno* and *end_col_offset* information for the current exception. + + .. versionadded:: 3.10 + + .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string @@ -331,6 +339,23 @@ For convenience, some of these functions will always return a use. +.. c:function:: PyObject *PyErr_ProgramTextObject(PyObject *filename, int lineno) + + Get the source line in *filename* at line *lineno*. *filename* should be a + Python :class:`str` object. + + On success, this function returns a Python string object with the found line. + On failure, this function returns ``NULL`` without an exception set. + + +.. c:function:: PyObject *PyErr_ProgramText(const char *filename, int lineno) + + Similar to :c:func:`PyErr_ProgramTextObject`, but *filename* is a + :c:expr:`const char *`, which is decoded with the + :term:`filesystem encoding and error handler`, instead of a + Python object reference. + + Issuing warnings ================ @@ -394,6 +419,15 @@ an error value). .. versionadded:: 3.2 +.. c:function:: int PyErr_WarnExplicitFormat(PyObject *category, const char *filename, int lineno, const char *module, PyObject *registry, const char *format, ...) + + Similar to :c:func:`PyErr_WarnExplicit`, but uses + :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is + an ASCII-encoded string. + + .. versionadded:: 3.2 + + .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is @@ -413,7 +447,7 @@ Querying the error indicator own a reference to the return value, so you do not need to :c:func:`Py_DECREF` it. - The caller must hold the GIL. + The caller must have an :term:`attached thread state`. .. note:: @@ -675,7 +709,7 @@ Signal Handling .. note:: This function is async-signal-safe. It can be called without - the :term:`GIL` and from a C signal handler. + an :term:`attached thread state` and from a C signal handler. .. c:function:: int PyErr_SetInterruptEx(int signum) @@ -702,7 +736,7 @@ Signal Handling .. note:: This function is async-signal-safe. It can be called without - the :term:`GIL` and from a C signal handler. + an :term:`attached thread state` and from a C signal handler. .. versionadded:: 3.10 @@ -749,9 +783,41 @@ Exception Classes .. versionadded:: 3.2 +.. c:function:: int PyExceptionClass_Check(PyObject *ob) + + Return non-zero if *ob* is an exception class, zero otherwise. This function always succeeds. + + +.. c:function:: const char *PyExceptionClass_Name(PyObject *ob) + + Return :c:member:`~PyTypeObject.tp_name` of the exception class *ob*. + + +.. c:macro:: PyException_HEAD + + This is a :term:`soft deprecated` macro including the base fields for an + exception object. + + This was included in Python's C API by mistake and is not designed for use + in extensions. For creating custom exception objects, use + :c:func:`PyErr_NewException` or otherwise create a class inheriting from + :c:data:`PyExc_BaseException`. + + Exception Objects ================= +.. c:function:: int PyExceptionInstance_Check(PyObject *op) + + Return true if *op* is an instance of :class:`BaseException`, false + otherwise. This function always succeeds. + + +.. c:macro:: PyExceptionInstance_Class(op) + + Equivalent to :c:func:`Py_TYPE(op) `. + + .. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) Return the traceback associated with the exception as a new reference, as @@ -853,12 +919,23 @@ The following functions are used to create and modify Unicode exceptions from C. *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. + If the :attr:`UnicodeError.object` is an empty sequence, the resulting + *start* is ``0``. Otherwise, it is clipped to ``[0, len(object) - 1]``. + + .. seealso:: :attr:`UnicodeError.start` + .. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) - Set the *start* attribute of the given exception object to *start*. Return - ``0`` on success, ``-1`` on failure. + Set the *start* attribute of the given exception object to *start*. + Return ``0`` on success, ``-1`` on failure. + + .. note:: + + While passing a negative *start* does not raise an exception, + the corresponding getters will not consider it as a relative + offset. .. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) @@ -868,6 +945,9 @@ The following functions are used to create and modify Unicode exceptions from C. *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. + If the :attr:`UnicodeError.object` is an empty sequence, the resulting + *end* is ``0``. Otherwise, it is clipped to ``[1, len(object)]``. + .. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) @@ -875,6 +955,8 @@ The following functions are used to create and modify Unicode exceptions from C. Set the *end* attribute of the given exception object to *end*. Return ``0`` on success, ``-1`` on failure. + .. seealso:: :attr:`UnicodeError.end` + .. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) @@ -905,11 +987,7 @@ because the :ref:`call protocol ` takes care of recursion handling. Marks a point where a recursive C-level call is about to be performed. - If :c:macro:`!USE_STACKCHECK` is defined, this function checks if the OS - stack overflowed using :c:func:`PyOS_CheckStack`. If this is the case, it - sets a :exc:`MemoryError` and returns a nonzero value. - - The function then checks if the recursion limit is reached. If this is the + The function then checks if the stack limit is reached. If this is the case, a :exc:`RecursionError` is set and a nonzero value is returned. Otherwise, zero is returned. @@ -917,6 +995,9 @@ because the :ref:`call protocol ` takes care of recursion handling. be concatenated to the :exc:`RecursionError` message caused by the recursion depth limit. + .. seealso:: + The :c:func:`PyUnstable_ThreadState_SetStackProtection` function. + .. versionchanged:: 3.9 This function is now also available in the :ref:`limited API `. @@ -957,184 +1038,159 @@ these are the C equivalent to :func:`reprlib.recursive_repr`. Ends a :c:func:`Py_ReprEnter`. Must be called once for each invocation of :c:func:`Py_ReprEnter` that returns zero. +.. c:function:: int Py_GetRecursionLimit(void) + + Get the recursion limit for the current interpreter. It can be set with + :c:func:`Py_SetRecursionLimit`. The recursion limit prevents the + Python interpreter stack from growing infinitely. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.getrecursionlimit` + +.. c:function:: void Py_SetRecursionLimit(int new_limit) + + Set the recursion limit for the current interpreter. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.setrecursionlimit` .. _standardexceptions: -Standard Exceptions -=================== - -All standard Python exceptions are available as global variables whose names are -``PyExc_`` followed by the Python exception name. These have the type -:c:expr:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_BaseException (C var) - single: PyExc_Exception (C var) - single: PyExc_ArithmeticError (C var) - single: PyExc_AssertionError (C var) - single: PyExc_AttributeError (C var) - single: PyExc_BlockingIOError (C var) - single: PyExc_BrokenPipeError (C var) - single: PyExc_BufferError (C var) - single: PyExc_ChildProcessError (C var) - single: PyExc_ConnectionAbortedError (C var) - single: PyExc_ConnectionError (C var) - single: PyExc_ConnectionRefusedError (C var) - single: PyExc_ConnectionResetError (C var) - single: PyExc_EOFError (C var) - single: PyExc_FileExistsError (C var) - single: PyExc_FileNotFoundError (C var) - single: PyExc_FloatingPointError (C var) - single: PyExc_GeneratorExit (C var) - single: PyExc_ImportError (C var) - single: PyExc_IndentationError (C var) - single: PyExc_IndexError (C var) - single: PyExc_InterruptedError (C var) - single: PyExc_IsADirectoryError (C var) - single: PyExc_KeyError (C var) - single: PyExc_KeyboardInterrupt (C var) - single: PyExc_LookupError (C var) - single: PyExc_MemoryError (C var) - single: PyExc_ModuleNotFoundError (C var) - single: PyExc_NameError (C var) - single: PyExc_NotADirectoryError (C var) - single: PyExc_NotImplementedError (C var) - single: PyExc_OSError (C var) - single: PyExc_OverflowError (C var) - single: PyExc_PermissionError (C var) - single: PyExc_ProcessLookupError (C var) - single: PyExc_PythonFinalizationError (C var) - single: PyExc_RecursionError (C var) - single: PyExc_ReferenceError (C var) - single: PyExc_RuntimeError (C var) - single: PyExc_StopAsyncIteration (C var) - single: PyExc_StopIteration (C var) - single: PyExc_SyntaxError (C var) - single: PyExc_SystemError (C var) - single: PyExc_SystemExit (C var) - single: PyExc_TabError (C var) - single: PyExc_TimeoutError (C var) - single: PyExc_TypeError (C var) - single: PyExc_UnboundLocalError (C var) - single: PyExc_UnicodeDecodeError (C var) - single: PyExc_UnicodeEncodeError (C var) - single: PyExc_UnicodeError (C var) - single: PyExc_UnicodeTranslateError (C var) - single: PyExc_ValueError (C var) - single: PyExc_ZeroDivisionError (C var) - -+-----------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+=========================================+=================================+==========+ -| :c:data:`PyExc_BaseException` | :exc:`BaseException` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_Exception` | :exc:`Exception` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NameError` | :exc:`NameError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OSError` | :exc:`OSError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PythonFinalizationError` | :exc:`PythonFinalizationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TabError` | :exc:`TabError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | -+-----------------------------------------+---------------------------------+----------+ +Exception and warning types +=========================== + +All standard Python exceptions and warning categories are available as global +variables whose names are ``PyExc_`` followed by the Python exception name. +These have the type :c:expr:`PyObject*`; they are all class objects. + +For completeness, here are all the variables: + +Exception types +--------------- + +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * * .. c:var:: PyObject *PyExc_BaseException + * :exc:`BaseException` + * * .. c:var:: PyObject *PyExc_BaseExceptionGroup + * :exc:`BaseExceptionGroup` + * * .. c:var:: PyObject *PyExc_Exception + * :exc:`Exception` + * * .. c:var:: PyObject *PyExc_ArithmeticError + * :exc:`ArithmeticError` + * * .. c:var:: PyObject *PyExc_AssertionError + * :exc:`AssertionError` + * * .. c:var:: PyObject *PyExc_AttributeError + * :exc:`AttributeError` + * * .. c:var:: PyObject *PyExc_BlockingIOError + * :exc:`BlockingIOError` + * * .. c:var:: PyObject *PyExc_BrokenPipeError + * :exc:`BrokenPipeError` + * * .. c:var:: PyObject *PyExc_BufferError + * :exc:`BufferError` + * * .. c:var:: PyObject *PyExc_ChildProcessError + * :exc:`ChildProcessError` + * * .. c:var:: PyObject *PyExc_ConnectionAbortedError + * :exc:`ConnectionAbortedError` + * * .. c:var:: PyObject *PyExc_ConnectionError + * :exc:`ConnectionError` + * * .. c:var:: PyObject *PyExc_ConnectionRefusedError + * :exc:`ConnectionRefusedError` + * * .. c:var:: PyObject *PyExc_ConnectionResetError + * :exc:`ConnectionResetError` + * * .. c:var:: PyObject *PyExc_EOFError + * :exc:`EOFError` + * * .. c:var:: PyObject *PyExc_FileExistsError + * :exc:`FileExistsError` + * * .. c:var:: PyObject *PyExc_FileNotFoundError + * :exc:`FileNotFoundError` + * * .. c:var:: PyObject *PyExc_FloatingPointError + * :exc:`FloatingPointError` + * * .. c:var:: PyObject *PyExc_GeneratorExit + * :exc:`GeneratorExit` + * * .. c:var:: PyObject *PyExc_ImportError + * :exc:`ImportError` + * * .. c:var:: PyObject *PyExc_IndentationError + * :exc:`IndentationError` + * * .. c:var:: PyObject *PyExc_IndexError + * :exc:`IndexError` + * * .. c:var:: PyObject *PyExc_InterruptedError + * :exc:`InterruptedError` + * * .. c:var:: PyObject *PyExc_IsADirectoryError + * :exc:`IsADirectoryError` + * * .. c:var:: PyObject *PyExc_KeyError + * :exc:`KeyError` + * * .. c:var:: PyObject *PyExc_KeyboardInterrupt + * :exc:`KeyboardInterrupt` + * * .. c:var:: PyObject *PyExc_LookupError + * :exc:`LookupError` + * * .. c:var:: PyObject *PyExc_MemoryError + * :exc:`MemoryError` + * * .. c:var:: PyObject *PyExc_ModuleNotFoundError + * :exc:`ModuleNotFoundError` + * * .. c:var:: PyObject *PyExc_NameError + * :exc:`NameError` + * * .. c:var:: PyObject *PyExc_NotADirectoryError + * :exc:`NotADirectoryError` + * * .. c:var:: PyObject *PyExc_NotImplementedError + * :exc:`NotImplementedError` + * * .. c:var:: PyObject *PyExc_OSError + * :exc:`OSError` + * * .. c:var:: PyObject *PyExc_OverflowError + * :exc:`OverflowError` + * * .. c:var:: PyObject *PyExc_PermissionError + * :exc:`PermissionError` + * * .. c:var:: PyObject *PyExc_ProcessLookupError + * :exc:`ProcessLookupError` + * * .. c:var:: PyObject *PyExc_PythonFinalizationError + * :exc:`PythonFinalizationError` + * * .. c:var:: PyObject *PyExc_RecursionError + * :exc:`RecursionError` + * * .. c:var:: PyObject *PyExc_ReferenceError + * :exc:`ReferenceError` + * * .. c:var:: PyObject *PyExc_RuntimeError + * :exc:`RuntimeError` + * * .. c:var:: PyObject *PyExc_StopAsyncIteration + * :exc:`StopAsyncIteration` + * * .. c:var:: PyObject *PyExc_StopIteration + * :exc:`StopIteration` + * * .. c:var:: PyObject *PyExc_SyntaxError + * :exc:`SyntaxError` + * * .. c:var:: PyObject *PyExc_SystemError + * :exc:`SystemError` + * * .. c:var:: PyObject *PyExc_SystemExit + * :exc:`SystemExit` + * * .. c:var:: PyObject *PyExc_TabError + * :exc:`TabError` + * * .. c:var:: PyObject *PyExc_TimeoutError + * :exc:`TimeoutError` + * * .. c:var:: PyObject *PyExc_TypeError + * :exc:`TypeError` + * * .. c:var:: PyObject *PyExc_UnboundLocalError + * :exc:`UnboundLocalError` + * * .. c:var:: PyObject *PyExc_UnicodeDecodeError + * :exc:`UnicodeDecodeError` + * * .. c:var:: PyObject *PyExc_UnicodeEncodeError + * :exc:`UnicodeEncodeError` + * * .. c:var:: PyObject *PyExc_UnicodeError + * :exc:`UnicodeError` + * * .. c:var:: PyObject *PyExc_UnicodeTranslateError + * :exc:`UnicodeTranslateError` + * * .. c:var:: PyObject *PyExc_ValueError + * :exc:`ValueError` + * * .. c:var:: PyObject *PyExc_ZeroDivisionError + * :exc:`ZeroDivisionError` .. versionadded:: 3.3 :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, @@ -1152,88 +1208,116 @@ the variables: .. versionadded:: 3.6 :c:data:`PyExc_ModuleNotFoundError`. -These are compatibility aliases to :c:data:`PyExc_OSError`: +.. versionadded:: 3.11 + :c:data:`PyExc_BaseExceptionGroup`. -.. index:: - single: PyExc_EnvironmentError (C var) - single: PyExc_IOError (C var) - single: PyExc_WindowsError (C var) -+-------------------------------------+----------+ -| C Name | Notes | -+=====================================+==========+ -| :c:data:`!PyExc_EnvironmentError` | | -+-------------------------------------+----------+ -| :c:data:`!PyExc_IOError` | | -+-------------------------------------+----------+ -| :c:data:`!PyExc_WindowsError` | [2]_ | -+-------------------------------------+----------+ +OSError aliases +--------------- + +The following are a compatibility aliases to :c:data:`PyExc_OSError`. .. versionchanged:: 3.3 These aliases used to be separate exception types. +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * Notes + * * .. c:var:: PyObject *PyExc_EnvironmentError + * :exc:`OSError` + * + * * .. c:var:: PyObject *PyExc_IOError + * :exc:`OSError` + * + * * .. c:var:: PyObject *PyExc_WindowsError + * :exc:`OSError` + * [win]_ + Notes: -.. [1] - This is a base class for other standard exceptions. +.. [win] + :c:var:`!PyExc_WindowsError` is only defined on Windows; protect code that + uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. -.. [2] - Only defined on Windows; protect code that uses this by testing that the - preprocessor macro ``MS_WINDOWS`` is defined. .. _standardwarningcategories: -Standard Warning Categories -=========================== - -All standard Python warning categories are available as global variables whose -names are ``PyExc_`` followed by the Python exception name. These have the type -:c:expr:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_Warning (C var) - single: PyExc_BytesWarning (C var) - single: PyExc_DeprecationWarning (C var) - single: PyExc_FutureWarning (C var) - single: PyExc_ImportWarning (C var) - single: PyExc_PendingDeprecationWarning (C var) - single: PyExc_ResourceWarning (C var) - single: PyExc_RuntimeWarning (C var) - single: PyExc_SyntaxWarning (C var) - single: PyExc_UnicodeWarning (C var) - single: PyExc_UserWarning (C var) - -+------------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+==========================================+=================================+==========+ -| :c:data:`PyExc_Warning` | :exc:`Warning` | [3]_ | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | -+------------------------------------------+---------------------------------+----------+ +Warning types +------------- + +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * * .. c:var:: PyObject *PyExc_Warning + * :exc:`Warning` + * * .. c:var:: PyObject *PyExc_BytesWarning + * :exc:`BytesWarning` + * * .. c:var:: PyObject *PyExc_DeprecationWarning + * :exc:`DeprecationWarning` + * * .. c:var:: PyObject *PyExc_EncodingWarning + * :exc:`EncodingWarning` + * * .. c:var:: PyObject *PyExc_FutureWarning + * :exc:`FutureWarning` + * * .. c:var:: PyObject *PyExc_ImportWarning + * :exc:`ImportWarning` + * * .. c:var:: PyObject *PyExc_PendingDeprecationWarning + * :exc:`PendingDeprecationWarning` + * * .. c:var:: PyObject *PyExc_ResourceWarning + * :exc:`ResourceWarning` + * * .. c:var:: PyObject *PyExc_RuntimeWarning + * :exc:`RuntimeWarning` + * * .. c:var:: PyObject *PyExc_SyntaxWarning + * :exc:`SyntaxWarning` + * * .. c:var:: PyObject *PyExc_UnicodeWarning + * :exc:`UnicodeWarning` + * * .. c:var:: PyObject *PyExc_UserWarning + * :exc:`UserWarning` .. versionadded:: 3.2 :c:data:`PyExc_ResourceWarning`. -Notes: +.. versionadded:: 3.10 + :c:data:`PyExc_EncodingWarning`. + + +Tracebacks +========== + +.. c:var:: PyTypeObject PyTraceBack_Type + + Type object for traceback objects. This is available as + :class:`types.TracebackType` in the Python layer. + + +.. c:function:: int PyTraceBack_Check(PyObject *op) + + Return true if *op* is a traceback object, false otherwise. This function + does not account for subtypes. + + +.. c:function:: int PyTraceBack_Here(PyFrameObject *f) + + Replace the :attr:`~BaseException.__traceback__` attribute on the current + exception with a new traceback prepending *f* to the existing chain. + + Calling this function without an exception set is undefined behavior. + + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. + + +.. c:function:: int PyTraceBack_Print(PyObject *tb, PyObject *f) + + Write the traceback *tb* into the file *f*. -.. [3] - This is a base class for other standard warning categories. + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst new file mode 100644 index 00000000000000..92b531665e135d --- /dev/null +++ b/Doc/c-api/extension-modules.rst @@ -0,0 +1,342 @@ +.. highlight:: c + +.. _extension-modules: + +Defining extension modules +-------------------------- + +A C extension for CPython is a shared library (for example, a ``.so`` file +on Linux, ``.pyd`` DLL on Windows), which is loadable into the Python process +(for example, it is compiled with compatible compiler settings), and which +exports an :dfn:`export hook` function (or an +old-style :ref:`initialization function `). + +To be importable by default (that is, by +:py:class:`importlib.machinery.ExtensionFileLoader`), +the shared library must be available on :py:attr:`sys.path`, +and must be named after the module name plus an extension listed in +:py:attr:`importlib.machinery.EXTENSION_SUFFIXES`. + +.. note:: + + Building, packaging and distributing extension modules is best done with + third-party tools, and is out of scope of this document. + One suitable tool is Setuptools, whose documentation can be found at + https://setuptools.pypa.io/en/latest/setuptools.html. + +.. _extension-export-hook: + +Extension export hook +..................... + +.. versionadded:: 3.15 + + Support for the :samp:`PyModExport_{}` export hook was added in Python + 3.15. The older way of defining modules is still available: consult either + the :ref:`extension-pyinit` section or earlier versions of this + documentation if you plan to support earlier Python versions. + +The export hook must be an exported function with the following signature: + +.. c:function:: PyModuleDef_Slot *PyModExport_modulename(void) + +For modules with ASCII-only names, the :ref:`export hook ` +must be named :samp:`PyModExport_{}`, +with ```` replaced by the module's name. + +For non-ASCII module names, the export hook must instead be named +:samp:`PyModExportU_{}` (note the ``U``), with ```` encoded using +Python's *punycode* encoding with hyphens replaced by underscores. In Python: + +.. code-block:: python + + def hook_name(name): + try: + suffix = b'_' + name.encode('ascii') + except UnicodeEncodeError: + suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') + return b'PyModExport' + suffix + +The export hook returns an array of :c:type:`PyModuleDef_Slot` entries, +terminated by an entry with a slot ID of ``0``. +These slots describe how the module should be created and initialized. + +This array must remain valid and constant until interpreter shutdown. +Typically, it should use ``static`` storage. +Prefer using the :c:macro:`Py_mod_create` and :c:macro:`Py_mod_exec` slots +for any dynamic behavior. + +The export hook may return ``NULL`` with an exception set to signal failure. + +It is recommended to define the export hook function using a helper macro: + +.. c:macro:: PyMODEXPORT_FUNC + + Declare an extension module export hook. + This macro: + + * specifies the :c:expr:`PyModuleDef_Slot*` return type, + * adds any special linkage declarations required by the platform, and + * for C++, declares the function as ``extern "C"``. + +For example, a module called ``spam`` would be defined like this:: + + PyABIInfo_VAR(abi_info); + + static PyModuleDef_Slot spam_slots[] = { + {Py_mod_abi, &abi_info}, + {Py_mod_name, "spam"}, + {Py_mod_init, spam_init_function}, + ... + {0, NULL}, + }; + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return spam_slots; + } + +The export hook is typically the only non-\ ``static`` +item defined in the module's C source. + +The hook should be kept short -- ideally, one line as above. +If you do need to use Python C API in this function, it is recommended to call +``PyABIInfo_Check(&abi_info, "modulename")`` first to raise an exception, +rather than crash, in common cases of ABI mismatch. + + +.. note:: + + It is possible to export multiple modules from a single shared library by + defining multiple export hooks. + However, importing them requires a custom importer or suitably named + copies/links of the extension file, because Python's import machinery only + finds the function corresponding to the filename. + See the `Multiple modules in one library `__ + section in :pep:`489` for details. + + +.. _multi-phase-initialization: + +Multi-phase initialization +.......................... + +The process of creating an extension module follows several phases: + +- Python finds and calls the export hook to get information on how to + create the module. +- Before any substantial code is executed, Python can determine which + capabilities the module supports, and it can adjust the environment or + refuse loading an incompatible extension. + Slots like :c:data:`Py_mod_abi`, :c:data:`Py_mod_gil` and + :c:data:`Py_mod_multiple_interpreters` influence this step. +- By default, Python itself then creates the module object -- that is, it does + the equivalent of calling :py:meth:`~object.__new__` when creating an object. + This step can be overridden using the :c:data:`Py_mod_create` slot. +- Python sets initial module attributes like :attr:`~module.__package__` and + :attr:`~module.__loader__`, and inserts the module object into + :py:attr:`sys.modules`. +- Afterwards, the module object is initialized in an extension-specific way + -- the equivalent of :py:meth:`~object.__init__` when creating an object, + or of executing top-level code in a Python-language module. + The behavior is specified using the :c:data:`Py_mod_exec` slot. + +This is called *multi-phase initialization* to distinguish it from the legacy +(but still supported) :ref:`single-phase initialization `, +where an initialization function returns a fully constructed module. + +.. versionchanged:: 3.5 + + Added support for multi-phase initialization (:pep:`489`). + + +Multiple module instances +......................... + +By default, extension modules are not singletons. +For example, if the :py:attr:`sys.modules` entry is removed and the module +is re-imported, a new module object is created and, typically, populated with +fresh method and type objects. +The old module is subject to normal garbage collection. +This mirrors the behavior of pure-Python modules. + +Additional module instances may be created in +:ref:`sub-interpreters ` +or after Python runtime reinitialization +(:c:func:`Py_Finalize` and :c:func:`Py_Initialize`). +In these cases, sharing Python objects between module instances would likely +cause crashes or undefined behavior. + +To avoid such issues, each instance of an extension module should +be *isolated*: changes to one instance should not implicitly affect the others, +and all state owned by the module, including references to Python objects, +should be specific to a particular module instance. +See :ref:`isolating-extensions-howto` for more details and a practical guide. + +A simpler way to avoid these issues is +:ref:`raising an error on repeated initialization `. + +All modules are expected to support +:ref:`sub-interpreters `, or otherwise explicitly +signal a lack of support. +This is usually achieved by isolation or blocking repeated initialization, +as above. +A module may also be limited to the main interpreter using +the :c:data:`Py_mod_multiple_interpreters` slot. + + +.. _extension-pyinit: + +``PyInit`` function +................... + +.. deprecated:: 3.15 + + This functionality is :term:`soft deprecated`. + It will not get new features, but there are no plans to remove it. + +Instead of :c:func:`PyModExport_modulename`, an extension module can define +an older-style :dfn:`initialization function` with the signature: + +.. c:function:: PyObject* PyInit_modulename(void) + +Its name should be :samp:`PyInit_{}`, with ```` replaced by the +name of the module. +For non-ASCII module names, use :samp:`PyInitU_{}` instead, with +```` encoded in the same way as for the +:ref:`export hook ` (that is, using Punycode +with underscores). + +If a module exports both :samp:`PyInit_{}` and +:samp:`PyModExport_{}`, the :samp:`PyInit_{}` function +is ignored. + +Like with :c:macro:`PyMODEXPORT_FUNC`, it is recommended to define the +initialization function using a helper macro: + +.. c:macro:: PyMODINIT_FUNC + + Declare an extension module initialization function. + This macro: + + * specifies the :c:expr:`PyObject*` return type, + * adds any special linkage declarations required by the platform, and + * for C++, declares the function as ``extern "C"``. + + +Normally, the initialization function (``PyInit_modulename``) returns +a :c:type:`PyModuleDef` instance with non-``NULL`` +:c:member:`~PyModuleDef.m_slots`. This allows Python to use +:ref:`multi-phase initialization `. + +Before it is returned, the ``PyModuleDef`` instance must be initialized +using the following function: + +.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) + + Ensure a module definition is a properly initialized Python object that + correctly reports its type and a reference count. + + Return *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. + + Calling this function is required before returning a :c:type:`PyModuleDef` + from a module initialization function. + It should not be used in other contexts. + + Note that Python assumes that ``PyModuleDef`` structures are statically + allocated. + This function may return either a new reference or a borrowed one; + this reference must not be released. + + .. versionadded:: 3.5 + + +For example, a module called ``spam`` would be defined like this:: + + static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + ... + }; + + PyMODINIT_FUNC + PyInit_spam(void) + { + return PyModuleDef_Init(&spam_module); + } + + +.. _single-phase-initialization: + +Legacy single-phase initialization +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. deprecated:: 3.15 + + Single-phase initialization is :term:`soft deprecated`. + It is a legacy mechanism to initialize extension + modules, with known drawbacks and design flaws. Extension module authors + are encouraged to use multi-phase initialization instead. + + However, there are no plans to remove support for it. + +In single-phase initialization, the old-style +:ref:`initialization function ` (``PyInit_modulename``) +should create, populate and return a module object. +This is typically done using :c:func:`PyModule_Create` and functions like +:c:func:`PyModule_AddObjectRef`. + +Single-phase initialization differs from the :ref:`default ` +in the following ways: + +* Single-phase modules are, or rather *contain*, “singletons”. + + When the module is first initialized, Python saves the contents of + the module's ``__dict__`` (that is, typically, the module's functions and + types). + + For subsequent imports, Python does not call the initialization function + again. + Instead, it creates a new module object with a new ``__dict__``, and copies + the saved contents to it. + For example, given a single-phase module ``_testsinglephase`` + [#testsinglephase]_ that defines a function ``sum`` and an exception class + ``error``: + + .. code-block:: python + + >>> import sys + >>> import _testsinglephase as one + >>> del sys.modules['_testsinglephase'] + >>> import _testsinglephase as two + >>> one is two + False + >>> one.__dict__ is two.__dict__ + False + >>> one.sum is two.sum + True + >>> one.error is two.error + True + + The exact behavior should be considered a CPython implementation detail. + +* To work around the fact that ``PyInit_modulename`` does not take a *spec* + argument, some state of the import machinery is saved and applied to the + first suitable module created during the ``PyInit_modulename`` call. + Specifically, when a sub-module is imported, this mechanism prepends the + parent package name to the name of the module. + + A single-phase ``PyInit_modulename`` function should create “its” module + object as soon as possible, before any other module objects can be created. + +* Non-ASCII module names (``PyInitU_modulename``) are not supported. + +* Single-phase modules support module lookup functions like + :c:func:`PyState_FindModule`. + +* The module's :c:member:`PyModuleDef.m_slots` must be NULL. + +.. [#testsinglephase] ``_testsinglephase`` is an internal module used + in CPython's self-test suite; your installation may or may not + include it. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index e9019a0d500f7e..0580e4c8f79db0 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -93,6 +93,29 @@ the :mod:`io` APIs instead. .. versionadded:: 3.8 +.. c:function:: PyObject *PyFile_OpenCodeObject(PyObject *path) + + Open *path* with the mode ``'rb'``. *path* must be a Python :class:`str` + object. The behavior of this function may be overridden by + :c:func:`PyFile_SetOpenCodeHook` to allow for some preprocessing of the + text. + + This is analogous to :func:`io.open_code` in Python. + + On success, this function returns a :term:`strong reference` to a Python + file object. On failure, this function returns ``NULL`` with an exception + set. + + .. versionadded:: 3.8 + + +.. c:function:: PyObject *PyFile_OpenCode(const char *path) + + Similar to :c:func:`PyFile_OpenCodeObject`, but *path* is a + UTF-8 encoded :c:expr:`const char*`. + + .. versionadded:: 3.8 + .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) @@ -108,3 +131,22 @@ the :mod:`io` APIs instead. Write string *s* to file object *p*. Return ``0`` on success or ``-1`` on failure; the appropriate exception will be set. + + +Deprecated API +^^^^^^^^^^^^^^ + + +These are :term:`soft deprecated` APIs that were included in Python's C API +by mistake. They are documented solely for completeness; use other +``PyFile*`` APIs instead. + +.. c:function:: PyObject *PyFile_NewStdPrinter(int fd) + + Use :c:func:`PyFile_FromFd` with defaults (``fd, NULL, "w", -1, NULL, NULL, NULL, 0``) instead. + +.. c:var:: PyTypeObject PyStdPrinter_Type + + Type of file-like objects used internally at Python startup when :py:mod:`io` is + not yet available. + Use Python :py:func:`open` or :c:func:`PyFile_FromFd` to create file objects instead. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 1da37a5bcaeef9..b0d440580b9886 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -78,6 +78,111 @@ Floating-Point Objects Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`. +.. c:macro:: Py_INFINITY + + This macro expands a to constant expression of type :c:expr:`double`, that + represents the positive infinity. + + It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard + ```` header. + + .. deprecated:: 3.15 + The macro is :term:`soft deprecated`. + + +.. c:macro:: Py_NAN + + This macro expands a to constant expression of type :c:expr:`double`, that + represents a quiet not-a-number (qNaN) value. + + On most platforms, this is equivalent to the :c:macro:`!NAN` macro from + the C11 standard ```` header. + + +.. c:macro:: Py_HUGE_VAL + + Equivalent to :c:macro:`!INFINITY`. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. + + +.. c:macro:: Py_MATH_E + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. + + +.. c:macro:: Py_MATH_El + + High precision (long double) definition of :data:`~math.e` constant. + + .. deprecated-removed:: 3.15 3.20 + + +.. c:macro:: Py_MATH_PI + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.pi` constant. + + +.. c:macro:: Py_MATH_PIl + + High precision (long double) definition of :data:`~math.pi` constant. + + .. deprecated-removed:: 3.15 3.20 + + +.. c:macro:: Py_MATH_TAU + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.tau` constant. + + .. versionadded:: 3.6 + + +.. c:macro:: Py_RETURN_NAN + + Return :data:`math.nan` from a function. + + On most platforms, this is equivalent to ``return PyFloat_FromDouble(NAN)``. + + +.. c:macro:: Py_RETURN_INF(sign) + + Return :data:`math.inf` or :data:`-math.inf ` from a function, + depending on the sign of *sign*. + + On most platforms, this is equivalent to the following:: + + return PyFloat_FromDouble(copysign(INFINITY, sign)); + + +.. c:macro:: Py_IS_FINITE(X) + + Return ``1`` if the given floating-point number *X* is finite, + that is, it is normal, subnormal or zero, but not infinite or NaN. + Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isfinite` instead. + + +.. c:macro:: Py_IS_INFINITY(X) + + Return ``1`` if the given floating-point number *X* is positive or negative + infinity. Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isinf` instead. + + +.. c:macro:: Py_IS_NAN(X) + + Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) + value. Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isnan` instead. + + Pack and Unpack functions ------------------------- @@ -96,6 +201,9 @@ NaNs (if such things exist on the platform) isn't handled correctly, and attempting to unpack a bytes string containing an IEEE INF or NaN will raise an exception. +Note that NaNs type may not be preserved on IEEE platforms (signaling NaN become +quiet NaN), for example on x86 systems in 32-bit mode. + On non-IEEE platforms with more precision, or larger dynamic range, than IEEE 754 supports, not all values can be packed; on non-IEEE platforms with less precision, or smaller dynamic range, not all values can be unpacked. What @@ -121,15 +229,15 @@ There are two problems on non-IEEE platforms: * What this does is undefined if *x* is a NaN or infinity. * ``-0.0`` and ``+0.0`` produce the same bytes string. -.. c:function:: int PyFloat_Pack2(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack2(double x, char *p, int le) Pack a C double as the IEEE 754 binary16 half-precision format. -.. c:function:: int PyFloat_Pack4(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack4(double x, char *p, int le) Pack a C double as the IEEE 754 binary32 single precision format. -.. c:function:: int PyFloat_Pack8(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack8(double x, char *p, int le) Pack a C double as the IEEE 754 binary64 double precision format. @@ -151,14 +259,14 @@ Return value: The unpacked double. On error, this is ``-1.0`` and Note that on a non-IEEE platform this will refuse to unpack a bytes string that represents a NaN or infinity. -.. c:function:: double PyFloat_Unpack2(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack2(const char *p, int le) Unpack the IEEE 754 binary16 half-precision format as a C double. -.. c:function:: double PyFloat_Unpack4(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack4(const char *p, int le) Unpack the IEEE 754 binary32 single precision format as a C double. -.. c:function:: double PyFloat_Unpack8(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack8(const char *p, int le) Unpack the IEEE 754 binary64 double precision format as a C double. diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 638a740e0c24da..fb17cf7f1da6b2 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -29,6 +29,12 @@ See also :ref:`Reflection `. Previously, this type was only available after including ````. +.. c:function:: PyFrameObject *PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, PyObject *locals) + + Create a new frame object. This function returns a :term:`strong reference` + to the new frame object on success, and returns ``NULL`` with an exception + set on failure. + .. c:function:: int PyFrame_Check(PyObject *obj) Return non-zero if *obj* is a frame object. @@ -132,7 +138,7 @@ See also :ref:`Reflection `. .. versionadded:: 3.11 .. versionchanged:: 3.13 - As part of :pep:`667`, return a proxy object for optimized scopes. + As part of :pep:`667`, return an instance of :c:var:`PyFrameLocalsProxy_Type`. .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) @@ -140,6 +146,77 @@ See also :ref:`Reflection `. Return the line number that *frame* is currently executing. +Frame Locals Proxies +^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 3.13 + +The :attr:`~frame.f_locals` attribute on a :ref:`frame object ` +is an instance of a "frame-locals proxy". The proxy object exposes a +write-through view of the underlying locals dictionary for the frame. This +ensures that the variables exposed by ``f_locals`` are always up to date with +the live local variables in the frame itself. + +See :pep:`667` for more information. + +.. c:var:: PyTypeObject PyFrameLocalsProxy_Type + + The type of frame :func:`locals` proxy objects. + +.. c:function:: int PyFrameLocalsProxy_Check(PyObject *obj) + + Return non-zero if *obj* is a frame :func:`locals` proxy. + + +Legacy Local Variable APIs +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. +They exist solely for backwards compatibility. + + +.. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) + + This function is :term:`soft deprecated` and does nothing. + + Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` + attribute of *f* to the internal "fast" array of local variables, allowing + changes in frame objects to be visible to the interpreter. If *clear* was + true, this function would process variables that were unset in the locals + dictionary. + + .. versionchanged:: 3.13 + This function now does nothing. + + +.. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) + + This function is :term:`soft deprecated` and does nothing. + + Prior to Python 3.13, this function would copy the internal "fast" array + of local variables (which is used by the interpreter) to the + :attr:`~frame.f_locals` attribute of *f*, allowing changes in local + variables to be visible to frame objects. + + .. versionchanged:: 3.13 + This function now does nothing. + + +.. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) + + This function is :term:`soft deprecated` and does nothing. + + Prior to Python 3.13, this function was similar to + :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and + ``-1`` with an exception set on failure. + + .. versionchanged:: 3.13 + This function now does nothing. + + +.. seealso:: + :pep:`667` + Internal Frames ^^^^^^^^^^^^^^^ diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index e7fb5090c09933..609b5e885b6b16 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -95,6 +95,22 @@ There are a few functions specific to Python functions. .. versionadded:: 3.12 + +.. c:function:: PyObject* PyFunction_GetKwDefaults(PyObject *op) + + Return the keyword-only argument default values of the function object *op*. This can be a + dictionary of arguments or ``NULL``. + + +.. c:function:: int PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults) + + Set the keyword-only argument default values of the function object *op*. + *defaults* must be a dictionary of keyword-only arguments or ``Py_None``. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + .. c:function:: PyObject* PyFunction_GetClosure(PyObject *op) Return the closure associated with the function object *op*. This can be ``NULL`` @@ -123,6 +139,19 @@ There are a few functions specific to Python functions. Raises :exc:`SystemError` and returns ``-1`` on failure. +.. c:function:: PyObject *PyFunction_GET_CODE(PyObject *op) + PyObject *PyFunction_GET_GLOBALS(PyObject *op) + PyObject *PyFunction_GET_MODULE(PyObject *op) + PyObject *PyFunction_GET_DEFAULTS(PyObject *op) + PyObject *PyFunction_GET_KW_DEFAULTS(PyObject *op) + PyObject *PyFunction_GET_CLOSURE(PyObject *op) + PyObject *PyFunction_GET_ANNOTATIONS(PyObject *op) + + These functions are similar to their ``PyFunction_Get*`` counterparts, but + do not do type checking. Passing anything other than an instance of + :c:data:`PyFunction_Type` is undefined behavior. + + .. c:function:: int PyFunction_AddWatcher(PyFunction_WatchCallback callback) Register *callback* as a function watcher for the current interpreter. @@ -145,15 +174,19 @@ There are a few functions specific to Python functions. .. c:type:: PyFunction_WatchEvent - Enumeration of possible function watcher events: - - ``PyFunction_EVENT_CREATE`` - - ``PyFunction_EVENT_DESTROY`` - - ``PyFunction_EVENT_MODIFY_CODE`` - - ``PyFunction_EVENT_MODIFY_DEFAULTS`` - - ``PyFunction_EVENT_MODIFY_KWDEFAULTS`` + Enumeration of possible function watcher events: + + - ``PyFunction_EVENT_CREATE`` + - ``PyFunction_EVENT_DESTROY`` + - ``PyFunction_EVENT_MODIFY_CODE`` + - ``PyFunction_EVENT_MODIFY_DEFAULTS`` + - ``PyFunction_EVENT_MODIFY_KWDEFAULTS`` .. versionadded:: 3.12 + - ``PyFunction_PYFUNC_EVENT_MODIFY_QUALNAME`` + + .. versionadded:: 3.15 .. c:type:: int (*PyFunction_WatchCallback)(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) @@ -168,7 +201,7 @@ There are a few functions specific to Python functions. unpredictable effects, including infinite recursion. If *event* is ``PyFunction_EVENT_CREATE``, then the callback is invoked - after `func` has been fully initialized. Otherwise, the callback is invoked + after *func* has been fully initialized. Otherwise, the callback is invoked before the modification to *func* takes place, so the prior state of *func* can be inspected. The runtime is permitted to optimize away the creation of function objects when possible. In such cases no event will be emitted. @@ -176,7 +209,7 @@ There are a few functions specific to Python functions. runtime behavior depending on optimization decisions, it does not change the semantics of the Python code being executed. - If *event* is ``PyFunction_EVENT_DESTROY``, Taking a reference in the + If *event* is ``PyFunction_EVENT_DESTROY``, taking a reference in the callback to the about-to-be-destroyed function will resurrect it, preventing it from being freed at this time. When the resurrected object is destroyed later, any watcher callbacks active at that time will be called again. diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 621da3eb069949..fed795b1e8c963 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -57,11 +57,49 @@ rules: Analogous to :c:macro:`PyObject_New` but for container objects with the :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + Memory allocated by this macro must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + + .. seealso:: + + * :c:func:`PyObject_GC_Del` + * :c:macro:`PyObject_New` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` + + .. c:macro:: PyObject_GC_NewVar(TYPE, typeobj, size) Analogous to :c:macro:`PyObject_NewVar` but for container objects with the :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + Memory allocated by this macro must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + + .. seealso:: + + * :c:func:`PyObject_GC_Del` + * :c:macro:`PyObject_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` + + .. c:function:: PyObject* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size) Analogous to :c:macro:`PyObject_GC_New` but allocates *extra_size* @@ -73,6 +111,10 @@ rules: The extra data will be deallocated with the object, but otherwise it is not managed by Python. + Memory allocated by this function must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + .. warning:: The function is marked as unstable because the final mechanism for reserving extra data after an instance is not yet decided. @@ -136,6 +178,21 @@ rules: Releases memory allocated to an object using :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. + Do not call this directly to free an object's memory; call the type's + :c:member:`~PyTypeObject.tp_free` slot instead. + + Do not use this for memory allocated by :c:macro:`PyObject_New`, + :c:macro:`PyObject_NewVar`, or related allocation functions; use + :c:func:`PyObject_Free` instead. + + .. seealso:: + + * :c:func:`PyObject_Free` is the non-GC equivalent of this function. + * :c:macro:`PyObject_GC_New` + * :c:macro:`PyObject_GC_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_free` + .. c:function:: void PyObject_GC_UnTrack(void *op) @@ -175,14 +232,18 @@ The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: object argument. If *visit* returns a non-zero value that value should be returned immediately. + The traversal function must not have any side effects. Implementations + may not modify the reference counts of any Python objects nor create or + destroy any Python objects. + To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation must name its arguments exactly *visit* and *arg*: -.. c:function:: void Py_VISIT(PyObject *o) +.. c:macro:: Py_VISIT(o) - If *o* is not ``NULL``, call the *visit* callback, with arguments *o* + If the :c:expr:`PyObject *` *o* is not ``NULL``, call the *visit* callback, with arguments *o* and *arg*. If *visit* returns a non-zero value, then return it. Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers look like:: @@ -277,7 +338,7 @@ the garbage collector. Type of the visitor function to be passed to :c:func:`PyUnstable_GC_VisitObjects`. *arg* is the same as the *arg* passed to ``PyUnstable_GC_VisitObjects``. - Return ``0`` to continue iteration, return ``1`` to stop iteration. Other return + Return ``1`` to continue iteration, return ``0`` to stop iteration. Other return values are reserved for now so behavior on returning anything else is undefined. .. versionadded:: 3.12 diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 0eb5922f6da75f..74db49a6814800 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -44,3 +44,53 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. A reference to *frame* is stolen by this function. The *frame* argument must not be ``NULL``. + + +.. c:function:: PyCodeObject* PyGen_GetCode(PyGenObject *gen) + + Return a new :term:`strong reference` to the code object wrapped by *gen*. + This function always succeeds. + + +Asynchronous Generator Objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. seealso:: + :pep:`525` + +.. c:var:: PyTypeObject PyAsyncGen_Type + + The type object corresponding to asynchronous generator objects. This is + available as :class:`types.AsyncGeneratorType` in the Python layer. + + .. versionadded:: 3.6 + +.. c:function:: PyObject *PyAsyncGen_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) + + Create a new asynchronous generator wrapping *frame*, with ``__name__`` and + ``__qualname__`` set to *name* and *qualname*. *frame* is stolen by this + function and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to the + new asynchronous generator. On failure, this function returns ``NULL`` + with an exception set. + + .. versionadded:: 3.6 + +.. c:function:: int PyAsyncGen_CheckExact(PyObject *op) + + Return true if *op* is an asynchronous generator object, false otherwise. + This function always succeeds. + + .. versionadded:: 3.6 + + +Deprecated API +^^^^^^^^^^^^^^ + +.. c:macro:: PyAsyncGenASend_CheckExact(op) + + This is a :term:`soft deprecated` API that was included in Python's C API + by mistake. + + It is solely here for completeness; do not use this API. diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index 00f8cb887dc7eb..1ad712b0ce4f2b 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -11,47 +11,103 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. versionadded:: 3.2 + .. c:type:: Py_uhash_t Hash value type: unsigned integer. .. versionadded:: 3.2 + +.. c:macro:: Py_HASH_ALGORITHM + + A numerical value indicating the algorithm for hashing of :class:`str`, + :class:`bytes`, and :class:`memoryview`. + + The algorithm name is exposed by :data:`sys.hash_info.algorithm`. + + .. versionadded:: 3.4 + + +.. c:macro:: Py_HASH_FNV + Py_HASH_SIPHASH24 + Py_HASH_SIPHASH13 + + Numerical values to compare to :c:macro:`Py_HASH_ALGORITHM` to determine + which algorithm is used for hashing. The hash algorithm can be configured + via the configure :option:`--with-hash-algorithm` option. + + .. versionadded:: 3.4 + Add :c:macro:`!Py_HASH_FNV` and :c:macro:`!Py_HASH_SIPHASH24`. + + .. versionadded:: 3.11 + Add :c:macro:`!Py_HASH_SIPHASH13`. + + +.. c:macro:: Py_HASH_CUTOFF + + Buffers of length in range ``[1, Py_HASH_CUTOFF)`` are hashed using DJBX33A + instead of the algorithm described by :c:macro:`Py_HASH_ALGORITHM`. + + - A :c:macro:`!Py_HASH_CUTOFF` of 0 disables the optimization. + - :c:macro:`!Py_HASH_CUTOFF` must be non-negative and less or equal than 7. + + 32-bit platforms should use a cutoff smaller than 64-bit platforms because + it is easier to create colliding strings. A cutoff of 7 on 64-bit platforms + and 5 on 32-bit platforms should provide a decent safety margin. + + This corresponds to the :data:`sys.hash_info.cutoff` constant. + + .. versionadded:: 3.4 + + .. c:macro:: PyHASH_MODULUS - The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. + The `Mersenne prime `_ ``P = 2**n -1``, + used for numeric hash scheme. + + This corresponds to the :data:`sys.hash_info.modulus` constant. .. versionadded:: 3.13 + .. c:macro:: PyHASH_BITS The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. .. versionadded:: 3.13 + .. c:macro:: PyHASH_MULTIPLIER Prime multiplier used in string and various other hashes. .. versionadded:: 3.13 + .. c:macro:: PyHASH_INF The hash value returned for a positive infinity. + This corresponds to the :data:`sys.hash_info.inf` constant. + .. versionadded:: 3.13 + .. c:macro:: PyHASH_IMAG The multiplier used for the imaginary part of a complex number. + This corresponds to the :data:`sys.hash_info.imag` constant. + .. versionadded:: 3.13 + .. c:type:: PyHash_FuncDef Hash function definition used by :c:func:`PyHash_GetFuncDef`. - .. c::member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) + .. c:member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) Hash function. @@ -59,14 +115,20 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. Hash function name (UTF-8 encoded string). + This corresponds to the :data:`sys.hash_info.algorithm` constant. + .. c:member:: const int hash_bits Internal size of the hash value in bits. + This corresponds to the :data:`sys.hash_info.hash_bits` constant. + .. c:member:: const int seed_bits Size of seed input in bits. + This corresponds to the :data:`sys.hash_info.seed_bits` constant. + .. versionadded:: 3.4 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 6e48644c8fef8b..a28c0713dd3b2f 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -16,19 +16,6 @@ Importing Modules This is a wrapper around :c:func:`PyImport_Import()` which takes a :c:expr:`const char *` as an argument instead of a :c:expr:`PyObject *`. -.. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name) - - This function is a deprecated alias of :c:func:`PyImport_ImportModule`. - - .. versionchanged:: 3.3 - This function used to fail immediately when the import lock was held - by another thread. In Python 3.3 though, the locking scheme switched - to per-module locks for most purposes, so this function's special - behaviour isn't needed anymore. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyImport_ImportModule` instead. - .. c:function:: PyObject* PyImport_ImportModuleEx(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist) @@ -142,8 +129,7 @@ Importing Modules of :class:`~importlib.machinery.SourceFileLoader` otherwise. The module's :attr:`~module.__file__` attribute will be set to the code - object's :attr:`~codeobject.co_filename`. If applicable, - :attr:`~module.__cached__` will also be set. + object's :attr:`~codeobject.co_filename`. This function will reload the module if it was already imported. See :c:func:`PyImport_ReloadModule` for the intended way to reload a module. @@ -155,10 +141,13 @@ Importing Modules :c:func:`PyImport_ExecCodeModuleWithPathnames`. .. versionchanged:: 3.12 - The setting of :attr:`~module.__cached__` and :attr:`~module.__loader__` + The setting of ``__cached__`` and :attr:`~module.__loader__` is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. + .. versionchanged:: 3.15 + ``__cached__`` is no longer set. + .. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) @@ -170,16 +159,19 @@ Importing Modules .. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname) - Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`~module.__cached__` - attribute of the module object is set to *cpathname* if it is - non-``NULL``. Of the three functions, this is the preferred one to use. + Like :c:func:`PyImport_ExecCodeModuleEx`, but the path to any compiled file + via *cpathname* is used appropriately when non-``NULL``. Of the three + functions, this is the preferred one to use. .. versionadded:: 3.3 .. versionchanged:: 3.12 - Setting :attr:`~module.__cached__` is deprecated. See + Setting ``__cached__`` is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. + .. versionchanged:: 3.15 + ``__cached__`` no longer set. + .. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, const char *pathname, const char *cpathname) @@ -325,3 +317,52 @@ Importing Modules If Python is initialized multiple times, :c:func:`PyImport_AppendInittab` or :c:func:`PyImport_ExtendInittab` must be called before each Python initialization. + + +.. c:var:: struct _inittab *PyImport_Inittab + + The table of built-in modules used by Python initialization. Do not use this directly; + use :c:func:`PyImport_AppendInittab` and :c:func:`PyImport_ExtendInittab` + instead. + + +.. c:function:: PyObject* PyImport_ImportModuleAttr(PyObject *mod_name, PyObject *attr_name) + + Import the module *mod_name* and get its attribute *attr_name*. + + Names must be Python :class:`str` objects. + + Helper function combining :c:func:`PyImport_Import` and + :c:func:`PyObject_GetAttr`. For example, it can raise :exc:`ImportError` if + the module is not found, and :exc:`AttributeError` if the attribute doesn't + exist. + + .. versionadded:: 3.14 + +.. c:function:: PyObject* PyImport_ImportModuleAttrString(const char *mod_name, const char *attr_name) + + Similar to :c:func:`PyImport_ImportModuleAttr`, but names are UTF-8 encoded + strings instead of Python :class:`str` objects. + + .. versionadded:: 3.14 + +.. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) + + This function is a building block that enables embedders to implement + the :py:meth:`~importlib.abc.Loader.create_module` step of custom + static extension importers (e.g. of statically-linked extensions). + + *spec* must be a :class:`~importlib.machinery.ModuleSpec` object. + + *initfunc* must be an :ref:`initialization function `, + the same as for :c:func:`PyImport_AppendInittab`. + + On success, create and return a module object. + This module will not be initialized; call :c:func:`PyModule_Exec` + to initialize it. + (Custom importers should do this in their + :py:meth:`~importlib.abc.Loader.exec_module` method.) + + On error, return NULL with an exception set. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index ba56b03c6ac8e7..e9df2a304d975b 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -17,6 +17,7 @@ document the API functions in detail. veryhigh.rst refcounting.rst exceptions.rst + extension-modules.rst utilities.rst abstract.rst concrete.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 6e881590131cab..7411644f9e110b 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -41,7 +41,6 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyObject_SetArenaAllocator` * :c:func:`Py_SetProgramName` * :c:func:`Py_SetPythonHome` - * :c:func:`PySys_ResetWarnOptions` * the configuration functions covered in :ref:`init-config` * Informative functions: @@ -77,10 +76,7 @@ The following functions can be safely called before Python is initialized: Despite their apparent similarity to some of the functions listed above, the following functions **should not be called** before the interpreter has - been initialized: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, - :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, - :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, - :c:func:`Py_GetProgramName`, :c:func:`PyEval_InitThreads`, and + been initialized: :c:func:`Py_EncodeLocale`, :c:func:`PyEval_InitThreads`, and :c:func:`Py_RunMain`. @@ -109,7 +105,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-b` option. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_DebugFlag @@ -123,7 +119,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_DontWriteBytecodeFlag @@ -137,7 +133,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_FrozenFlag @@ -145,12 +141,9 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. :c:member:`PyConfig.pathconfig_warnings` should be used instead, see :ref:`Python Initialization Configuration `. - Suppress error messages when calculating the module search path in - :c:func:`Py_GetPath`. - Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_HashRandomizationFlag @@ -165,7 +158,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment variable to initialize the secret hash seed. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_IgnoreEnvironmentFlag @@ -178,7 +171,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-E` and :option:`-I` options. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_InspectFlag @@ -193,7 +186,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_InteractiveFlag @@ -203,7 +196,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_IsolatedFlag @@ -218,7 +211,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.4 - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_LegacyWindowsFSEncodingFlag @@ -237,7 +230,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_LegacyWindowsStdioFlag @@ -255,7 +248,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_NoSiteFlag @@ -270,7 +263,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-S` option. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_NoUserSiteDirectory @@ -284,7 +277,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-s` and :option:`-I` options, and the :envvar:`PYTHONNOUSERSITE` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_OptimizeFlag @@ -295,7 +288,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_QuietFlag @@ -309,7 +302,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.2 - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_UnbufferedStdioFlag @@ -322,7 +315,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_VerboseFlag @@ -338,7 +331,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 Initializing and finalizing the interpreter @@ -498,17 +491,8 @@ Initializing and finalizing the interpreter strings other than those passed in (however, the contents of the strings pointed to by the argument list are not modified). - The return value will be ``0`` if the interpreter exits normally (i.e., - without an exception), ``1`` if the interpreter exits due to an exception, - or ``2`` if the argument list does not represent a valid Python command - line. - - Note that if an otherwise unhandled :exc:`SystemExit` is raised, this - function will not return ``1``, but exit the process, as long as - ``Py_InspectFlag`` is not set. If ``Py_InspectFlag`` is set, execution will - drop into the interactive Python prompt, at which point a second otherwise - unhandled :exc:`SystemExit` will still exit the process, while any other - means of exiting will set the return value as described above. + The return value is ``2`` if the argument list does not represent a valid + Python command line, and otherwise the same as :c:func:`Py_RunMain`. In terms of the CPython runtime configuration APIs documented in the :ref:`runtime configuration ` section (and without accounting @@ -545,28 +529,32 @@ Initializing and finalizing the interpreter If :c:member:`PyConfig.inspect` is not set (the default), the return value will be ``0`` if the interpreter exits normally (that is, without raising - an exception), or ``1`` if the interpreter exits due to an exception. If an - otherwise unhandled :exc:`SystemExit` is raised, the function will immediately - exit the process instead of returning ``1``. + an exception), the exit status of an unhandled :exc:`SystemExit`, or ``1`` + for any other unhandled exception. If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option is used), rather than returning when the interpreter exits, execution will instead resume in an interactive Python prompt (REPL) using the ``__main__`` module's global namespace. If the interpreter exited with an exception, it is immediately raised in the REPL session. The function return value is - then determined by the way the *REPL session* terminates: returning ``0`` - if the session terminates without raising an unhandled exception, exiting - immediately for an unhandled :exc:`SystemExit`, and returning ``1`` for - any other unhandled exception. + then determined by the way the *REPL session* terminates: ``0``, ``1``, or + the status of a :exc:`SystemExit`, as specified above. - This function always finalizes the Python interpreter regardless of whether - it returns a value or immediately exits the process due to an unhandled - :exc:`SystemExit` exception. + This function always finalizes the Python interpreter before it returns. See :ref:`Python Configuration ` for an example of a customized Python that always runs in isolated mode using :c:func:`Py_RunMain`. +.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) + + Register an :mod:`atexit` callback for the target interpreter *interp*. + This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and + data pointer for the callback. + + There must be an :term:`attached thread state` for *interp*. + + .. versionadded:: 3.13 Process-wide parameters ======================= @@ -577,7 +565,6 @@ Process-wide parameters .. index:: single: Py_Initialize() single: main() - single: Py_GetPath() This API is kept for backward compatibility: setting :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python @@ -587,7 +574,7 @@ Process-wide parameters the first time, if it is called at all. It tells the interpreter the value of the ``argv[0]`` argument to the :c:func:`main` function of the program (converted to wide characters). - This is used by :c:func:`Py_GetPath` and some other functions below to find + This is used by some other functions below to find the Python run-time libraries relative to the interpreter executable. The default value is ``'python'``. The argument should point to a zero-terminated wide character string in static storage whose contents will not @@ -595,143 +582,9 @@ Process-wide parameters interpreter will change the contents of this storage. Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. - - .. deprecated:: 3.11 - - -.. c:function:: wchar_t* Py_GetProgramName() - - Return the program name set with :c:member:`PyConfig.program_name`, or the default. - The returned string points into static storage; the caller should not modify its - value. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.executable` instead. - - -.. c:function:: wchar_t* Py_GetPrefix() - - Return the *prefix* for installed platform-independent files. This is derived - through a number of complicated rules from the program name set with - :c:member:`PyConfig.program_name` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The - returned string points into static storage; the caller should not modify its - value. This corresponds to the :makevar:`prefix` variable in the top-level - :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure` - script at build time. The value is available to Python code as ``sys.base_prefix``. - It is only useful on Unix. See also the next function. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.base_prefix` instead, or :data:`sys.prefix` if - :ref:`virtual environments ` need to be handled. - - -.. c:function:: wchar_t* Py_GetExecPrefix() - - Return the *exec-prefix* for installed platform-*dependent* files. This is - derived through a number of complicated rules from the program name set with - :c:member:`PyConfig.program_name` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the exec-prefix is - ``'/usr/local'``. The returned string points into static storage; the caller - should not modify its value. This corresponds to the :makevar:`exec_prefix` - variable in the top-level :file:`Makefile` and the ``--exec-prefix`` - argument to the :program:`configure` script at build time. The value is - available to Python code as ``sys.base_exec_prefix``. It is only useful on - Unix. - - Background: The exec-prefix differs from the prefix when platform dependent - files (such as executables and shared libraries) are installed in a different - directory tree. In a typical installation, platform dependent files may be - installed in the :file:`/usr/local/plat` subtree while platform independent may - be installed in :file:`/usr/local`. - - Generally speaking, a platform is a combination of hardware and software - families, e.g. Sparc machines running the Solaris 2.x operating system are - considered the same platform, but Intel machines running Solaris 2.x are another - platform, and Intel machines running Linux are yet another platform. Different - major revisions of the same operating system generally also form different - platforms. Non-Unix operating systems are a different story; the installation - strategies on those systems are so different that the prefix and exec-prefix are - meaningless, and set to the empty string. Note that compiled Python bytecode - files are platform independent (but not independent from the Python version by - which they were compiled!). - - System administrators will know how to configure the :program:`mount` or - :program:`automount` programs to share :file:`/usr/local` between platforms - while having :file:`/usr/local/plat` be a different filesystem for each - platform. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.base_exec_prefix` instead, or :data:`sys.exec_prefix` if - :ref:`virtual environments ` need to be handled. - - -.. c:function:: wchar_t* Py_GetProgramFullPath() + :c:expr:`wchar_t*` string. - .. index:: - single: executable (in module sys) - - Return the full program name of the Python executable; this is computed as a - side-effect of deriving the default module search path from the program name - (set by :c:member:`PyConfig.program_name`). The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.executable``. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.executable` instead. - - -.. c:function:: wchar_t* Py_GetPath() - - .. index:: - triple: module; search; path - single: path (in module sys) - - Return the default module search path; this is computed from the program name - (set by :c:member:`PyConfig.program_name`) and some environment variables. - The returned string consists of a series of directory names separated by a - platform dependent delimiter character. The delimiter character is ``':'`` - on Unix and macOS, ``';'`` on Windows. The returned string points into - static storage; the caller should not modify its value. The list - :data:`sys.path` is initialized with this value on interpreter startup; it - can be (and usually is) modified later to change the search path for loading - modules. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. XXX should give the exact rules - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.path` instead. + .. deprecated-removed:: 3.11 3.15 .. c:function:: const char* Py_GetVersion() @@ -837,7 +690,7 @@ Process-wide parameters directory (``"."``). Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. + :c:expr:`wchar_t*` string. See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` members of the :ref:`Python Initialization Configuration `. @@ -859,7 +712,7 @@ Process-wide parameters .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; check w/ Guido. - .. deprecated:: 3.11 + .. deprecated-removed:: 3.11 3.15 .. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) @@ -873,14 +726,14 @@ Process-wide parameters :option:`-I`. Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. + :c:expr:`wchar_t*` string. See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` members of the :ref:`Python Initialization Configuration `. .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. - .. deprecated:: 3.11 + .. deprecated-removed:: 3.11 3.15 .. c:function:: void Py_SetPythonHome(const wchar_t *home) @@ -899,26 +752,9 @@ Process-wide parameters this storage. Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. - - .. deprecated:: 3.11 - - -.. c:function:: wchar_t* Py_GetPythonHome() - - Return the default "home", that is, the value set by - :c:member:`PyConfig.home`, or the value of the :envvar:`PYTHONHOME` - environment variable if it is set. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + :c:expr:`wchar_t*` string. - .. deprecated-removed:: 3.13 3.15 - Get :c:member:`PyConfig.home` or :envvar:`PYTHONHOME` environment - variable instead. + .. deprecated-removed:: 3.11 3.15 .. _threads: @@ -931,7 +767,8 @@ Thread State and the Global Interpreter Lock single: interpreter lock single: lock, interpreter -The Python interpreter is not fully thread-safe. In order to support +Unless on a :term:`free-threaded ` build of :term:`CPython`, +the Python interpreter is not fully thread-safe. In order to support multi-threaded Python programs, there's a global lock, called the :term:`global interpreter lock` or :term:`GIL`, that must be held by the current thread before it can safely access Python objects. Without the lock, even the simplest @@ -952,20 +789,30 @@ a file, so that other Python threads can run in the meantime. single: PyThreadState (C type) The Python interpreter keeps some thread-specific bookkeeping information -inside a data structure called :c:type:`PyThreadState`. There's also one -global variable pointing to the current :c:type:`PyThreadState`: it can -be retrieved using :c:func:`PyThreadState_Get`. - -Releasing the GIL from extension code -------------------------------------- - -Most extension code manipulating the :term:`GIL` has the following simple +inside a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. +Each OS thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state +referenced by this pointer is considered to be :term:`attached `. + +A thread can only have one :term:`attached thread state` at a time. An attached +thread state is typically analogous with holding the :term:`GIL`, except on +:term:`free-threaded ` builds. On builds with the :term:`GIL` enabled, +:term:`attaching ` a thread state will block until the :term:`GIL` +can be acquired. However, even on builds with the :term:`GIL` disabled, it is still required +to have an attached thread state to call most of the C API. + +In general, there will always be an :term:`attached thread state` when using Python's C API. +Only in some specific cases (such as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block) will the +thread not have an attached thread state. If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns +``NULL``. + +Detaching the thread state from extension code +---------------------------------------------- + +Most extension code manipulating the :term:`thread state` has the following simple structure:: Save the thread state in a local variable. - Release the global interpreter lock. ... Do some blocking I/O operation ... - Reacquire the global interpreter lock. Restore the thread state from the local variable. This is so common that a pair of macros exists to simplify it:: @@ -994,21 +841,30 @@ The block above expands to the following code:: single: PyEval_RestoreThread (C function) single: PyEval_SaveThread (C function) -Here is how these functions work: the global interpreter lock is used to protect the pointer to the -current thread state. When releasing the lock and saving the thread state, -the current thread state pointer must be retrieved before the lock is released -(since another thread could immediately acquire the lock and store its own thread -state in the global variable). Conversely, when acquiring the lock and restoring -the thread state, the lock must be acquired before storing the thread state -pointer. +Here is how these functions work: + +The :term:`attached thread state` holds the :term:`GIL` for the entire interpreter. When detaching +the :term:`attached thread state`, the :term:`GIL` is released, allowing other threads to attach +a thread state to their own thread, thus getting the :term:`GIL` and can start executing. +The pointer to the prior :term:`attached thread state` is stored as a local variable. +Upon reaching :c:macro:`Py_END_ALLOW_THREADS`, the thread state that was +previously :term:`attached ` is passed to :c:func:`PyEval_RestoreThread`. +This function will block until another releases its :term:`thread state `, +thus allowing the old :term:`thread state ` to get re-attached and the +C API can be called again. + +For :term:`free-threaded ` builds, the :term:`GIL` is normally +out of the question, but detaching the :term:`thread state ` is still required +for blocking I/O and long operations. The difference is that threads don't have to wait for the :term:`GIL` +to be released to attach their thread state, allowing true multi-core parallelism. .. note:: - Calling system I/O functions is the most common use case for releasing - the GIL, but it can also be useful before calling long-running computations - which don't need access to Python objects, such as compression or - cryptographic functions operating over memory buffers. For example, the - standard :mod:`zlib` and :mod:`hashlib` modules release the GIL when - compressing or hashing data. + Calling system I/O functions is the most common use case for detaching + the :term:`thread state `, but it can also be useful before calling + long-running computations which don't need access to Python objects, such + as compression or cryptographic functions operating over memory buffers. + For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the + :term:`thread state ` when compressing or hashing data. .. _gilstate: @@ -1020,16 +876,15 @@ When threads are created using the dedicated Python APIs (such as the :mod:`threading` module), a thread state is automatically associated to them and the code showed above is therefore correct. However, when threads are created from C (for example by a third-party library with its own thread -management), they don't hold the GIL, nor is there a thread state structure -for them. +management), they don't hold the :term:`GIL`, because they don't have an +:term:`attached thread state`. If you need to call Python code from these threads (often this will be part of a callback API provided by the aforementioned third-party library), you must first register these threads with the interpreter by -creating a thread state data structure, then acquiring the GIL, and finally -storing their thread state pointer, before you can start using the Python/C -API. When you are done, you should reset the thread state pointer, release -the GIL, and finally free the thread state data structure. +creating an :term:`attached thread state` before you can start using the Python/C +API. When you are done, you should detach the :term:`thread state `, and +finally free it. The :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` functions do all of the above automatically. The typical idiom for calling into Python @@ -1049,8 +904,36 @@ Note that the ``PyGILState_*`` functions assume there is only one global interpreter (created automatically by :c:func:`Py_Initialize`). Python supports the creation of additional interpreters (using :c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the -``PyGILState_*`` API is unsupported. +``PyGILState_*`` API is unsupported. This is because :c:func:`PyGILState_Ensure` +and similar functions default to :term:`attaching ` a +:term:`thread state` for the main interpreter, meaning that the thread can't safely +interact with the calling subinterpreter. + +Supporting subinterpreters in non-Python threads +------------------------------------------------ + +If you would like to support subinterpreters with non-Python created threads, you +must use the ``PyThreadState_*`` API instead of the traditional ``PyGILState_*`` +API. + +In particular, you must store the interpreter state from the calling +function and pass it to :c:func:`PyThreadState_New`, which will ensure that +the :term:`thread state` is targeting the correct interpreter:: + + /* The return value of PyInterpreterState_Get() from the + function that created this thread. */ + PyInterpreterState *interp = ThreadData->interp; + PyThreadState *tstate = PyThreadState_New(interp); + PyThreadState_Swap(tstate); + + /* GIL of the subinterpreter is now held. + Perform Python actions here. */ + result = CallSomeFunction(); + /* evaluate result or handle exception */ + /* Destroy the thread state. No Python API allowed beyond this point. */ + PyThreadState_Clear(tstate); + PyThreadState_DeleteCurrent(); .. _fork-and-threads: @@ -1097,26 +980,23 @@ Cautions regarding runtime finalization In the late stage of :term:`interpreter shutdown`, after attempting to wait for non-daemon threads to exit (though this can be interrupted by :class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime -is marked as *finalizing*: :c:func:`_Py_IsFinalizing` and +is marked as *finalizing*: :c:func:`Py_IsFinalizing` and :func:`sys.is_finalizing` return true. At this point, only the *finalization thread* that initiated finalization (typically the main thread) is allowed to acquire the :term:`GIL`. -If any thread, other than the finalization thread, attempts to acquire the GIL -during finalization, either explicitly via :c:func:`PyGILState_Ensure`, -:c:macro:`Py_END_ALLOW_THREADS`, :c:func:`PyEval_AcquireThread`, or -:c:func:`PyEval_AcquireLock`, or implicitly when the interpreter attempts to -reacquire it after having yielded it, the thread enters **a permanently blocked -state** where it remains until the program exits. In most cases this is -harmless, but this can result in deadlock if a later stage of finalization -attempts to acquire a lock owned by the blocked thread, or otherwise waits on -the blocked thread. +If any thread, other than the finalization thread, attempts to attach a :term:`thread state` +during finalization, either explicitly or +implicitly, the thread enters **a permanently blocked state** +where it remains until the program exits. In most cases this is harmless, but this can result +in deadlock if a later stage of finalization attempts to acquire a lock owned by the +blocked thread, or otherwise waits on the blocked thread. Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ finalizations further up the call stack when such threads were forcibly exited -here in CPython 3.13 and earlier. The CPython runtime GIL acquiring C APIs -have never had any error reporting or handling expectations at GIL acquisition -time that would've allowed for graceful exit from this situation. Changing that +here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs +have never had any error reporting or handling expectations at :term:`thread state` +attachment time that would've allowed for graceful exit from this situation. Changing that would require new stable C APIs and rewriting the majority of C code in the CPython ecosystem to use those with error handling. @@ -1139,6 +1019,12 @@ code, or when embedding the Python interpreter: interpreter lock is also shared by all threads, regardless of to which interpreter they belong. + .. versionchanged:: 3.12 + + :pep:`684` introduced the possibility + of a :ref:`per-interpreter GIL `. + See :c:func:`Py_NewInterpreterFromConfig`. + .. c:type:: PyThreadState @@ -1179,18 +1065,15 @@ code, or when embedding the Python interpreter: .. c:function:: PyThreadState* PyEval_SaveThread() - Release the global interpreter lock (if it has been created) and reset the - thread state to ``NULL``, returning the previous thread state (which is not - ``NULL``). If the lock has been created, the current thread must have - acquired it. + Detach the :term:`attached thread state` and return it. + The thread will have no :term:`thread state` upon returning. .. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) - Acquire the global interpreter lock (if it has been created) and set the - thread state to *tstate*, which must not be ``NULL``. If the lock has been - created, the current thread must not have acquired it, otherwise deadlock - ensues. + Set the :term:`attached thread state` to *tstate*. + The passed :term:`thread state` **should not** be :term:`attached `, + otherwise deadlock ensues. *tstate* will be attached upon returning. .. note:: Calling this function from a thread when the runtime is finalizing will @@ -1204,13 +1087,13 @@ code, or when embedding the Python interpreter: .. c:function:: PyThreadState* PyThreadState_Get() - Return the current thread state. The global interpreter lock must be held. - When the current thread state is ``NULL``, this issues a fatal error (so that - the caller needn't check for ``NULL``). + Return the :term:`attached thread state`. If the thread has no attached + thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` + block), then this issues a fatal error (so that the caller needn't check + for ``NULL``). See also :c:func:`PyThreadState_GetUnchecked`. - .. c:function:: PyThreadState* PyThreadState_GetUnchecked() Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a @@ -1224,18 +1107,40 @@ code, or when embedding the Python interpreter: .. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) - Swap the current thread state with the thread state given by the argument - *tstate*, which may be ``NULL``. The global interpreter lock must be held - and is not released. + Set the :term:`attached thread state` to *tstate*, and return the + :term:`thread state` that was attached prior to calling. + + This function is safe to call without an :term:`attached thread state`; it + will simply return ``NULL`` indicating that there was no prior thread state. + + .. seealso:: + :c:func:`PyEval_ReleaseThread` + + .. note:: + Similar to :c:func:`PyGILState_Ensure`, this function will hang the + thread if the runtime is finalizing. The following functions use thread-local storage, and are not compatible with sub-interpreters: +.. c:type:: PyGILState_STATE + + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED + + The GIL was already held when :c:func:`PyGILState_Ensure` was called. + + .. c:enumerator:: PyGILState_UNLOCKED + + The GIL was not held when :c:func:`PyGILState_Ensure` was called. + .. c:function:: PyGILState_STATE PyGILState_Ensure() Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the global interpreter lock. This may + of the current state of Python, or of the :term:`attached thread state`. This may be called as many times as desired by a thread as long as each call is matched with a call to :c:func:`PyGILState_Release`. In general, other thread-related APIs may be used between :c:func:`PyGILState_Ensure` and @@ -1244,20 +1149,20 @@ with sub-interpreters: :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is acceptable. - The return value is an opaque "handle" to the thread state when + The return value is an opaque "handle" to the :term:`attached thread state` when :c:func:`PyGILState_Ensure` was called, and must be passed to :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even though recursive calls are allowed, these handles *cannot* be shared - each unique call to :c:func:`PyGILState_Ensure` must save the handle for its call to :c:func:`PyGILState_Release`. - When the function returns, the current thread will hold the GIL and be able - to call arbitrary Python code. Failure is a fatal error. + When the function returns, there will be an :term:`attached thread state` + and the thread will be able to call arbitrary Python code. Failure is a fatal error. - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to + .. warning:: + Calling this function when the runtime is finalizing is unsafe. Doing + so will either hang the thread until the program ends, or fully crash + the interpreter in rare cases. Refer to :ref:`cautions-regarding-runtime-finalization` for more details. .. versionchanged:: 3.14 @@ -1274,26 +1179,37 @@ with sub-interpreters: Every call to :c:func:`PyGILState_Ensure` must be matched by a call to :c:func:`PyGILState_Release` on the same thread. - .. c:function:: PyThreadState* PyGILState_GetThisThreadState() - Get the current thread state for this thread. May return ``NULL`` if no + Get the :term:`attached thread state` for this thread. May return ``NULL`` if no GILState API has been used on the current thread. Note that the main thread always has such a thread-state, even if no auto-thread-state call has been made on the main thread. This is mainly a helper/diagnostic function. + .. note:: + This function may return non-``NULL`` even when the :term:`thread state` + is detached. + Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + for most cases. + + .. seealso:: :c:func:`PyThreadState_Get` .. c:function:: int PyGILState_Check() - Return ``1`` if the current thread is holding the GIL and ``0`` otherwise. + Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. This function can be called from any thread at any time. - Only if it has had its Python thread state initialized and currently is - holding the GIL will it return ``1``. + Only if it has had its :term:`thread state ` initialized + via :c:func:`PyGILState_Ensure` will it return ``1``. This is mainly a helper/diagnostic function. It can be useful for example in callback contexts or memory allocation functions when - knowing that the GIL is locked can allow the caller to perform sensitive + knowing that the :term:`GIL` is locked can allow the caller to perform sensitive actions or otherwise behave differently. + .. note:: + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` + for most cases. + .. versionadded:: 3.4 @@ -1336,13 +1252,14 @@ Low-level API All of the following functions must be called after :c:func:`Py_Initialize`. .. versionchanged:: 3.7 - :c:func:`Py_Initialize()` now initializes the :term:`GIL`. + :c:func:`Py_Initialize()` now initializes the :term:`GIL` + and sets an :term:`attached thread state`. .. c:function:: PyInterpreterState* PyInterpreterState_New() - Create a new interpreter state object. The global interpreter lock need not - be held, but may be held if it is necessary to serialize calls to this + Create a new interpreter state object. An :term:`attached thread state` is not needed, + but may optionally exist if it is necessary to serialize calls to this function. .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New @@ -1350,50 +1267,52 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) - Reset all information in an interpreter state object. The global interpreter - lock must be held. + Reset all information in an interpreter state object. There must be + an :term:`attached thread state` for the interpreter. .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear .. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) - Destroy an interpreter state object. The global interpreter lock need not be - held. The interpreter state must have been reset with a previous call to - :c:func:`PyInterpreterState_Clear`. + Destroy an interpreter state object. There **should not** be an + :term:`attached thread state` for the target interpreter. The interpreter + state must have been reset with a previous call to :c:func:`PyInterpreterState_Clear`. .. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) Create a new thread state object belonging to the given interpreter object. - The global interpreter lock need not be held, but may be held if it is - necessary to serialize calls to this function. - + An :term:`attached thread state` is not needed. .. c:function:: void PyThreadState_Clear(PyThreadState *tstate) - Reset all information in a thread state object. The global interpreter lock - must be held. + Reset all information in a :term:`thread state` object. *tstate* + must be :term:`attached ` .. versionchanged:: 3.9 - This function now calls the :c:member:`PyThreadState.on_delete` callback. + This function now calls the :c:member:`!PyThreadState.on_delete` callback. Previously, that happened in :c:func:`PyThreadState_Delete`. + .. versionchanged:: 3.13 + The :c:member:`!PyThreadState.on_delete` callback was removed. + .. c:function:: void PyThreadState_Delete(PyThreadState *tstate) - Destroy a thread state object. The global interpreter lock need not be held. - The thread state must have been reset with a previous call to + Destroy a :term:`thread state` object. *tstate* should not + be :term:`attached ` to any thread. + *tstate* must have been reset with a previous call to :c:func:`PyThreadState_Clear`. .. c:function:: void PyThreadState_DeleteCurrent(void) - Destroy the current thread state and release the global interpreter lock. - Like :c:func:`PyThreadState_Delete`, the global interpreter lock must - be held. The thread state must have been reset with a previous call - to :c:func:`PyThreadState_Clear`. + Detach the :term:`attached thread state` (which must have been reset + with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. + No :term:`thread state` will be :term:`attached ` upon + returning. .. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) @@ -1404,16 +1323,16 @@ All of the following functions must be called after :c:func:`Py_Initialize`. See also :c:func:`PyEval_GetFrame`. - *tstate* must not be ``NULL``. + *tstate* must not be ``NULL``, and must be :term:`attached `. .. versionadded:: 3.9 .. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) - Get the unique thread state identifier of the Python thread state *tstate*. + Get the unique :term:`thread state` identifier of the Python thread state *tstate*. - *tstate* must not be ``NULL``. + *tstate* must not be ``NULL``, and must be :term:`attached `. .. versionadded:: 3.9 @@ -1422,7 +1341,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Get the interpreter of the Python thread state *tstate*. - *tstate* must not be ``NULL``. + *tstate* must not be ``NULL``, and must be :term:`attached `. .. versionadded:: 3.9 @@ -1447,14 +1366,49 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.11 +.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) + + Set the stack protection start address and stack protection size + of a Python thread state. + + On success, return ``0``. + On failure, set an exception and return ``-1``. + + CPython implements :ref:`recursion control ` for C code by raising + :py:exc:`RecursionError` when it notices that the machine execution stack is close + to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. + For this, it needs to know the location of the current thread's stack, which it + normally gets from the operating system. + When the stack is changed, for example using context switching techniques like the + Boost library's ``boost::context``, you must call + :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. + + Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before + or after changing the stack. + Do not call any other Python C API between the call and the stack + change. + + See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. + + .. versionadded:: 3.15 + + +.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) + + Reset the stack protection start address and stack protection size + of a Python thread state to the operating system defaults. + + See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. + + .. versionadded:: 3.15 + + .. c:function:: PyInterpreterState* PyInterpreterState_Get(void) Get the current interpreter. - Issue a fatal error if there no current Python thread state or no current - interpreter. It cannot return NULL. - - The caller must hold the GIL. + Issue a fatal error if there no :term:`attached thread state`. + It cannot return NULL. .. versionadded:: 3.9 @@ -1464,7 +1418,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Return the interpreter's unique ID. If there was any error in doing so then ``-1`` is returned and an error is set. - The caller must hold the GIL. + The caller must have an :term:`attached thread state`. .. versionadded:: 3.7 @@ -1478,8 +1432,12 @@ All of the following functions must be called after :c:func:`Py_Initialize`. This is not a replacement for :c:func:`PyModule_GetState()`, which extensions should use to store interpreter-specific state information. + The returned dictionary is borrowed from the interpreter and is valid until + interpreter shutdown. + .. versionadded:: 3.8 + .. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) Type of a frame evaluation function. @@ -1514,9 +1472,10 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Return a dictionary in which extensions can store thread-specific state information. Each extension should use a unique key to use to store state in - the dictionary. It is okay to call this function when no current thread state - is available. If this function returns ``NULL``, no exception has been raised and - the caller should assume no current thread state is available. + the dictionary. It is okay to call this function when no :term:`thread state` + is :term:`attached `. If this function returns + ``NULL``, no exception has been raised and the caller should assume no + thread state is attached. .. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) @@ -1524,7 +1483,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Asynchronously raise an exception in a thread. The *id* argument is the thread id of the target thread; *exc* is the exception object to be raised. This function does not steal any references to *exc*. To prevent naive misuse, you - must write your own C extension to call this. Must be called with the GIL held. + must write your own C extension to call this. Must be called with an :term:`attached thread state`. Returns the number of thread states modified; this is normally one, but will be zero if the thread id isn't found. If *exc* is ``NULL``, the pending exception (if any) for the thread is cleared. This raises no exceptions. @@ -1535,9 +1494,10 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) - Acquire the global interpreter lock and set the current thread state to - *tstate*, which must not be ``NULL``. The lock must have been created earlier. - If this thread already has the lock, deadlock ensues. + :term:`Attach ` *tstate* to the current thread, + which must not be ``NULL`` or already :term:`attached `. + + The calling thread must not already have an :term:`attached thread state`. .. note:: Calling this function from a thread when the runtime is finalizing will @@ -1560,10 +1520,9 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - Reset the current thread state to ``NULL`` and release the global interpreter - lock. The lock must have been created earlier and must be held by the current - thread. The *tstate* argument, which must not be ``NULL``, is only used to check - that it represents the current thread state --- if it isn't, a fatal error is + Detach the :term:`attached thread state`. + The *tstate* argument, which must not be ``NULL``, is only used to check + that it represents the :term:`attached thread state` --- if it isn't, a fatal error is reported. :c:func:`PyEval_SaveThread` is a higher-level function which is always @@ -1703,23 +1662,23 @@ function. You can create and destroy them using the following functions: The given *config* controls the options with which the interpreter is initialized. - Upon success, *tstate_p* will be set to the first thread state - created in the new - sub-interpreter. This thread state is made in the current thread state. + Upon success, *tstate_p* will be set to the first :term:`thread state` + created in the new sub-interpreter. This thread state is + :term:`attached `. Note that no actual thread is created; see the discussion of thread states below. If creation of the new interpreter is unsuccessful, *tstate_p* is set to ``NULL``; no exception is set since the exception state is stored in the - current thread state and there may not be a current thread state. + :term:`attached thread state`, which might not exist. - Like all other Python/C API functions, the global interpreter lock - must be held before calling this function and is still held when it - returns. Likewise a current thread state must be set on entry. On - success, the returned thread state will be set as current. If the - sub-interpreter is created with its own GIL then the GIL of the - calling interpreter will be released. When the function returns, - the new interpreter's GIL will be held by the current thread and - the previously interpreter's GIL will remain released here. + Like all other Python/C API functions, an :term:`attached thread state` + must be present before calling this function, but it might be detached upon + returning. On success, the returned thread state will be :term:`attached `. + If the sub-interpreter is created with its own :term:`GIL` then the + :term:`attached thread state` of the calling interpreter will be detached. + When the function returns, the new interpreter's :term:`thread state` + will be :term:`attached ` to the current thread and + the previous interpreter's :term:`attached thread state` will remain detached. .. versionadded:: 3.12 @@ -1735,7 +1694,11 @@ function. You can create and destroy them using the following functions: .check_multi_interp_extensions = 1, .gil = PyInterpreterConfig_OWN_GIL, }; - PyThreadState *tstate = Py_NewInterpreterFromConfig(&config); + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } Note that the config is used only briefly and does not get modified. During initialization the config's values are converted into various @@ -1754,7 +1717,8 @@ function. You can create and destroy them using the following functions: Only C-level static and global variables are shared between these module objects. - * For modules using single-phase initialization, + * For modules using legacy + :ref:`single-phase initialization `, e.g. :c:func:`PyModule_Create`, the first time a particular extension is imported, it is initialized normally, and a (shallow) copy of its module's dictionary is squirreled away. @@ -1797,18 +1761,17 @@ function. You can create and destroy them using the following functions: .. index:: single: Py_FinalizeEx (C function) - Destroy the (sub-)interpreter represented by the given thread state. - The given thread state must be the current thread state. See the - discussion of thread states below. When the call returns, - the current thread state is ``NULL``. All thread states associated - with this interpreter are destroyed. The global interpreter lock - used by the target interpreter must be held before calling this - function. No GIL is held when it returns. + Destroy the (sub-)interpreter represented by the given :term:`thread state`. + The given thread state must be :term:`attached `. + When the call returns, there will be no :term:`attached thread state`. + All thread states associated with this interpreter are destroyed. :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that haven't been explicitly destroyed at that point. +.. _per-interpreter-gil: + A Per-Interpreter GIL --------------------- @@ -1820,7 +1783,7 @@ being blocked by other interpreters or blocking any others. Thus a single Python process can truly take advantage of multiple CPU cores when running Python code. The isolation also encourages a different approach to concurrency than that of just using threads. -(See :pep:`554`.) +(See :pep:`554` and :pep:`684`.) Using an isolated interpreter requires vigilance in preserving that isolation. That especially means not sharing any objects or mutable @@ -1897,20 +1860,17 @@ pointer and a void pointer argument. both these conditions met: * on a :term:`bytecode` boundary; - * with the main thread holding the :term:`global interpreter lock` + * with the main thread holding an :term:`attached thread state` (*func* can therefore use the full C API). *func* must return ``0`` on success, or ``-1`` on failure with an exception set. *func* won't be interrupted to perform another asynchronous notification recursively, but it can still be interrupted to switch - threads if the global interpreter lock is released. + threads if the :term:`thread state ` is detached. - This function doesn't need a current thread state to run, and it doesn't - need the global interpreter lock. - - To call this function in a subinterpreter, the caller must hold the GIL. - Otherwise, the function *func* can be scheduled to be called from the wrong - interpreter. + This function doesn't need an :term:`attached thread state`. However, to call this + function in a subinterpreter, the caller must have an :term:`attached thread state`. + Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. .. warning:: This is a low-level function, only useful for very special cases. @@ -1928,6 +1888,29 @@ pointer and a void pointer argument. called from the main interpreter. Each subinterpreter now has its own list of scheduled calls. + .. versionchanged:: 3.12 + This function now always schedules *func* to be run in the main + interpreter. + + +.. c:function:: int Py_MakePendingCalls(void) + + Execute all pending calls. This is usually executed automatically by the + interpreter. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + If this is not called in the main thread of the main + interpreter, this function does nothing and returns ``0``. + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.12 + This function only runs pending calls in the main interpreter. + + .. _profiling: Profiling and Tracing @@ -2051,14 +2034,14 @@ Python-level trace functions in previous versions. See also the :func:`sys.setprofile` function. - The caller must hold the :term:`GIL`. + The caller must have an :term:`attached thread state`. .. c:function:: void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *obj) Like :c:func:`PyEval_SetProfile` but sets the profile function in all running threads belonging to the current interpreter instead of the setting it only on the current thread. - The caller must hold the :term:`GIL`. + The caller must have an :term:`attached thread state`. As :c:func:`PyEval_SetProfile`, this function ignores any exceptions raised while setting the profile functions in all threads. @@ -2077,14 +2060,14 @@ Python-level trace functions in previous versions. See also the :func:`sys.settrace` function. - The caller must hold the :term:`GIL`. + The caller must have an :term:`attached thread state`. .. c:function:: void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *obj) Like :c:func:`PyEval_SetTrace` but sets the tracing function in all running threads belonging to the current interpreter instead of the setting it only on the current thread. - The caller must hold the :term:`GIL`. + The caller must have an :term:`attached thread state`. As :c:func:`PyEval_SetTrace`, this function ignores any exceptions raised while setting the trace functions in all threads. @@ -2104,6 +2087,11 @@ Reference tracing is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer that was provided when :c:func:`PyRefTracer_SetTracer` was called. + If a new tracing function is registered replacing the current a call to the + trace function will be made with the object set to **NULL** and **event** set to + :c:data:`PyRefTracer_TRACKER_REMOVED`. This will happen just before the new + function is registered. + .. versionadded:: 3.13 .. c:var:: int PyRefTracer_CREATE @@ -2116,6 +2104,13 @@ Reference tracing The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python object has been destroyed. +.. c:var:: int PyRefTracer_TRACKER_REMOVED + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when the + current tracer is about to be replaced by a new one. + + .. versionadded:: 3.14 + .. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) Register a reference tracer function. The function will be called when a new @@ -2126,10 +2121,14 @@ Reference tracing Not that tracer functions **must not** create Python objects inside or otherwise the call will be re-entrant. The tracer also **must not** clear - any existing exception or set an exception. The GIL will be held every time - the tracer function is called. + any existing exception or set an exception. A :term:`thread state` will be active + every time the tracer function is called. + + There must be an :term:`attached thread state` when calling this function. - The GIL must be held when calling this function. + If another tracer function was already registered, the old function will be + called with **event** set to :c:data:`PyRefTracer_TRACKER_REMOVED` just before + the new function is registered. .. versionadded:: 3.13 @@ -2140,7 +2139,7 @@ Reference tracing If no tracer was registered this function will return NULL and will set the **data** pointer to NULL. - The GIL must be held when calling this function. + There must be an :term:`attached thread state` when calling this function. .. versionadded:: 3.13 @@ -2197,8 +2196,8 @@ CPython C level APIs are similar to those offered by pthreads and Windows: use a thread key and functions to associate a :c:expr:`void*` value per thread. -The GIL does *not* need to be held when calling these functions; they supply -their own locking. +A :term:`thread state` does *not* need to be :term:`attached ` +when calling these functions; they supply their own locking. Note that :file:`Python.h` does not include the declaration of the TLS APIs, you need to include :file:`pythread.h` to use thread-local storage. @@ -2367,7 +2366,7 @@ The C-API provides a basic mutual exclusion lock. Lock mutex *m*. If another thread has already locked it, the calling thread will block until the mutex is unlocked. While blocked, the thread - will temporarily release the :term:`GIL` if it is held. + will temporarily detach the :term:`thread state ` if one exists. .. versionadded:: 3.13 @@ -2378,6 +2377,18 @@ The C-API provides a basic mutual exclusion lock. .. versionadded:: 3.13 +.. c:function:: int PyMutex_IsLocked(PyMutex *m) + + Returns non-zero if the mutex *m* is currently locked, zero otherwise. + + .. note:: + + This function is intended for use in assertions and debugging only and + should not be used to make concurrency control decisions, as the lock + state may change immediately after the check. + + .. versionadded:: 3.14 + .. _python-critical-section-api: Python Critical Section API @@ -2388,12 +2399,26 @@ per-object locks for :term:`free-threaded ` CPython. They are intended to replace reliance on the :term:`global interpreter lock`, and are no-ops in versions of Python with the global interpreter lock. +Critical sections are intended to be used for custom types implemented +in C-API extensions. They should generally not be used with built-in types like +:class:`list` and :class:`dict` because their public C-APIs +already use critical sections internally, with the notable +exception of :c:func:`PyDict_Next`, which requires critical section +to be acquired externally. + Critical sections avoid deadlocks by implicitly suspending active critical -sections and releasing the locks during calls to :c:func:`PyEval_SaveThread`. -When :c:func:`PyEval_RestoreThread` is called, the most recent critical section -is resumed, and its locks reacquired. This means the critical section API -provides weaker guarantees than traditional locks -- they are useful because -their behavior is similar to the :term:`GIL`. +sections, hence, they do not provide exclusive access such as provided by +traditional locks like :c:type:`PyMutex`. When a critical section is started, +the per-object lock for the object is acquired. If the code executed inside the +critical section calls C-API functions then it can suspend the critical section thereby +releasing the per-object lock, so other threads can acquire the per-object lock +for the same object. + +Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also +available. Use these variants to start a critical section in a situation where +there is no :c:type:`PyObject` -- for example, when working with a C type that +does not extend or wrap :c:type:`PyObject` but still needs to call into the C +API in a manner that might lead to deadlocks. The functions and structs used by the macros are exposed for cases where C macros are not available. They should only be used as in the @@ -2440,6 +2465,23 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. .. versionadded:: 3.13 +.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) + + Locks the mutex *m* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_BeginMutex(&_py_cs, m) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for + the argument of the macro - it must be a :c:type:`PyMutex` pointer. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + .. c:macro:: Py_END_CRITICAL_SECTION() Ends the critical section and releases the per-object lock. @@ -2463,21 +2505,255 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. { PyCriticalSection2 _py_cs2; - PyCriticalSection_Begin2(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) + PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) In the default build, this macro expands to ``{``. .. versionadded:: 3.13 +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) + + Locks the mutexes *m1* and *m2* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for + the arguments of the macro - they must be :c:type:`PyMutex` pointers. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + .. c:macro:: Py_END_CRITICAL_SECTION2() Ends the critical section and releases the per-object locks. In the free-threaded build, this macro expands to:: - PyCriticalSection_End2(&_py_cs2); + PyCriticalSection2_End(&_py_cs2); } In the default build, this macro expands to ``}``. .. versionadded:: 3.13 + + +Legacy Locking APIs +------------------- + +These APIs are obsolete since Python 3.13 with the introduction of +:c:type:`PyMutex`. + +.. versionchanged:: 3.15 + These APIs are now a simple wrapper around ``PyMutex``. + + +.. c:type:: PyThread_type_lock + + A pointer to a mutual exclusion lock. + + +.. c:type:: PyLockStatus + + The result of acquiring a lock with a timeout. + + .. c:namespace:: NULL + + .. c:enumerator:: PY_LOCK_FAILURE + + Failed to acquire the lock. + + .. c:enumerator:: PY_LOCK_ACQUIRED + + The lock was successfully acquired. + + .. c:enumerator:: PY_LOCK_INTR + + The lock was interrupted by a signal. + + +.. c:function:: PyThread_type_lock PyThread_allocate_lock(void) + + Allocate a new lock. + + On success, this function returns a lock; on failure, this + function returns ``0`` without an exception set. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionchanged:: 3.15 + This function now always uses :c:type:`PyMutex`. In prior versions, this + would use a lock provided by the operating system. + + +.. c:function:: void PyThread_free_lock(PyThread_type_lock lock) + + Destroy *lock*. The lock should not be held by any thread when calling + this. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock lock, long long microseconds, int intr_flag) + + Acquire *lock* with a timeout. + + This will wait for *microseconds* microseconds to acquire the lock. If the + timeout expires, this function returns :c:enumerator:`PY_LOCK_FAILURE`. + If *microseconds* is ``-1``, this will wait indefinitely until the lock has + been released. + + If *intr_flag* is ``1``, acquiring the lock may be interrupted by a signal, + in which case this function returns :c:enumerator:`PY_LOCK_INTR`. Upon + interruption, it's generally expected that the caller makes a call to + :c:func:`Py_MakePendingCalls` to propagate an exception to Python code. + + If the lock is successfully acquired, this function returns + :c:enumerator:`PY_LOCK_ACQUIRED`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) + + Acquire *lock*. + + If *waitflag* is ``1`` and another thread currently holds the lock, this + function will wait until the lock can be acquired and will always return + ``1``. + + If *waitflag* is ``0`` and another thread holds the lock, this function will + not wait and instead return ``0``. If the lock is not held by any other + thread, then this function will acquire it and return ``1``. + + Unlike :c:func:`PyThread_acquire_lock_timed`, acquiring the lock cannot be + interrupted by a signal. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_release_lock(PyThread_type_lock lock) + + Release *lock*. If *lock* is not held, then this function issues a + fatal error. + + The caller does not need to hold an :term:`attached thread state`. + + +Operating System Thread APIs +============================ + +.. c:macro:: PYTHREAD_INVALID_THREAD_ID + + Sentinel value for an invalid thread ID. + + This is currently equivalent to ``(unsigned long)-1``. + + +.. c:function:: unsigned long PyThread_start_new_thread(void (*func)(void *), void *arg) + + Start function *func* in a new thread with argument *arg*. + The resulting thread is not intended to be joined. + + *func* must not be ``NULL``, but *arg* may be ``NULL``. + + On success, this function returns the identifier of the new thread; on failure, + this returns :c:macro:`PYTHREAD_INVALID_THREAD_ID`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: unsigned long PyThread_get_thread_ident(void) + + Return the identifier of the current thread, which will never be zero. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_ident` + + +.. c:function:: PyObject *PyThread_GetInfo(void) + + Get general information about the current thread in the form of a + :ref:`struct sequence ` object. This information is + accessible as :py:attr:`sys.thread_info` in Python. + + On success, this returns a new :term:`strong reference` to the thread + information; on failure, this returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + +.. c:macro:: PY_HAVE_THREAD_NATIVE_ID + + This macro is defined when the system supports native thread IDs. + + +.. c:function:: unsigned long PyThread_get_thread_native_id(void) + + Get the native identifier of the current thread as it was assigned by the operating + system's kernel, which will never be less than zero. + + This function is only available when :c:macro:`PY_HAVE_THREAD_NATIVE_ID` is + defined. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_native_id` + + +.. c:function:: void PyThread_exit_thread(void) + + Terminate the current thread. This function is generally considered unsafe + and should be avoided. It is kept solely for backwards compatibility. + + This function is only safe to call if all functions in the full call + stack are written to safely allow it. + + .. warning:: + + If the current system uses POSIX threads (also known as "pthreads"), + this calls :manpage:`pthread_exit(3)`, which attempts to unwind the stack + and call C++ destructors on some libc implementations. However, if a + ``noexcept`` function is reached, it may terminate the process. + Other systems, such as macOS, do unwinding. + + On Windows, this function calls ``_endthreadex()``, which kills the thread + without calling C++ destructors. + + In any case, there is a risk of corruption on the thread's stack. + + .. deprecated:: 3.14 + + +.. c:function:: void PyThread_init_thread(void) + + Initialize ``PyThread*`` APIs. Python executes this function automatically, + so there's little need to call it from an extension module. + + +.. c:function:: int PyThread_set_stacksize(size_t size) + + Set the stack size of the current thread to *size* bytes. + + This function returns ``0`` on success, ``-1`` if *size* is invalid, or + ``-2`` if the system does not support changing the stack size. This function + does not set exceptions. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: size_t PyThread_get_stacksize(void) + + Return the stack size of the current thread in bytes, or ``0`` if the system's + default stack size is in use. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 6194d7446c73e4..c345029e4acd49 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -6,26 +6,15 @@ Python Initialization Configuration *********************************** -PyConfig C API -============== - -.. versionadded:: 3.8 -Python can be initialized with :c:func:`Py_InitializeFromConfig` and the -:c:type:`PyConfig` structure. It can be preinitialized with -:c:func:`Py_PreInitialize` and the :c:type:`PyPreConfig` structure. +.. _pyinitconfig_api: -There are two kinds of configuration: +PyInitConfig C API +================== -* The :ref:`Python Configuration ` can be used to build a - customized Python which behaves as the regular Python. For example, - environment variables and command line arguments are used to configure - Python. +.. versionadded:: 3.14 -* The :ref:`Isolated Configuration ` can be used to embed - Python into an application. It isolates Python from the system. For example, - environment variables are ignored, the LC_CTYPE locale is left unchanged and - no signal handler is registered. +Python can be initialized with :c:func:`Py_InitializeFromInitConfig`. The :c:func:`Py_RunMain` function can be used to write a customized Python program. @@ -33,1974 +22,2290 @@ program. See also :ref:`Initialization, Finalization, and Threads `. .. seealso:: - :pep:`587` "Python Initialization Configuration". + :pep:`741` "Python Configuration C API". Example ------- -Example of customized Python always running in isolated mode:: - - int main(int argc, char **argv) - { - PyStatus status; +Example of customized Python always running with the :ref:`Python Development +Mode ` enabled; return ``-1`` on error: - PyConfig config; - PyConfig_InitPythonConfig(&config); - config.isolated = 1; +.. code-block:: c - /* Decode command line arguments. - Implicitly preinitialize Python (in isolated mode). */ - status = PyConfig_SetBytesArgv(&config, argc, argv); - if (PyStatus_Exception(status)) { - goto exception; + int init_python(void) + { + PyInitConfig *config = PyInitConfig_Create(); + if (config == NULL) { + printf("PYTHON INIT ERROR: memory allocation failed\n"); + return -1; } - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - goto exception; + // Enable the Python Development Mode + if (PyInitConfig_SetInt(config, "dev_mode", 1) < 0) { + goto error; } - PyConfig_Clear(&config); - return Py_RunMain(); + // Initialize Python with the configuration + if (Py_InitializeFromInitConfig(config) < 0) { + goto error; + } + PyInitConfig_Free(config); + return 0; - exception: - PyConfig_Clear(&config); - if (PyStatus_IsExit(status)) { - return status.exitcode; + error: + { + // Display the error message. + // + // This uncommon braces style is used, because you cannot make + // goto targets point to variable declarations. + const char *err_msg; + (void)PyInitConfig_GetError(config, &err_msg); + printf("PYTHON INIT ERROR: %s\n", err_msg); + PyInitConfig_Free(config); + return -1; } - /* Display the error message and exit the process with - non-zero exit code */ - Py_ExitStatusException(status); } +Create Config +------------- -PyWideStringList ----------------- +.. c:struct:: PyInitConfig -.. c:type:: PyWideStringList + Opaque structure to configure the Python initialization. - List of ``wchar_t*`` strings. - If *length* is non-zero, *items* must be non-``NULL`` and all strings must be - non-``NULL``. +.. c:function:: PyInitConfig* PyInitConfig_Create(void) - .. c:namespace:: NULL + Create a new initialization configuration using :ref:`Isolated Configuration + ` default values. - Methods: + It must be freed by :c:func:`PyInitConfig_Free`. - .. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item) + Return ``NULL`` on memory allocation failure. - Append *item* to *list*. - Python must be preinitialized to call this function. +.. c:function:: void PyInitConfig_Free(PyInitConfig *config) - .. c:function:: PyStatus PyWideStringList_Insert(PyWideStringList *list, Py_ssize_t index, const wchar_t *item) + Free memory of the initialization configuration *config*. - Insert *item* into *list* at *index*. + If *config* is ``NULL``, no operation is performed. - If *index* is greater than or equal to *list* length, append *item* to - *list*. - *index* must be greater than or equal to ``0``. +Error Handling +-------------- - Python must be preinitialized to call this function. +.. c:function:: int PyInitConfig_GetError(PyInitConfig* config, const char **err_msg) - .. c:namespace:: PyWideStringList + Get the *config* error message. - Structure fields: + * Set *\*err_msg* and return ``1`` if an error is set. + * Set *\*err_msg* to ``NULL`` and return ``0`` otherwise. - .. c:member:: Py_ssize_t length + An error message is a UTF-8 encoded string. - List length. + If *config* has an exit code, format the exit code as an error + message. - .. c:member:: wchar_t** items + The error message remains valid until another ``PyInitConfig`` + function is called with *config*. The caller doesn't have to free the + error message. - List items. -PyStatus --------- +.. c:function:: int PyInitConfig_GetExitCode(PyInitConfig* config, int *exitcode) -.. c:type:: PyStatus + Get the *config* exit code. - Structure to store an initialization function status: success, error - or exit. + * Set *\*exitcode* and return ``1`` if *config* has an exit code set. + * Return ``0`` if *config* has no exit code set. - For an error, it can store the C function name which created the error. + Only the ``Py_InitializeFromInitConfig()`` function can set an exit + code if the ``parse_argv`` option is non-zero. - Structure fields: + An exit code can be set when parsing the command line failed (exit + code ``2``) or when a command line option asks to display the command + line help (exit code ``0``). - .. c:member:: int exitcode - Exit code. Argument passed to ``exit()``. +Get Options +----------- - .. c:member:: const char *err_msg +The configuration option *name* parameter must be a non-NULL null-terminated +UTF-8 encoded string. See :ref:`Configuration Options `. - Error message. +.. c:function:: int PyInitConfig_HasOption(PyInitConfig *config, const char *name) - .. c:member:: const char *func + Test if the configuration has an option called *name*. - Name of the function which created an error, can be ``NULL``. + Return ``1`` if the option exists, or return ``0`` otherwise. - .. c:namespace:: NULL - Functions to create a status: +.. c:function:: int PyInitConfig_GetInt(PyInitConfig *config, const char *name, int64_t *value) - .. c:function:: PyStatus PyStatus_Ok(void) + Get an integer configuration option. - Success. + * Set *\*value*, and return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. - .. c:function:: PyStatus PyStatus_Error(const char *err_msg) - Initialization error with a message. +.. c:function:: int PyInitConfig_GetStr(PyInitConfig *config, const char *name, char **value) - *err_msg* must not be ``NULL``. + Get a string configuration option as a null-terminated UTF-8 + encoded string. - .. c:function:: PyStatus PyStatus_NoMemory(void) + * Set *\*value*, and return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. - Memory allocation failure (out of memory). + *\*value* can be set to ``NULL`` if the option is an optional string and the + option is unset. - .. c:function:: PyStatus PyStatus_Exit(int exitcode) + On success, the string must be released with ``free(value)`` if it's not + ``NULL``. - Exit Python with the specified exit code. - Functions to handle a status: +.. c:function:: int PyInitConfig_GetStrList(PyInitConfig *config, const char *name, size_t *length, char ***items) - .. c:function:: int PyStatus_Exception(PyStatus status) + Get a string list configuration option as an array of + null-terminated UTF-8 encoded strings. - Is the status an error or an exit? If true, the exception must be - handled; by calling :c:func:`Py_ExitStatusException` for example. + * Set *\*length* and *\*value*, and return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. - .. c:function:: int PyStatus_IsError(PyStatus status) + On success, the string list must be released with + ``PyInitConfig_FreeStrList(length, items)``. - Is the result an error? - .. c:function:: int PyStatus_IsExit(PyStatus status) +.. c:function:: void PyInitConfig_FreeStrList(size_t length, char **items) - Is the result an exit? + Free memory of a string list created by + ``PyInitConfig_GetStrList()``. - .. c:function:: void Py_ExitStatusException(PyStatus status) - Call ``exit(exitcode)`` if *status* is an exit. Print the error - message and exit with a non-zero exit code if *status* is an error. Must - only be called if ``PyStatus_Exception(status)`` is non-zero. +Set Options +----------- -.. note:: - Internally, Python uses macros which set ``PyStatus.func``, - whereas functions to create a status set ``func`` to ``NULL``. +The configuration option *name* parameter must be a non-NULL null-terminated +UTF-8 encoded string. See :ref:`Configuration Options `. -Example:: +Some configuration options have side effects on other options. This logic is +only implemented when ``Py_InitializeFromInitConfig()`` is called, not by the +"Set" functions below. For example, setting ``dev_mode`` to ``1`` does not set +``faulthandler`` to ``1``. - PyStatus alloc(void **ptr, size_t size) - { - *ptr = PyMem_RawMalloc(size); - if (*ptr == NULL) { - return PyStatus_NoMemory(); - } - return PyStatus_Ok(); - } +.. c:function:: int PyInitConfig_SetInt(PyInitConfig *config, const char *name, int64_t value) - int main(int argc, char **argv) - { - void *ptr; - PyStatus status = alloc(&ptr, 16); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - PyMem_Free(ptr); - return 0; - } + Set an integer configuration option. + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. -PyPreConfig ------------ -.. c:type:: PyPreConfig +.. c:function:: int PyInitConfig_SetStr(PyInitConfig *config, const char *name, const char *value) - Structure used to preinitialize Python. + Set a string configuration option from a null-terminated UTF-8 + encoded string. The string is copied. - .. c:namespace:: NULL + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. - Function to initialize a preconfiguration: - .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) +.. c:function:: int PyInitConfig_SetStrList(PyInitConfig *config, const char *name, size_t length, char * const *items) - Initialize the preconfiguration with :ref:`Python Configuration - `. + Set a string list configuration option from an array of + null-terminated UTF-8 encoded strings. The string list is copied. - .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. - Initialize the preconfiguration with :ref:`Isolated Configuration - `. - .. c:namespace:: PyPreConfig +Module +------ - Structure fields: +.. c:function:: int PyInitConfig_AddModule(PyInitConfig *config, const char *name, PyObject* (*initfunc)(void)) - .. c:member:: int allocator + Add a built-in extension module to the table of built-in modules. - Name of the Python memory allocators: + The new module can be imported by the name *name*, and uses the function + *initfunc* as the initialization function called on the first attempted + import. - * ``PYMEM_ALLOCATOR_NOT_SET`` (``0``): don't change memory allocators - (use defaults). - * ``PYMEM_ALLOCATOR_DEFAULT`` (``1``): :ref:`default memory allocators - `. - * ``PYMEM_ALLOCATOR_DEBUG`` (``2``): :ref:`default memory allocators - ` with :ref:`debug hooks - `. - * ``PYMEM_ALLOCATOR_MALLOC`` (``3``): use ``malloc()`` of the C library. - * ``PYMEM_ALLOCATOR_MALLOC_DEBUG`` (``4``): force usage of - ``malloc()`` with :ref:`debug hooks `. - * ``PYMEM_ALLOCATOR_PYMALLOC`` (``5``): :ref:`Python pymalloc memory - allocator `. - * ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` (``6``): :ref:`Python pymalloc - memory allocator ` with :ref:`debug hooks - `. - * ``PYMEM_ALLOCATOR_MIMALLOC`` (``6``): use ``mimalloc``, a fast - malloc replacement. - * ``PYMEM_ALLOCATOR_MIMALLOC_DEBUG`` (``7``): use ``mimalloc``, a fast - malloc replacement with :ref:`debug hooks `. + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + If Python is initialized multiple times, ``PyInitConfig_AddModule()`` must + be called at each Python initialization. - ``PYMEM_ALLOCATOR_PYMALLOC`` and ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` are - not supported if Python is :option:`configured using --without-pymalloc - <--without-pymalloc>`. + Similar to the :c:func:`PyImport_AppendInittab` function. - ``PYMEM_ALLOCATOR_MIMALLOC`` and ``PYMEM_ALLOCATOR_MIMALLOC_DEBUG`` are - not supported if Python is :option:`configured using --without-mimalloc - <--without-mimalloc>` or if the underlying atomic support isn't - available. - See :ref:`Memory Management `. +Initialize Python +----------------- - Default: ``PYMEM_ALLOCATOR_NOT_SET``. +.. c:function:: int Py_InitializeFromInitConfig(PyInitConfig *config) - .. c:member:: int configure_locale + Initialize Python from the initialization configuration. - Set the LC_CTYPE locale to the user preferred locale. + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + * Set an exit code in *config* and return ``-1`` if Python wants to + exit. - If equals to ``0``, set :c:member:`~PyPreConfig.coerce_c_locale` and - :c:member:`~PyPreConfig.coerce_c_locale_warn` members to ``0``. + See ``PyInitConfig_GetExitcode()`` for the exit code case. - See the :term:`locale encoding`. - Default: ``1`` in Python config, ``0`` in isolated config. +.. _pyinitconfig-opts: + +Configuration Options +===================== + +.. list-table:: + :header-rows: 1 + + * - Option + - PyConfig/PyPreConfig member + - Type + - Visibility + * - ``"allocator"`` + - :c:member:`allocator ` + - ``int`` + - Read-only + * - ``"argv"`` + - :c:member:`argv ` + - ``list[str]`` + - Public + * - ``"base_exec_prefix"`` + - :c:member:`base_exec_prefix ` + - ``str`` + - Public + * - ``"base_executable"`` + - :c:member:`base_executable ` + - ``str`` + - Public + * - ``"base_prefix"`` + - :c:member:`base_prefix ` + - ``str`` + - Public + * - ``"buffered_stdio"`` + - :c:member:`buffered_stdio ` + - ``bool`` + - Read-only + * - ``"bytes_warning"`` + - :c:member:`bytes_warning ` + - ``int`` + - Public + * - ``"check_hash_pycs_mode"`` + - :c:member:`check_hash_pycs_mode ` + - ``str`` + - Read-only + * - ``"code_debug_ranges"`` + - :c:member:`code_debug_ranges ` + - ``bool`` + - Read-only + * - ``"coerce_c_locale"`` + - :c:member:`coerce_c_locale ` + - ``bool`` + - Read-only + * - ``"coerce_c_locale_warn"`` + - :c:member:`coerce_c_locale_warn ` + - ``bool`` + - Read-only + * - ``"configure_c_stdio"`` + - :c:member:`configure_c_stdio ` + - ``bool`` + - Read-only + * - ``"configure_locale"`` + - :c:member:`configure_locale ` + - ``bool`` + - Read-only + * - ``"cpu_count"`` + - :c:member:`cpu_count ` + - ``int`` + - Public + * - ``"dev_mode"`` + - :c:member:`dev_mode ` + - ``bool`` + - Read-only + * - ``"dump_refs"`` + - :c:member:`dump_refs ` + - ``bool`` + - Read-only + * - ``"dump_refs_file"`` + - :c:member:`dump_refs_file ` + - ``str`` + - Read-only + * - ``"exec_prefix"`` + - :c:member:`exec_prefix ` + - ``str`` + - Public + * - ``"executable"`` + - :c:member:`executable ` + - ``str`` + - Public + * - ``"faulthandler"`` + - :c:member:`faulthandler ` + - ``bool`` + - Read-only + * - ``"filesystem_encoding"`` + - :c:member:`filesystem_encoding ` + - ``str`` + - Read-only + * - ``"filesystem_errors"`` + - :c:member:`filesystem_errors ` + - ``str`` + - Read-only + * - ``"hash_seed"`` + - :c:member:`hash_seed ` + - ``int`` + - Read-only + * - ``"home"`` + - :c:member:`home ` + - ``str`` + - Read-only + * - ``"import_time"`` + - :c:member:`import_time ` + - ``int`` + - Read-only + * - ``"inspect"`` + - :c:member:`inspect ` + - ``bool`` + - Public + * - ``"install_signal_handlers"`` + - :c:member:`install_signal_handlers ` + - ``bool`` + - Read-only + * - ``"int_max_str_digits"`` + - :c:member:`int_max_str_digits ` + - ``int`` + - Public + * - ``"interactive"`` + - :c:member:`interactive ` + - ``bool`` + - Public + * - ``"isolated"`` + - :c:member:`isolated ` + - ``bool`` + - Read-only + * - ``"legacy_windows_fs_encoding"`` + - :c:member:`legacy_windows_fs_encoding ` + - ``bool`` + - Read-only + * - ``"legacy_windows_stdio"`` + - :c:member:`legacy_windows_stdio ` + - ``bool`` + - Read-only + * - ``"malloc_stats"`` + - :c:member:`malloc_stats ` + - ``bool`` + - Read-only + * - ``"module_search_paths"`` + - :c:member:`module_search_paths ` + - ``list[str]`` + - Public + * - ``"optimization_level"`` + - :c:member:`optimization_level ` + - ``int`` + - Public + * - ``"orig_argv"`` + - :c:member:`orig_argv ` + - ``list[str]`` + - Read-only + * - ``"parse_argv"`` + - :c:member:`parse_argv ` + - ``bool`` + - Read-only + * - ``"parser_debug"`` + - :c:member:`parser_debug ` + - ``bool`` + - Public + * - ``"pathconfig_warnings"`` + - :c:member:`pathconfig_warnings ` + - ``bool`` + - Read-only + * - ``"perf_profiling"`` + - :c:member:`perf_profiling ` + - ``bool`` + - Read-only + * - ``"platlibdir"`` + - :c:member:`platlibdir ` + - ``str`` + - Public + * - ``"prefix"`` + - :c:member:`prefix ` + - ``str`` + - Public + * - ``"program_name"`` + - :c:member:`program_name ` + - ``str`` + - Read-only + * - ``"pycache_prefix"`` + - :c:member:`pycache_prefix ` + - ``str`` + - Public + * - ``"quiet"`` + - :c:member:`quiet ` + - ``bool`` + - Public + * - ``"run_command"`` + - :c:member:`run_command ` + - ``str`` + - Read-only + * - ``"run_filename"`` + - :c:member:`run_filename ` + - ``str`` + - Read-only + * - ``"run_module"`` + - :c:member:`run_module ` + - ``str`` + - Read-only + * - ``"run_presite"`` + - :c:member:`run_presite ` + - ``str`` + - Read-only + * - ``"safe_path"`` + - :c:member:`safe_path ` + - ``bool`` + - Read-only + * - ``"show_ref_count"`` + - :c:member:`show_ref_count ` + - ``bool`` + - Read-only + * - ``"site_import"`` + - :c:member:`site_import ` + - ``bool`` + - Read-only + * - ``"skip_source_first_line"`` + - :c:member:`skip_source_first_line ` + - ``bool`` + - Read-only + * - ``"stdio_encoding"`` + - :c:member:`stdio_encoding ` + - ``str`` + - Read-only + * - ``"stdio_errors"`` + - :c:member:`stdio_errors ` + - ``str`` + - Read-only + * - ``"stdlib_dir"`` + - :c:member:`stdlib_dir ` + - ``str`` + - Public + * - ``"tracemalloc"`` + - :c:member:`tracemalloc ` + - ``int`` + - Read-only + * - ``"use_environment"`` + - :c:member:`use_environment ` + - ``bool`` + - Public + * - ``"use_frozen_modules"`` + - :c:member:`use_frozen_modules ` + - ``bool`` + - Read-only + * - ``"use_hash_seed"`` + - :c:member:`use_hash_seed ` + - ``bool`` + - Read-only + * - ``"use_system_logger"`` + - :c:member:`use_system_logger ` + - ``bool`` + - Read-only + * - ``"user_site_directory"`` + - :c:member:`user_site_directory ` + - ``bool`` + - Read-only + * - ``"utf8_mode"`` + - :c:member:`utf8_mode ` + - ``bool`` + - Read-only + * - ``"verbose"`` + - :c:member:`verbose ` + - ``int`` + - Public + * - ``"warn_default_encoding"`` + - :c:member:`warn_default_encoding ` + - ``bool`` + - Read-only + * - ``"warnoptions"`` + - :c:member:`warnoptions ` + - ``list[str]`` + - Public + * - ``"write_bytecode"`` + - :c:member:`write_bytecode ` + - ``bool`` + - Public + * - ``"xoptions"`` + - :c:member:`xoptions ` + - ``dict[str, str]`` + - Public + * - ``"_pystats"`` + - :c:member:`_pystats ` + - ``bool`` + - Read-only + +Visibility: + +* Public: Can by get by :c:func:`PyConfig_Get` and set by + :c:func:`PyConfig_Set`. +* Read-only: Can by get by :c:func:`PyConfig_Get`, but cannot be set by + :c:func:`PyConfig_Set`. - .. c:member:: int coerce_c_locale - If equals to ``2``, coerce the C locale. +Runtime Python configuration API +================================ - If equals to ``1``, read the LC_CTYPE locale to decide if it should be - coerced. +At runtime, it's possible to get and set configuration options using +:c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions. - See the :term:`locale encoding`. +The configuration option *name* parameter must be a non-NULL null-terminated +UTF-8 encoded string. See :ref:`Configuration Options `. - Default: ``-1`` in Python config, ``0`` in isolated config. +Some options are read from the :mod:`sys` attributes. For example, the option +``"argv"`` is read from :data:`sys.argv`. - .. c:member:: int coerce_c_locale_warn - If non-zero, emit a warning if the C locale is coerced. +.. c:function:: PyObject* PyConfig_Get(const char *name) - Default: ``-1`` in Python config, ``0`` in isolated config. + Get the current runtime value of a configuration option as a Python object. - .. c:member:: int dev_mode + * Return a new reference on success. + * Set an exception and return ``NULL`` on error. - :ref:`Python Development Mode `: see - :c:member:`PyConfig.dev_mode`. + The object type depends on the configuration option. It can be: - Default: ``-1`` in Python mode, ``0`` in isolated mode. + * ``bool`` + * ``int`` + * ``str`` + * ``list[str]`` + * ``dict[str, str]`` - .. c:member:: int isolated + The caller must have an :term:`attached thread state`. The function cannot + be called before Python initialization nor after Python finalization. - Isolated mode: see :c:member:`PyConfig.isolated`. + .. versionadded:: 3.14 - Default: ``0`` in Python mode, ``1`` in isolated mode. - .. c:member:: int legacy_windows_fs_encoding +.. c:function:: int PyConfig_GetInt(const char *name, int *value) - If non-zero: + Similar to :c:func:`PyConfig_Get`, but get the value as a C int. - * Set :c:member:`PyPreConfig.utf8_mode` to ``0``, - * Set :c:member:`PyConfig.filesystem_encoding` to ``"mbcs"``, - * Set :c:member:`PyConfig.filesystem_errors` to ``"replace"``. + * Return ``0`` on success. + * Set an exception and return ``-1`` on error. - Initialized from the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment - variable value. + .. versionadded:: 3.14 - Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for - Windows specific code. - Default: ``0``. +.. c:function:: PyObject* PyConfig_Names(void) - .. c:member:: int parse_argv + Get all configuration option names as a ``frozenset``. - If non-zero, :c:func:`Py_PreInitializeFromArgs` and - :c:func:`Py_PreInitializeFromBytesArgs` parse their ``argv`` argument the - same way the regular Python parses command line arguments: see - :ref:`Command Line Arguments `. + * Return a new reference on success. + * Set an exception and return ``NULL`` on error. - Default: ``1`` in Python config, ``0`` in isolated config. + The caller must have an :term:`attached thread state`. The function cannot + be called before Python initialization nor after Python finalization. - .. c:member:: int use_environment + .. versionadded:: 3.14 - Use :ref:`environment variables `? See - :c:member:`PyConfig.use_environment`. - Default: ``1`` in Python config and ``0`` in isolated config. +.. c:function:: int PyConfig_Set(const char *name, PyObject *value) - .. c:member:: int utf8_mode + Set the current runtime value of a configuration option. - If non-zero, enable the :ref:`Python UTF-8 Mode `. + * Raise a :exc:`ValueError` if there is no option *name*. + * Raise a :exc:`ValueError` if *value* is an invalid value. + * Raise a :exc:`ValueError` if the option is read-only (cannot be set). + * Raise a :exc:`TypeError` if *value* has not the proper type. - Set to ``0`` or ``1`` by the :option:`-X utf8 <-X>` command line option - and the :envvar:`PYTHONUTF8` environment variable. + The caller must have an :term:`attached thread state`. The function cannot + be called before Python initialization nor after Python finalization. - Also set to ``1`` if the ``LC_CTYPE`` locale is ``C`` or ``POSIX``. + .. audit-event:: cpython.PyConfig_Set name,value c.PyConfig_Set - Default: ``-1`` in Python config and ``0`` in isolated config. + .. versionadded:: 3.14 -.. _c-preinit: +.. _pyconfig_api: -Preinitialize Python with PyPreConfig -------------------------------------- +PyConfig C API +============== -The preinitialization of Python: +.. versionadded:: 3.8 -* Set the Python memory allocators (:c:member:`PyPreConfig.allocator`) -* Configure the LC_CTYPE locale (:term:`locale encoding`) -* Set the :ref:`Python UTF-8 Mode ` - (:c:member:`PyPreConfig.utf8_mode`) +Python can be initialized with :c:func:`Py_InitializeFromConfig` and the +:c:type:`PyConfig` structure. It can be preinitialized with +:c:func:`Py_PreInitialize` and the :c:type:`PyPreConfig` structure. -The current preconfiguration (``PyPreConfig`` type) is stored in -``_PyRuntime.preconfig``. +There are two kinds of configuration: -Functions to preinitialize Python: +* The :ref:`Python Configuration ` can be used to build a + customized Python which behaves as the regular Python. For example, + environment variables and command line arguments are used to configure + Python. -.. c:function:: PyStatus Py_PreInitialize(const PyPreConfig *preconfig) +* The :ref:`Isolated Configuration ` can be used to embed + Python into an application. It isolates Python from the system. For example, + environment variables are ignored, the LC_CTYPE locale is left unchanged and + no signal handler is registered. - Preinitialize Python from *preconfig* preconfiguration. +The :c:func:`Py_RunMain` function can be used to write a customized Python +program. - *preconfig* must not be ``NULL``. +See also :ref:`Initialization, Finalization, and Threads `. -.. c:function:: PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *preconfig, int argc, char * const *argv) +.. seealso:: + :pep:`587` "Python Initialization Configuration". - Preinitialize Python from *preconfig* preconfiguration. - Parse *argv* command line arguments (bytes strings) if - :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. +Example +------- - *preconfig* must not be ``NULL``. +Example of customized Python always running in isolated mode:: -.. c:function:: PyStatus Py_PreInitializeFromArgs(const PyPreConfig *preconfig, int argc, wchar_t * const * argv) + int main(int argc, char **argv) + { + PyStatus status; - Preinitialize Python from *preconfig* preconfiguration. + PyConfig config; + PyConfig_InitPythonConfig(&config); + config.isolated = 1; - Parse *argv* command line arguments (wide strings) if - :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. + /* Decode command line arguments. + Implicitly preinitialize Python (in isolated mode). */ + status = PyConfig_SetBytesArgv(&config, argc, argv); + if (PyStatus_Exception(status)) { + goto exception; + } - *preconfig* must not be ``NULL``. + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto exception; + } + PyConfig_Clear(&config); -The caller is responsible to handle exceptions (error or exit) using -:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. + return Py_RunMain(); -For :ref:`Python Configuration ` -(:c:func:`PyPreConfig_InitPythonConfig`), if Python is initialized with -command line arguments, the command line arguments must also be passed to -preinitialize Python, since they have an effect on the pre-configuration -like encodings. For example, the :option:`-X utf8 <-X>` command line option -enables the :ref:`Python UTF-8 Mode `. + exception: + PyConfig_Clear(&config); + if (PyStatus_IsExit(status)) { + return status.exitcode; + } + /* Display the error message and exit the process with + non-zero exit code */ + Py_ExitStatusException(status); + } -``PyMem_SetAllocator()`` can be called after :c:func:`Py_PreInitialize` and -before :c:func:`Py_InitializeFromConfig` to install a custom memory allocator. -It can be called before :c:func:`Py_PreInitialize` if -:c:member:`PyPreConfig.allocator` is set to ``PYMEM_ALLOCATOR_NOT_SET``. -Python memory allocation functions like :c:func:`PyMem_RawMalloc` must not be -used before the Python preinitialization, whereas calling directly ``malloc()`` -and ``free()`` is always safe. :c:func:`Py_DecodeLocale` must not be called -before the Python preinitialization. +PyWideStringList +---------------- -Example using the preinitialization to enable -the :ref:`Python UTF-8 Mode `:: +.. c:type:: PyWideStringList - PyStatus status; - PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); + List of ``wchar_t*`` strings. - preconfig.utf8_mode = 1; + If *length* is non-zero, *items* must be non-``NULL`` and all strings must be + non-``NULL``. - status = Py_PreInitialize(&preconfig); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } + .. c:namespace:: NULL - /* at this point, Python speaks UTF-8 */ + Methods: - Py_Initialize(); - /* ... use Python API here ... */ - Py_Finalize(); + .. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item) + Append *item* to *list*. -PyConfig --------- + Python must be preinitialized to call this function. -.. c:type:: PyConfig + .. c:function:: PyStatus PyWideStringList_Insert(PyWideStringList *list, Py_ssize_t index, const wchar_t *item) - Structure containing most parameters to configure Python. + Insert *item* into *list* at *index*. - When done, the :c:func:`PyConfig_Clear` function must be used to release the - configuration memory. + If *index* is greater than or equal to *list* length, append *item* to + *list*. - .. c:namespace:: NULL + *index* must be greater than or equal to ``0``. - Structure methods: + Python must be preinitialized to call this function. - .. c:function:: void PyConfig_InitPythonConfig(PyConfig *config) + .. c:namespace:: PyWideStringList - Initialize configuration with the :ref:`Python Configuration - `. + Structure fields: - .. c:function:: void PyConfig_InitIsolatedConfig(PyConfig *config) + .. c:member:: Py_ssize_t length - Initialize configuration with the :ref:`Isolated Configuration - `. + List length. - .. c:function:: PyStatus PyConfig_SetString(PyConfig *config, wchar_t * const *config_str, const wchar_t *str) + .. c:member:: wchar_t** items - Copy the wide character string *str* into ``*config_str``. + List items. - :ref:`Preinitialize Python ` if needed. +PyStatus +-------- - .. c:function:: PyStatus PyConfig_SetBytesString(PyConfig *config, wchar_t * const *config_str, const char *str) +.. c:type:: PyStatus - Decode *str* using :c:func:`Py_DecodeLocale` and set the result into - ``*config_str``. + Structure to store an initialization function status: success, error + or exit. - :ref:`Preinitialize Python ` if needed. + For an error, it can store the C function name which created the error. - .. c:function:: PyStatus PyConfig_SetArgv(PyConfig *config, int argc, wchar_t * const *argv) + Structure fields: - Set command line arguments (:c:member:`~PyConfig.argv` member of - *config*) from the *argv* list of wide character strings. + .. c:member:: int exitcode - :ref:`Preinitialize Python ` if needed. + Exit code. Argument passed to ``exit()``. - .. c:function:: PyStatus PyConfig_SetBytesArgv(PyConfig *config, int argc, char * const *argv) + .. c:member:: const char *err_msg - Set command line arguments (:c:member:`~PyConfig.argv` member of - *config*) from the *argv* list of bytes strings. Decode bytes using - :c:func:`Py_DecodeLocale`. + Error message. - :ref:`Preinitialize Python ` if needed. + .. c:member:: const char *func - .. c:function:: PyStatus PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, Py_ssize_t length, wchar_t **items) + Name of the function which created an error, can be ``NULL``. - Set the list of wide strings *list* to *length* and *items*. + .. c:namespace:: NULL - :ref:`Preinitialize Python ` if needed. + Functions to create a status: - .. c:function:: PyStatus PyConfig_Read(PyConfig *config) + .. c:function:: PyStatus PyStatus_Ok(void) - Read all Python configuration. + Success. - Fields which are already initialized are left unchanged. + .. c:function:: PyStatus PyStatus_Error(const char *err_msg) - Fields for :ref:`path configuration ` are no longer - calculated or modified when calling this function, as of Python 3.11. + Initialization error with a message. - The :c:func:`PyConfig_Read` function only parses - :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` - is set to ``2`` after arguments are parsed. Since Python arguments are - stripped from :c:member:`PyConfig.argv`, parsing arguments twice would - parse the application options as Python options. + *err_msg* must not be ``NULL``. - :ref:`Preinitialize Python ` if needed. + .. c:function:: PyStatus PyStatus_NoMemory(void) - .. versionchanged:: 3.10 - The :c:member:`PyConfig.argv` arguments are now only parsed once, - :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are - parsed, and arguments are only parsed if - :c:member:`PyConfig.parse_argv` equals ``1``. + Memory allocation failure (out of memory). - .. versionchanged:: 3.11 - :c:func:`PyConfig_Read` no longer calculates all paths, and so fields - listed under :ref:`Python Path Configuration ` may - no longer be updated until :c:func:`Py_InitializeFromConfig` is - called. + .. c:function:: PyStatus PyStatus_Exit(int exitcode) - .. c:function:: void PyConfig_Clear(PyConfig *config) + Exit Python with the specified exit code. - Release configuration memory. + Functions to handle a status: - Most ``PyConfig`` methods :ref:`preinitialize Python ` if needed. - In that case, the Python preinitialization configuration - (:c:type:`PyPreConfig`) in based on the :c:type:`PyConfig`. If configuration - fields which are in common with :c:type:`PyPreConfig` are tuned, they must - be set before calling a :c:type:`PyConfig` method: + .. c:function:: int PyStatus_Exception(PyStatus status) - * :c:member:`PyConfig.dev_mode` - * :c:member:`PyConfig.isolated` - * :c:member:`PyConfig.parse_argv` - * :c:member:`PyConfig.use_environment` + Is the status an error or an exit? If true, the exception must be + handled; by calling :c:func:`Py_ExitStatusException` for example. - Moreover, if :c:func:`PyConfig_SetArgv` or :c:func:`PyConfig_SetBytesArgv` - is used, this method must be called before other methods, since the - preinitialization configuration depends on command line arguments (if - :c:member:`~PyConfig.parse_argv` is non-zero). + .. c:function:: int PyStatus_IsError(PyStatus status) - The caller of these methods is responsible to handle exceptions (error or - exit) using ``PyStatus_Exception()`` and ``Py_ExitStatusException()``. + Is the result an error? - .. c:namespace:: PyConfig + .. c:function:: int PyStatus_IsExit(PyStatus status) - Structure fields: + Is the result an exit? - .. c:member:: PyWideStringList argv + .. c:function:: void Py_ExitStatusException(PyStatus status) - .. index:: - single: main() - single: argv (in module sys) + Call ``exit(exitcode)`` if *status* is an exit. Print the error + message and exit with a non-zero exit code if *status* is an error. Must + only be called if ``PyStatus_Exception(status)`` is non-zero. - Set :data:`sys.argv` command line arguments based on - :c:member:`~PyConfig.argv`. These parameters are similar to those passed - to the program's :c:func:`main` function with the difference that the - first entry should refer to the script file to be executed rather than - the executable hosting the Python interpreter. If there isn't a script - that will be run, the first entry in :c:member:`~PyConfig.argv` can be an - empty string. +.. note:: + Internally, Python uses macros which set ``PyStatus.func``, + whereas functions to create a status set ``func`` to ``NULL``. - Set :c:member:`~PyConfig.parse_argv` to ``1`` to parse - :c:member:`~PyConfig.argv` the same way the regular Python parses Python - command line arguments and then to strip Python arguments from - :c:member:`~PyConfig.argv`. +Example:: - If :c:member:`~PyConfig.argv` is empty, an empty string is added to - ensure that :data:`sys.argv` always exists and is never empty. + PyStatus alloc(void **ptr, size_t size) + { + *ptr = PyMem_RawMalloc(size); + if (*ptr == NULL) { + return PyStatus_NoMemory(); + } + return PyStatus_Ok(); + } - Default: ``NULL``. + int main(int argc, char **argv) + { + void *ptr; + PyStatus status = alloc(&ptr, 16); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + PyMem_Free(ptr); + return 0; + } - See also the :c:member:`~PyConfig.orig_argv` member. - .. c:member:: int safe_path +PyPreConfig +----------- - If equals to zero, ``Py_RunMain()`` prepends a potentially unsafe path to - :data:`sys.path` at startup: +.. c:type:: PyPreConfig - * If :c:member:`argv[0] ` is equal to ``L"-m"`` - (``python -m module``), prepend the current working directory. - * If running a script (``python script.py``), prepend the script's - directory. If it's a symbolic link, resolve symbolic links. - * Otherwise (``python -c code`` and ``python``), prepend an empty string, - which means the current working directory. + Structure used to preinitialize Python. - Set to ``1`` by the :option:`-P` command line option and the - :envvar:`PYTHONSAFEPATH` environment variable. + .. c:namespace:: NULL - Default: ``0`` in Python config, ``1`` in isolated config. + Function to initialize a preconfiguration: - .. versionadded:: 3.11 + .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) - .. c:member:: wchar_t* base_exec_prefix + Initialize the preconfiguration with :ref:`Python Configuration + `. - :data:`sys.base_exec_prefix`. + .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) - Default: ``NULL``. + Initialize the preconfiguration with :ref:`Isolated Configuration + `. - Part of the :ref:`Python Path Configuration ` output. + .. c:namespace:: PyPreConfig - See also :c:member:`PyConfig.exec_prefix`. + Structure fields: - .. c:member:: wchar_t* base_executable + .. c:member:: int allocator - Python base executable: :data:`sys._base_executable`. + Name of the Python memory allocators: - Set by the :envvar:`__PYVENV_LAUNCHER__` environment variable. + * ``PYMEM_ALLOCATOR_NOT_SET`` (``0``): don't change memory allocators + (use defaults). + * ``PYMEM_ALLOCATOR_DEFAULT`` (``1``): :ref:`default memory allocators + `. + * ``PYMEM_ALLOCATOR_DEBUG`` (``2``): :ref:`default memory allocators + ` with :ref:`debug hooks + `. + * ``PYMEM_ALLOCATOR_MALLOC`` (``3``): use ``malloc()`` of the C library. + * ``PYMEM_ALLOCATOR_MALLOC_DEBUG`` (``4``): force usage of + ``malloc()`` with :ref:`debug hooks `. + * ``PYMEM_ALLOCATOR_PYMALLOC`` (``5``): :ref:`Python pymalloc memory + allocator `. + * ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` (``6``): :ref:`Python pymalloc + memory allocator ` with :ref:`debug hooks + `. + * ``PYMEM_ALLOCATOR_MIMALLOC`` (``6``): use ``mimalloc``, a fast + malloc replacement. + * ``PYMEM_ALLOCATOR_MIMALLOC_DEBUG`` (``7``): use ``mimalloc``, a fast + malloc replacement with :ref:`debug hooks `. - Set from :c:member:`PyConfig.executable` if ``NULL``. - Default: ``NULL``. + ``PYMEM_ALLOCATOR_PYMALLOC`` and ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` are + not supported if Python is :option:`configured using --without-pymalloc + <--without-pymalloc>`. - Part of the :ref:`Python Path Configuration ` output. + ``PYMEM_ALLOCATOR_MIMALLOC`` and ``PYMEM_ALLOCATOR_MIMALLOC_DEBUG`` are + not supported if Python is :option:`configured using --without-mimalloc + <--without-mimalloc>` or if the underlying atomic support isn't + available. - See also :c:member:`PyConfig.executable`. + See :ref:`Memory Management `. - .. c:member:: wchar_t* base_prefix + Default: ``PYMEM_ALLOCATOR_NOT_SET``. - :data:`sys.base_prefix`. + .. c:member:: int configure_locale - Default: ``NULL``. + Set the LC_CTYPE locale to the user preferred locale. - Part of the :ref:`Python Path Configuration ` output. + If equals to ``0``, set :c:member:`~PyPreConfig.coerce_c_locale` and + :c:member:`~PyPreConfig.coerce_c_locale_warn` members to ``0``. - See also :c:member:`PyConfig.prefix`. + See the :term:`locale encoding`. - .. c:member:: int buffered_stdio + Default: ``1`` in Python config, ``0`` in isolated config. - If equals to ``0`` and :c:member:`~PyConfig.configure_c_stdio` is non-zero, - disable buffering on the C streams stdout and stderr. + .. c:member:: int coerce_c_locale - Set to ``0`` by the :option:`-u` command line option and the - :envvar:`PYTHONUNBUFFERED` environment variable. + If equals to ``2``, coerce the C locale. - stdin is always opened in buffered mode. + If equals to ``1``, read the LC_CTYPE locale to decide if it should be + coerced. - Default: ``1``. + See the :term:`locale encoding`. - .. c:member:: int bytes_warning + Default: ``-1`` in Python config, ``0`` in isolated config. - If equals to ``1``, issue a warning when comparing :class:`bytes` or - :class:`bytearray` with :class:`str`, or comparing :class:`bytes` with - :class:`int`. + .. c:member:: int coerce_c_locale_warn - If equal or greater to ``2``, raise a :exc:`BytesWarning` exception in these - cases. + If non-zero, emit a warning if the C locale is coerced. - Incremented by the :option:`-b` command line option. + Default: ``-1`` in Python config, ``0`` in isolated config. - Default: ``0``. + .. c:member:: int dev_mode - .. c:member:: int warn_default_encoding + :ref:`Python Development Mode `: see + :c:member:`PyConfig.dev_mode`. - If non-zero, emit a :exc:`EncodingWarning` warning when :class:`io.TextIOWrapper` - uses its default encoding. See :ref:`io-encoding-warning` for details. + Default: ``-1`` in Python mode, ``0`` in isolated mode. - Default: ``0``. + .. c:member:: int isolated - .. versionadded:: 3.10 + Isolated mode: see :c:member:`PyConfig.isolated`. - .. c:member:: int code_debug_ranges + Default: ``0`` in Python mode, ``1`` in isolated mode. - If equals to ``0``, disables the inclusion of the end line and column - mappings in code objects. Also disables traceback printing carets to - specific error locations. + .. c:member:: int legacy_windows_fs_encoding - Set to ``0`` by the :envvar:`PYTHONNODEBUGRANGES` environment variable - and by the :option:`-X no_debug_ranges <-X>` command line option. + If non-zero: - Default: ``1``. + * Set :c:member:`PyPreConfig.utf8_mode` to ``0``, + * Set :c:member:`PyConfig.filesystem_encoding` to ``"mbcs"``, + * Set :c:member:`PyConfig.filesystem_errors` to ``"replace"``. - .. versionadded:: 3.11 + Initialized from the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + variable value. - .. c:member:: wchar_t* check_hash_pycs_mode + Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for + Windows specific code. - Control the validation behavior of hash-based ``.pyc`` files: - value of the :option:`--check-hash-based-pycs` command line option. + Default: ``0``. - Valid values: + .. c:member:: int parse_argv - - ``L"always"``: Hash the source file for invalidation regardless of - value of the 'check_source' flag. - - ``L"never"``: Assume that hash-based pycs always are valid. - - ``L"default"``: The 'check_source' flag in hash-based pycs - determines invalidation. + If non-zero, :c:func:`Py_PreInitializeFromArgs` and + :c:func:`Py_PreInitializeFromBytesArgs` parse their ``argv`` argument the + same way the regular Python parses command line arguments: see + :ref:`Command Line Arguments `. - Default: ``L"default"``. + Default: ``1`` in Python config, ``0`` in isolated config. - See also :pep:`552` "Deterministic pycs". + .. c:member:: int use_environment - .. c:member:: int configure_c_stdio + Use :ref:`environment variables `? See + :c:member:`PyConfig.use_environment`. - If non-zero, configure C standard streams: + Default: ``1`` in Python config and ``0`` in isolated config. - * On Windows, set the binary mode (``O_BINARY``) on stdin, stdout and - stderr. - * If :c:member:`~PyConfig.buffered_stdio` equals zero, disable buffering - of stdin, stdout and stderr streams. - * If :c:member:`~PyConfig.interactive` is non-zero, enable stream - buffering on stdin and stdout (only stdout on Windows). + .. c:member:: int utf8_mode - Default: ``1`` in Python config, ``0`` in isolated config. + If non-zero, enable the :ref:`Python UTF-8 Mode `. - .. c:member:: int dev_mode + Set to ``0`` or ``1`` by the :option:`-X utf8 <-X>` command line option + and the :envvar:`PYTHONUTF8` environment variable. - If non-zero, enable the :ref:`Python Development Mode `. + Default: ``1``. - Set to ``1`` by the :option:`-X dev <-X>` option and the - :envvar:`PYTHONDEVMODE` environment variable. - Default: ``-1`` in Python mode, ``0`` in isolated mode. +.. _c-preinit: - .. c:member:: int dump_refs +Preinitialize Python with PyPreConfig +------------------------------------- - Dump Python references? +The preinitialization of Python: - If non-zero, dump all objects which are still alive at exit. +* Set the Python memory allocators (:c:member:`PyPreConfig.allocator`) +* Configure the LC_CTYPE locale (:term:`locale encoding`) +* Set the :ref:`Python UTF-8 Mode ` + (:c:member:`PyPreConfig.utf8_mode`) - Set to ``1`` by the :envvar:`PYTHONDUMPREFS` environment variable. +The current preconfiguration (``PyPreConfig`` type) is stored in +``_PyRuntime.preconfig``. - Needs a special build of Python with the ``Py_TRACE_REFS`` macro defined: - see the :option:`configure --with-trace-refs option <--with-trace-refs>`. +Functions to preinitialize Python: - Default: ``0``. +.. c:function:: PyStatus Py_PreInitialize(const PyPreConfig *preconfig) - .. c:member:: wchar_t* exec_prefix + Preinitialize Python from *preconfig* preconfiguration. - The site-specific directory prefix where the platform-dependent Python - files are installed: :data:`sys.exec_prefix`. + *preconfig* must not be ``NULL``. - Default: ``NULL``. +.. c:function:: PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *preconfig, int argc, char * const *argv) - Part of the :ref:`Python Path Configuration ` output. + Preinitialize Python from *preconfig* preconfiguration. - See also :c:member:`PyConfig.base_exec_prefix`. + Parse *argv* command line arguments (bytes strings) if + :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. - .. c:member:: wchar_t* executable + *preconfig* must not be ``NULL``. - The absolute path of the executable binary for the Python interpreter: - :data:`sys.executable`. +.. c:function:: PyStatus Py_PreInitializeFromArgs(const PyPreConfig *preconfig, int argc, wchar_t * const * argv) - Default: ``NULL``. + Preinitialize Python from *preconfig* preconfiguration. - Part of the :ref:`Python Path Configuration ` output. + Parse *argv* command line arguments (wide strings) if + :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. - See also :c:member:`PyConfig.base_executable`. + *preconfig* must not be ``NULL``. - .. c:member:: int faulthandler +The caller is responsible to handle exceptions (error or exit) using +:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. - Enable faulthandler? +For :ref:`Python Configuration ` +(:c:func:`PyPreConfig_InitPythonConfig`), if Python is initialized with +command line arguments, the command line arguments must also be passed to +preinitialize Python, since they have an effect on the pre-configuration +like encodings. For example, the :option:`-X utf8 <-X>` command line option +enables the :ref:`Python UTF-8 Mode `. - If non-zero, call :func:`faulthandler.enable` at startup. +``PyMem_SetAllocator()`` can be called after :c:func:`Py_PreInitialize` and +before :c:func:`Py_InitializeFromConfig` to install a custom memory allocator. +It can be called before :c:func:`Py_PreInitialize` if +:c:member:`PyPreConfig.allocator` is set to ``PYMEM_ALLOCATOR_NOT_SET``. - Set to ``1`` by :option:`-X faulthandler <-X>` and the - :envvar:`PYTHONFAULTHANDLER` environment variable. +Python memory allocation functions like :c:func:`PyMem_RawMalloc` must not be +used before the Python preinitialization, whereas calling directly ``malloc()`` +and ``free()`` is always safe. :c:func:`Py_DecodeLocale` must not be called +before the Python preinitialization. - Default: ``-1`` in Python mode, ``0`` in isolated mode. +Example using the preinitialization to enable +the :ref:`Python UTF-8 Mode `:: - .. c:member:: wchar_t* filesystem_encoding + PyStatus status; + PyPreConfig preconfig; + PyPreConfig_InitPythonConfig(&preconfig); - :term:`Filesystem encoding `: - :func:`sys.getfilesystemencoding`. + preconfig.utf8_mode = 1; - On macOS, Android and VxWorks: use ``"utf-8"`` by default. + status = Py_PreInitialize(&preconfig); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } - On Windows: use ``"utf-8"`` by default, or ``"mbcs"`` if - :c:member:`~PyPreConfig.legacy_windows_fs_encoding` of - :c:type:`PyPreConfig` is non-zero. + /* at this point, Python speaks UTF-8 */ - Default encoding on other platforms: + Py_Initialize(); + /* ... use Python API here ... */ + Py_Finalize(); - * ``"utf-8"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero. - * ``"ascii"`` if Python detects that ``nl_langinfo(CODESET)`` announces - the ASCII encoding, whereas the ``mbstowcs()`` function - decodes from a different encoding (usually Latin1). - * ``"utf-8"`` if ``nl_langinfo(CODESET)`` returns an empty string. - * Otherwise, use the :term:`locale encoding`: - ``nl_langinfo(CODESET)`` result. - At Python startup, the encoding name is normalized to the Python codec - name. For example, ``"ANSI_X3.4-1968"`` is replaced with ``"ascii"``. +PyConfig +-------- - See also the :c:member:`~PyConfig.filesystem_errors` member. +.. c:type:: PyConfig - .. c:member:: wchar_t* filesystem_errors + Structure containing most parameters to configure Python. - :term:`Filesystem error handler `: - :func:`sys.getfilesystemencodeerrors`. + When done, the :c:func:`PyConfig_Clear` function must be used to release the + configuration memory. - On Windows: use ``"surrogatepass"`` by default, or ``"replace"`` if - :c:member:`~PyPreConfig.legacy_windows_fs_encoding` of - :c:type:`PyPreConfig` is non-zero. + .. c:namespace:: NULL - On other platforms: use ``"surrogateescape"`` by default. + Structure methods: - Supported error handlers: + .. c:function:: void PyConfig_InitPythonConfig(PyConfig *config) - * ``"strict"`` - * ``"surrogateescape"`` - * ``"surrogatepass"`` (only supported with the UTF-8 encoding) + Initialize configuration with the :ref:`Python Configuration + `. - See also the :c:member:`~PyConfig.filesystem_encoding` member. + .. c:function:: void PyConfig_InitIsolatedConfig(PyConfig *config) - .. c:member:: unsigned long hash_seed - .. c:member:: int use_hash_seed + Initialize configuration with the :ref:`Isolated Configuration + `. - Randomized hash function seed. + .. c:function:: PyStatus PyConfig_SetString(PyConfig *config, wchar_t * const *config_str, const wchar_t *str) - If :c:member:`~PyConfig.use_hash_seed` is zero, a seed is chosen randomly - at Python startup, and :c:member:`~PyConfig.hash_seed` is ignored. + Copy the wide character string *str* into ``*config_str``. - Set by the :envvar:`PYTHONHASHSEED` environment variable. + :ref:`Preinitialize Python ` if needed. - Default *use_hash_seed* value: ``-1`` in Python mode, ``0`` in isolated - mode. + .. c:function:: PyStatus PyConfig_SetBytesString(PyConfig *config, wchar_t * const *config_str, const char *str) - .. c:member:: wchar_t* home + Decode *str* using :c:func:`Py_DecodeLocale` and set the result into + ``*config_str``. - Set the default Python "home" directory, that is, the location of the - standard Python libraries (see :envvar:`PYTHONHOME`). + :ref:`Preinitialize Python ` if needed. - Set by the :envvar:`PYTHONHOME` environment variable. + .. c:function:: PyStatus PyConfig_SetArgv(PyConfig *config, int argc, wchar_t * const *argv) - Default: ``NULL``. + Set command line arguments (:c:member:`~PyConfig.argv` member of + *config*) from the *argv* list of wide character strings. - Part of the :ref:`Python Path Configuration ` input. + :ref:`Preinitialize Python ` if needed. - .. c:member:: int import_time + .. c:function:: PyStatus PyConfig_SetBytesArgv(PyConfig *config, int argc, char * const *argv) - If non-zero, profile import time. + Set command line arguments (:c:member:`~PyConfig.argv` member of + *config*) from the *argv* list of bytes strings. Decode bytes using + :c:func:`Py_DecodeLocale`. - Set the ``1`` by the :option:`-X importtime <-X>` option and the - :envvar:`PYTHONPROFILEIMPORTTIME` environment variable. + :ref:`Preinitialize Python ` if needed. - Default: ``0``. + .. c:function:: PyStatus PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, Py_ssize_t length, wchar_t **items) - .. c:member:: int inspect + Set the list of wide strings *list* to *length* and *items*. - Enter interactive mode after executing a script or a command. + :ref:`Preinitialize Python ` if needed. - If greater than ``0``, enable inspect: when a script is passed as first - argument or the -c option is used, enter interactive mode after executing - the script or the command, even when :data:`sys.stdin` does not appear to - be a terminal. + .. c:function:: PyStatus PyConfig_Read(PyConfig *config) - Incremented by the :option:`-i` command line option. Set to ``1`` if the - :envvar:`PYTHONINSPECT` environment variable is non-empty. + Read all Python configuration. - Default: ``0``. + Fields which are already initialized are left unchanged. - .. c:member:: int install_signal_handlers + Fields for :ref:`path configuration ` are no longer + calculated or modified when calling this function, as of Python 3.11. - Install Python signal handlers? + The :c:func:`PyConfig_Read` function only parses + :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` + is set to ``2`` after arguments are parsed. Since Python arguments are + stripped from :c:member:`PyConfig.argv`, parsing arguments twice would + parse the application options as Python options. - Default: ``1`` in Python mode, ``0`` in isolated mode. + :ref:`Preinitialize Python ` if needed. - .. c:member:: int interactive + .. versionchanged:: 3.10 + The :c:member:`PyConfig.argv` arguments are now only parsed once, + :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are + parsed, and arguments are only parsed if + :c:member:`PyConfig.parse_argv` equals ``1``. - If greater than ``0``, enable the interactive mode (REPL). + .. versionchanged:: 3.11 + :c:func:`PyConfig_Read` no longer calculates all paths, and so fields + listed under :ref:`Python Path Configuration ` may + no longer be updated until :c:func:`Py_InitializeFromConfig` is + called. - Incremented by the :option:`-i` command line option. + .. c:function:: void PyConfig_Clear(PyConfig *config) - Default: ``0``. - - .. c:member:: int int_max_str_digits - - Configures the :ref:`integer string conversion length limitation - `. An initial value of ``-1`` means the value will - be taken from the command line or environment or otherwise default to - 4300 (:data:`sys.int_info.default_max_str_digits`). A value of ``0`` - disables the limitation. Values greater than zero but less than 640 - (:data:`sys.int_info.str_digits_check_threshold`) are unsupported and - will produce an error. - - Configured by the :option:`-X int_max_str_digits <-X>` command line - flag or the :envvar:`PYTHONINTMAXSTRDIGITS` environment variable. - - Default: ``-1`` in Python mode. 4300 - (:data:`sys.int_info.default_max_str_digits`) in isolated mode. + Release configuration memory. - .. versionadded:: 3.12 + Most ``PyConfig`` methods :ref:`preinitialize Python ` if needed. + In that case, the Python preinitialization configuration + (:c:type:`PyPreConfig`) in based on the :c:type:`PyConfig`. If configuration + fields which are in common with :c:type:`PyPreConfig` are tuned, they must + be set before calling a :c:type:`PyConfig` method: - .. c:member:: int cpu_count + * :c:member:`PyConfig.dev_mode` + * :c:member:`PyConfig.isolated` + * :c:member:`PyConfig.parse_argv` + * :c:member:`PyConfig.use_environment` - If the value of :c:member:`~PyConfig.cpu_count` is not ``-1`` then it will - override the return values of :func:`os.cpu_count`, - :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. + Moreover, if :c:func:`PyConfig_SetArgv` or :c:func:`PyConfig_SetBytesArgv` + is used, this method must be called before other methods, since the + preinitialization configuration depends on command line arguments (if + :c:member:`~PyConfig.parse_argv` is non-zero). - Configured by the :samp:`-X cpu_count={n|default}` command line - flag or the :envvar:`PYTHON_CPU_COUNT` environment variable. + The caller of these methods is responsible to handle exceptions (error or + exit) using ``PyStatus_Exception()`` and ``Py_ExitStatusException()``. - Default: ``-1``. + .. c:namespace:: PyConfig - .. versionadded:: 3.13 + Structure fields: - .. c:member:: int isolated + .. c:member:: PyWideStringList argv - If greater than ``0``, enable isolated mode: + .. index:: + single: main() + single: argv (in module sys) - * Set :c:member:`~PyConfig.safe_path` to ``1``: - don't prepend a potentially unsafe path to :data:`sys.path` at Python - startup, such as the current directory, the script's directory or an - empty string. - * Set :c:member:`~PyConfig.use_environment` to ``0``: ignore ``PYTHON`` - environment variables. - * Set :c:member:`~PyConfig.user_site_directory` to ``0``: don't add the user - site directory to :data:`sys.path`. - * Python REPL doesn't import :mod:`readline` nor enable default readline - configuration on interactive prompts. + Set :data:`sys.argv` command line arguments based on + :c:member:`~PyConfig.argv`. These parameters are similar to those passed + to the program's :c:func:`main` function with the difference that the + first entry should refer to the script file to be executed rather than + the executable hosting the Python interpreter. If there isn't a script + that will be run, the first entry in :c:member:`~PyConfig.argv` can be an + empty string. - Set to ``1`` by the :option:`-I` command line option. + Set :c:member:`~PyConfig.parse_argv` to ``1`` to parse + :c:member:`~PyConfig.argv` the same way the regular Python parses Python + command line arguments and then to strip Python arguments from + :c:member:`~PyConfig.argv`. - Default: ``0`` in Python mode, ``1`` in isolated mode. + If :c:member:`~PyConfig.argv` is empty, an empty string is added to + ensure that :data:`sys.argv` always exists and is never empty. - See also the :ref:`Isolated Configuration ` and - :c:member:`PyPreConfig.isolated`. + Default: ``NULL``. - .. c:member:: int legacy_windows_stdio + See also the :c:member:`~PyConfig.orig_argv` member. - If non-zero, use :class:`io.FileIO` instead of - :class:`!io._WindowsConsoleIO` for :data:`sys.stdin`, :data:`sys.stdout` - and :data:`sys.stderr`. + .. c:member:: int safe_path - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment - variable is set to a non-empty string. + If equals to zero, ``Py_RunMain()`` prepends a potentially unsafe path to + :data:`sys.path` at startup: - Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for - Windows specific code. + * If :c:member:`argv[0] ` is equal to ``L"-m"`` + (``python -m module``), prepend the current working directory. + * If running a script (``python script.py``), prepend the script's + directory. If it's a symbolic link, resolve symbolic links. + * Otherwise (``python -c code`` and ``python``), prepend an empty string, + which means the current working directory. - Default: ``0``. + Set to ``1`` by the :option:`-P` command line option and the + :envvar:`PYTHONSAFEPATH` environment variable. - See also the :pep:`528` (Change Windows console encoding to UTF-8). + Default: ``0`` in Python config, ``1`` in isolated config. - .. c:member:: int malloc_stats + .. versionadded:: 3.11 - If non-zero, dump statistics on :ref:`Python pymalloc memory allocator - ` at exit. + .. c:member:: wchar_t* base_exec_prefix - Set to ``1`` by the :envvar:`PYTHONMALLOCSTATS` environment variable. + :data:`sys.base_exec_prefix`. - The option is ignored if Python is :option:`configured using - the --without-pymalloc option <--without-pymalloc>`. + Default: ``NULL``. - Default: ``0``. + Part of the :ref:`Python Path Configuration ` output. - .. c:member:: wchar_t* platlibdir + See also :c:member:`PyConfig.exec_prefix`. - Platform library directory name: :data:`sys.platlibdir`. + .. c:member:: wchar_t* base_executable - Set by the :envvar:`PYTHONPLATLIBDIR` environment variable. + Python base executable: :data:`sys._base_executable`. - Default: value of the ``PLATLIBDIR`` macro which is set by the - :option:`configure --with-platlibdir option <--with-platlibdir>` - (default: ``"lib"``, or ``"DLLs"`` on Windows). + Set by the :envvar:`__PYVENV_LAUNCHER__` environment variable. - Part of the :ref:`Python Path Configuration ` input. + Set from :c:member:`PyConfig.executable` if ``NULL``. - .. versionadded:: 3.9 + Default: ``NULL``. - .. versionchanged:: 3.11 - This macro is now used on Windows to locate the standard - library extension modules, typically under ``DLLs``. However, - for compatibility, note that this value is ignored for any - non-standard layouts, including in-tree builds and virtual - environments. + Part of the :ref:`Python Path Configuration ` output. - .. c:member:: wchar_t* pythonpath_env + See also :c:member:`PyConfig.executable`. - Module search paths (:data:`sys.path`) as a string separated by ``DELIM`` - (:data:`os.pathsep`). + .. c:member:: wchar_t* base_prefix - Set by the :envvar:`PYTHONPATH` environment variable. + :data:`sys.base_prefix`. Default: ``NULL``. - Part of the :ref:`Python Path Configuration ` input. + Part of the :ref:`Python Path Configuration ` output. - .. c:member:: PyWideStringList module_search_paths - .. c:member:: int module_search_paths_set + See also :c:member:`PyConfig.prefix`. - Module search paths: :data:`sys.path`. + .. c:member:: int buffered_stdio - If :c:member:`~PyConfig.module_search_paths_set` is equal to ``0``, - :c:func:`Py_InitializeFromConfig` will replace - :c:member:`~PyConfig.module_search_paths` and sets - :c:member:`~PyConfig.module_search_paths_set` to ``1``. + If equals to ``0`` and :c:member:`~PyConfig.configure_c_stdio` is non-zero, + disable buffering on the C streams stdout and stderr. - Default: empty list (``module_search_paths``) and ``0`` - (``module_search_paths_set``). + Set to ``0`` by the :option:`-u` command line option and the + :envvar:`PYTHONUNBUFFERED` environment variable. - Part of the :ref:`Python Path Configuration ` output. + stdin is always opened in buffered mode. - .. c:member:: int optimization_level + Default: ``1``. - Compilation optimization level: + .. c:member:: int bytes_warning - * ``0``: Peephole optimizer, set ``__debug__`` to ``True``. - * ``1``: Level 0, remove assertions, set ``__debug__`` to ``False``. - * ``2``: Level 1, strip docstrings. + If equals to ``1``, issue a warning when comparing :class:`bytes` or + :class:`bytearray` with :class:`str`, or comparing :class:`bytes` with + :class:`int`. - Incremented by the :option:`-O` command line option. Set to the - :envvar:`PYTHONOPTIMIZE` environment variable value. + If equal or greater to ``2``, raise a :exc:`BytesWarning` exception in these + cases. + + Incremented by the :option:`-b` command line option. Default: ``0``. - .. c:member:: PyWideStringList orig_argv + .. deprecated-removed:: 3.15 3.17 - The list of the original command line arguments passed to the Python - executable: :data:`sys.orig_argv`. + The :option:`-b` and :option:`!-bb` options will become no-op in 3.17. + :c:member:`~PyConfig.bytes_warning` member will be removed in 3.17. - If :c:member:`~PyConfig.orig_argv` list is empty and - :c:member:`~PyConfig.argv` is not a list only containing an empty - string, :c:func:`PyConfig_Read` copies :c:member:`~PyConfig.argv` into - :c:member:`~PyConfig.orig_argv` before modifying - :c:member:`~PyConfig.argv` (if :c:member:`~PyConfig.parse_argv` is - non-zero). + .. c:member:: int warn_default_encoding - See also the :c:member:`~PyConfig.argv` member and the - :c:func:`Py_GetArgcArgv` function. + If non-zero, emit a :exc:`EncodingWarning` warning when :class:`io.TextIOWrapper` + uses its default encoding. See :ref:`io-encoding-warning` for details. - Default: empty list. + Default: ``0``. .. versionadded:: 3.10 - .. c:member:: int parse_argv + .. c:member:: int code_debug_ranges - Parse command line arguments? + If equals to ``0``, disables the inclusion of the end line and column + mappings in code objects. Also disables traceback printing carets to + specific error locations. - If equals to ``1``, parse :c:member:`~PyConfig.argv` the same way the regular - Python parses :ref:`command line arguments `, and strip - Python arguments from :c:member:`~PyConfig.argv`. + Set to ``0`` by the :envvar:`PYTHONNODEBUGRANGES` environment variable + and by the :option:`-X no_debug_ranges <-X>` command line option. - The :c:func:`PyConfig_Read` function only parses - :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` - is set to ``2`` after arguments are parsed. Since Python arguments are - stripped from :c:member:`PyConfig.argv`, parsing arguments twice would - parse the application options as Python options. + Default: ``1``. - Default: ``1`` in Python mode, ``0`` in isolated mode. + .. versionadded:: 3.11 - .. versionchanged:: 3.10 - The :c:member:`PyConfig.argv` arguments are now only parsed if - :c:member:`PyConfig.parse_argv` equals to ``1``. + .. c:member:: wchar_t* check_hash_pycs_mode - .. c:member:: int parser_debug + Control the validation behavior of hash-based ``.pyc`` files: + value of the :option:`--check-hash-based-pycs` command line option. - Parser debug mode. If greater than ``0``, turn on parser debugging output (for expert only, depending - on compilation options). + Valid values: - Incremented by the :option:`-d` command line option. Set to the - :envvar:`PYTHONDEBUG` environment variable value. + - ``L"always"``: Hash the source file for invalidation regardless of + value of the 'check_source' flag. + - ``L"never"``: Assume that hash-based pycs always are valid. + - ``L"default"``: The 'check_source' flag in hash-based pycs + determines invalidation. - Needs a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro - must be defined). + Default: ``L"default"``. - Default: ``0``. + See also :pep:`552` "Deterministic pycs". - .. c:member:: int pathconfig_warnings + .. c:member:: int configure_c_stdio - If non-zero, calculation of path configuration is allowed to log - warnings into ``stderr``. If equals to ``0``, suppress these warnings. + If non-zero, configure C standard streams: - Default: ``1`` in Python mode, ``0`` in isolated mode. + * On Windows, set the binary mode (``O_BINARY``) on stdin, stdout and + stderr. + * If :c:member:`~PyConfig.buffered_stdio` equals zero, disable buffering + of stdin, stdout and stderr streams. + * If :c:member:`~PyConfig.interactive` is non-zero, enable stream + buffering on stdin and stdout (only stdout on Windows). - Part of the :ref:`Python Path Configuration ` input. + Default: ``1`` in Python config, ``0`` in isolated config. - .. versionchanged:: 3.11 - Now also applies on Windows. + .. c:member:: int dev_mode - .. c:member:: wchar_t* prefix + If non-zero, enable the :ref:`Python Development Mode `. - The site-specific directory prefix where the platform independent Python - files are installed: :data:`sys.prefix`. + Set to ``1`` by the :option:`-X dev <-X>` option and the + :envvar:`PYTHONDEVMODE` environment variable. - Default: ``NULL``. - - Part of the :ref:`Python Path Configuration ` output. - - See also :c:member:`PyConfig.base_prefix`. + Default: ``-1`` in Python mode, ``0`` in isolated mode. - .. c:member:: wchar_t* program_name + .. c:member:: int dump_refs - Program name used to initialize :c:member:`~PyConfig.executable` and in - early error messages during Python initialization. + Dump Python references? - * On macOS, use :envvar:`PYTHONEXECUTABLE` environment variable if set. - * If the ``WITH_NEXT_FRAMEWORK`` macro is defined, use - :envvar:`__PYVENV_LAUNCHER__` environment variable if set. - * Use ``argv[0]`` of :c:member:`~PyConfig.argv` if available and - non-empty. - * Otherwise, use ``L"python"`` on Windows, or ``L"python3"`` on other - platforms. + If non-zero, dump all objects which are still alive at exit. - Default: ``NULL``. + Set to ``1`` by the :envvar:`PYTHONDUMPREFS` environment variable. - Part of the :ref:`Python Path Configuration ` input. + Needs a special build of Python with the ``Py_TRACE_REFS`` macro defined: + see the :option:`configure --with-trace-refs option <--with-trace-refs>`. - .. c:member:: wchar_t* pycache_prefix + Default: ``0``. - Directory where cached ``.pyc`` files are written: - :data:`sys.pycache_prefix`. + .. c:member:: wchar_t* dump_refs_file - Set by the :option:`-X pycache_prefix=PATH <-X>` command line option and - the :envvar:`PYTHONPYCACHEPREFIX` environment variable. - The command-line option takes precedence. + Filename where to dump Python references. - If ``NULL``, :data:`sys.pycache_prefix` is set to ``None``. + Set by the :envvar:`PYTHONDUMPREFSFILE` environment variable. Default: ``NULL``. - .. c:member:: int quiet + .. versionadded:: 3.11 - Quiet mode. If greater than ``0``, don't display the copyright and version at - Python startup in interactive mode. + .. c:member:: wchar_t* exec_prefix - Incremented by the :option:`-q` command line option. + The site-specific directory prefix where the platform-dependent Python + files are installed: :data:`sys.exec_prefix`. - Default: ``0``. + Default: ``NULL``. - .. c:member:: wchar_t* run_command + Part of the :ref:`Python Path Configuration ` output. - Value of the :option:`-c` command line option. + See also :c:member:`PyConfig.base_exec_prefix`. - Used by :c:func:`Py_RunMain`. + .. c:member:: wchar_t* executable + + The absolute path of the executable binary for the Python interpreter: + :data:`sys.executable`. Default: ``NULL``. - .. c:member:: wchar_t* run_filename + Part of the :ref:`Python Path Configuration ` output. - Filename passed on the command line: trailing command line argument - without :option:`-c` or :option:`-m`. It is used by the - :c:func:`Py_RunMain` function. + See also :c:member:`PyConfig.base_executable`. - For example, it is set to ``script.py`` by the ``python3 script.py arg`` - command line. + .. c:member:: int faulthandler - See also the :c:member:`PyConfig.skip_source_first_line` option. + Enable faulthandler? - Default: ``NULL``. + If non-zero, call :func:`faulthandler.enable` at startup. - .. c:member:: wchar_t* run_module + Set to ``1`` by :option:`-X faulthandler <-X>` and the + :envvar:`PYTHONFAULTHANDLER` environment variable. - Value of the :option:`-m` command line option. + Default: ``-1`` in Python mode, ``0`` in isolated mode. - Used by :c:func:`Py_RunMain`. + .. c:member:: wchar_t* filesystem_encoding - Default: ``NULL``. + :term:`Filesystem encoding `: + :func:`sys.getfilesystemencoding`. - .. c:member:: wchar_t* run_presite + On macOS, Android and VxWorks: use ``"utf-8"`` by default. - ``package.module`` path to module that should be imported before - ``site.py`` is run. + On Windows: use ``"utf-8"`` by default, or ``"mbcs"`` if + :c:member:`~PyPreConfig.legacy_windows_fs_encoding` of + :c:type:`PyPreConfig` is non-zero. - Set by the :option:`-X presite=package.module <-X>` command-line - option and the :envvar:`PYTHON_PRESITE` environment variable. - The command-line option takes precedence. + Default encoding on other platforms: - Needs a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro - must be defined). + * ``"utf-8"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero. + * ``"ascii"`` if Python detects that ``nl_langinfo(CODESET)`` announces + the ASCII encoding, whereas the ``mbstowcs()`` function + decodes from a different encoding (usually Latin1). + * ``"utf-8"`` if ``nl_langinfo(CODESET)`` returns an empty string. + * Otherwise, use the :term:`locale encoding`: + ``nl_langinfo(CODESET)`` result. - Default: ``NULL``. + At Python startup, the encoding name is normalized to the Python codec + name. For example, ``"ANSI_X3.4-1968"`` is replaced with ``"ascii"``. - .. c:member:: int show_ref_count + See also the :c:member:`~PyConfig.filesystem_errors` member. - Show total reference count at exit (excluding :term:`immortal` objects)? + .. c:member:: wchar_t* filesystem_errors - Set to ``1`` by :option:`-X showrefcount <-X>` command line option. + :term:`Filesystem error handler `: + :func:`sys.getfilesystemencodeerrors`. - Needs a :ref:`debug build of Python ` (the ``Py_REF_DEBUG`` - macro must be defined). + On Windows: use ``"surrogatepass"`` by default, or ``"replace"`` if + :c:member:`~PyPreConfig.legacy_windows_fs_encoding` of + :c:type:`PyPreConfig` is non-zero. - Default: ``0``. + On other platforms: use ``"surrogateescape"`` by default. - .. c:member:: int site_import + Supported error handlers: - Import the :mod:`site` module at startup? + * ``"strict"`` + * ``"surrogateescape"`` + * ``"surrogatepass"`` (only supported with the UTF-8 encoding) - If equal to zero, disable the import of the module site and the - site-dependent manipulations of :data:`sys.path` that it entails. + See also the :c:member:`~PyConfig.filesystem_encoding` member. - Also disable these manipulations if the :mod:`site` module is explicitly - imported later (call :func:`site.main` if you want them to be triggered). + .. c:member:: int use_frozen_modules - Set to ``0`` by the :option:`-S` command line option. + If non-zero, use frozen modules. - :data:`sys.flags.no_site ` is set to the inverted value of - :c:member:`~PyConfig.site_import`. + Set by the :envvar:`PYTHON_FROZEN_MODULES` environment variable. - Default: ``1``. + Default: ``1`` in a release build, or ``0`` in a :ref:`debug build + `. - .. c:member:: int skip_source_first_line + .. c:member:: unsigned long hash_seed + .. c:member:: int use_hash_seed - If non-zero, skip the first line of the :c:member:`PyConfig.run_filename` - source. + Randomized hash function seed. - It allows the usage of non-Unix forms of ``#!cmd``. This is intended for - a DOS specific hack only. + If :c:member:`~PyConfig.use_hash_seed` is zero, a seed is chosen randomly + at Python startup, and :c:member:`~PyConfig.hash_seed` is ignored. - Set to ``1`` by the :option:`-x` command line option. + Set by the :envvar:`PYTHONHASHSEED` environment variable. - Default: ``0``. + Default *use_hash_seed* value: ``-1`` in Python mode, ``0`` in isolated + mode. - .. c:member:: wchar_t* stdio_encoding - .. c:member:: wchar_t* stdio_errors + .. c:member:: wchar_t* home - Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and - :data:`sys.stderr` (but :data:`sys.stderr` always uses - ``"backslashreplace"`` error handler). + Set the default Python "home" directory, that is, the location of the + standard Python libraries (see :envvar:`PYTHONHOME`). - Use the :envvar:`PYTHONIOENCODING` environment variable if it is - non-empty. + Set by the :envvar:`PYTHONHOME` environment variable. - Default encoding: + Default: ``NULL``. - * ``"UTF-8"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero. - * Otherwise, use the :term:`locale encoding`. + Part of the :ref:`Python Path Configuration ` input. - Default error handler: + .. c:member:: int import_time - * On Windows: use ``"surrogateescape"``. - * ``"surrogateescape"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero, - or if the LC_CTYPE locale is "C" or "POSIX". - * ``"strict"`` otherwise. + If ``1``, profile import time. + If ``2``, include additional output that indicates + when an imported module has already been loaded. - See also :c:member:`PyConfig.legacy_windows_stdio`. + Set by the :option:`-X importtime <-X>` option and the + :envvar:`PYTHONPROFILEIMPORTTIME` environment variable. - .. c:member:: int tracemalloc + Default: ``0``. - Enable tracemalloc? + .. versionchanged:: 3.14 - If non-zero, call :func:`tracemalloc.start` at startup. + Added support for ``import_time = 2`` - Set by :option:`-X tracemalloc=N <-X>` command line option and by the - :envvar:`PYTHONTRACEMALLOC` environment variable. + .. c:member:: int inspect - Default: ``-1`` in Python mode, ``0`` in isolated mode. + Enter interactive mode after executing a script or a command. - .. c:member:: int perf_profiling + If greater than ``0``, enable inspect: when a script is passed as first + argument or the -c option is used, enter interactive mode after executing + the script or the command, even when :data:`sys.stdin` does not appear to + be a terminal. - Enable the Linux ``perf`` profiler support? + Incremented by the :option:`-i` command line option. Set to ``1`` if the + :envvar:`PYTHONINSPECT` environment variable is non-empty. - If equals to ``1``, enable support for the Linux ``perf`` profiler. + Default: ``0``. - If equals to ``2``, enable support for the Linux ``perf`` profiler with - DWARF JIT support. + .. c:member:: int install_signal_handlers - Set to ``1`` by :option:`-X perf <-X>` command-line option and the - :envvar:`PYTHONPERFSUPPORT` environment variable. + Install Python signal handlers? - Set to ``2`` by the :option:`-X perf_jit <-X>` command-line option and - the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable. + Default: ``1`` in Python mode, ``0`` in isolated mode. - Default: ``-1``. + .. c:member:: int interactive - .. seealso:: - See :ref:`perf_profiling` for more information. + If greater than ``0``, enable the interactive mode (REPL). - .. versionadded:: 3.12 + Incremented by the :option:`-i` command line option. - .. c:member:: int use_environment + Default: ``0``. - Use :ref:`environment variables `? + .. c:member:: int int_max_str_digits - If equals to zero, ignore the :ref:`environment variables - `. + Configures the :ref:`integer string conversion length limitation + `. An initial value of ``-1`` means the value will + be taken from the command line or environment or otherwise default to + 4300 (:data:`sys.int_info.default_max_str_digits`). A value of ``0`` + disables the limitation. Values greater than zero but less than 640 + (:data:`sys.int_info.str_digits_check_threshold`) are unsupported and + will produce an error. - Set to ``0`` by the :option:`-E` environment variable. + Configured by the :option:`-X int_max_str_digits <-X>` command line + flag or the :envvar:`PYTHONINTMAXSTRDIGITS` environment variable. - Default: ``1`` in Python config and ``0`` in isolated config. + Default: ``-1`` in Python mode. 4300 + (:data:`sys.int_info.default_max_str_digits`) in isolated mode. - .. c:member:: int user_site_directory + .. versionadded:: 3.12 - If non-zero, add the user site directory to :data:`sys.path`. + .. c:member:: int cpu_count - Set to ``0`` by the :option:`-s` and :option:`-I` command line options. + If the value of :c:member:`~PyConfig.cpu_count` is not ``-1`` then it will + override the return values of :func:`os.cpu_count`, + :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. - Set to ``0`` by the :envvar:`PYTHONNOUSERSITE` environment variable. + Configured by the :samp:`-X cpu_count={n|default}` command line + flag or the :envvar:`PYTHON_CPU_COUNT` environment variable. - Default: ``1`` in Python mode, ``0`` in isolated mode. + Default: ``-1``. - .. c:member:: int verbose + .. versionadded:: 3.13 - Verbose mode. If greater than ``0``, print a message each time a module is - imported, showing the place (filename or built-in module) from which - it is loaded. + .. c:member:: int isolated - If greater than or equal to ``2``, print a message for each file that is - checked for when searching for a module. Also provides information on - module cleanup at exit. + If greater than ``0``, enable isolated mode: - Incremented by the :option:`-v` command line option. + * Set :c:member:`~PyConfig.safe_path` to ``1``: + don't prepend a potentially unsafe path to :data:`sys.path` at Python + startup, such as the current directory, the script's directory or an + empty string. + * Set :c:member:`~PyConfig.use_environment` to ``0``: ignore ``PYTHON`` + environment variables. + * Set :c:member:`~PyConfig.user_site_directory` to ``0``: don't add the user + site directory to :data:`sys.path`. + * Python REPL doesn't import :mod:`readline` nor enable default readline + configuration on interactive prompts. - Set by the :envvar:`PYTHONVERBOSE` environment variable value. + Set to ``1`` by the :option:`-I` command line option. - Default: ``0``. + Default: ``0`` in Python mode, ``1`` in isolated mode. - .. c:member:: PyWideStringList warnoptions + See also the :ref:`Isolated Configuration ` and + :c:member:`PyPreConfig.isolated`. - Options of the :mod:`warnings` module to build warnings filters, lowest - to highest priority: :data:`sys.warnoptions`. + .. c:member:: int legacy_windows_stdio - The :mod:`warnings` module adds :data:`sys.warnoptions` in the reverse - order: the last :c:member:`PyConfig.warnoptions` item becomes the first - item of :data:`warnings.filters` which is checked first (highest - priority). + If non-zero, use :class:`io.FileIO` instead of + :class:`!io._WindowsConsoleIO` for :data:`sys.stdin`, :data:`sys.stdout` + and :data:`sys.stderr`. - The :option:`-W` command line options adds its value to - :c:member:`~PyConfig.warnoptions`, it can be used multiple times. + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment + variable is set to a non-empty string. - The :envvar:`PYTHONWARNINGS` environment variable can also be used to add - warning options. Multiple options can be specified, separated by commas - (``,``). + Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for + Windows specific code. - Default: empty list. + Default: ``0``. - .. c:member:: int write_bytecode + See also the :pep:`528` (Change Windows console encoding to UTF-8). - If equal to ``0``, Python won't try to write ``.pyc`` files on the import of - source modules. + .. c:member:: int malloc_stats - Set to ``0`` by the :option:`-B` command line option and the - :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. + If non-zero, dump statistics on :ref:`Python pymalloc memory allocator + ` at exit. - :data:`sys.dont_write_bytecode` is initialized to the inverted value of - :c:member:`~PyConfig.write_bytecode`. + Set to ``1`` by the :envvar:`PYTHONMALLOCSTATS` environment variable. - Default: ``1``. + The option is ignored if Python is :option:`configured using + the --without-pymalloc option <--without-pymalloc>`. - .. c:member:: PyWideStringList xoptions + Default: ``0``. - Values of the :option:`-X` command line options: :data:`sys._xoptions`. + .. c:member:: wchar_t* platlibdir + + Platform library directory name: :data:`sys.platlibdir`. + + Set by the :envvar:`PYTHONPLATLIBDIR` environment variable. + + Default: value of the ``PLATLIBDIR`` macro which is set by the + :option:`configure --with-platlibdir option <--with-platlibdir>` + (default: ``"lib"``, or ``"DLLs"`` on Windows). + + Part of the :ref:`Python Path Configuration ` input. + + .. versionadded:: 3.9 + + .. versionchanged:: 3.11 + This macro is now used on Windows to locate the standard + library extension modules, typically under ``DLLs``. However, + for compatibility, note that this value is ignored for any + non-standard layouts, including in-tree builds and virtual + environments. + + .. c:member:: wchar_t* pythonpath_env + + Module search paths (:data:`sys.path`) as a string separated by ``DELIM`` + (:data:`os.pathsep`). + + Set by the :envvar:`PYTHONPATH` environment variable. + + Default: ``NULL``. + + Part of the :ref:`Python Path Configuration ` input. + + .. c:member:: PyWideStringList module_search_paths + .. c:member:: int module_search_paths_set + + Module search paths: :data:`sys.path`. + + If :c:member:`~PyConfig.module_search_paths_set` is equal to ``0``, + :c:func:`Py_InitializeFromConfig` will replace + :c:member:`~PyConfig.module_search_paths` and sets + :c:member:`~PyConfig.module_search_paths_set` to ``1``. + + Default: empty list (``module_search_paths``) and ``0`` + (``module_search_paths_set``). + + Part of the :ref:`Python Path Configuration ` output. + + .. c:member:: int optimization_level + + Compilation optimization level: + + * ``0``: Peephole optimizer, set ``__debug__`` to ``True``. + * ``1``: Level 0, remove assertions, set ``__debug__`` to ``False``. + * ``2``: Level 1, strip docstrings. + + Incremented by the :option:`-O` command line option. Set to the + :envvar:`PYTHONOPTIMIZE` environment variable value. + + Default: ``0``. + + .. c:member:: PyWideStringList orig_argv + + The list of the original command line arguments passed to the Python + executable: :data:`sys.orig_argv`. + + If :c:member:`~PyConfig.orig_argv` list is empty and + :c:member:`~PyConfig.argv` is not a list only containing an empty + string, :c:func:`PyConfig_Read` copies :c:member:`~PyConfig.argv` into + :c:member:`~PyConfig.orig_argv` before modifying + :c:member:`~PyConfig.argv` (if :c:member:`~PyConfig.parse_argv` is + non-zero). + + See also the :c:member:`~PyConfig.argv` member and the + :c:func:`Py_GetArgcArgv` function. Default: empty list. -If :c:member:`~PyConfig.parse_argv` is non-zero, :c:member:`~PyConfig.argv` -arguments are parsed the same way the regular Python parses :ref:`command line -arguments `, and Python arguments are stripped from -:c:member:`~PyConfig.argv`. + .. versionadded:: 3.10 -The :c:member:`~PyConfig.xoptions` options are parsed to set other options: see -the :option:`-X` command line option. + .. c:member:: int parse_argv -.. versionchanged:: 3.9 + Parse command line arguments? - The ``show_alloc_count`` field has been removed. + If equals to ``1``, parse :c:member:`~PyConfig.argv` the same way the regular + Python parses :ref:`command line arguments `, and strip + Python arguments from :c:member:`~PyConfig.argv`. + The :c:func:`PyConfig_Read` function only parses + :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` + is set to ``2`` after arguments are parsed. Since Python arguments are + stripped from :c:member:`PyConfig.argv`, parsing arguments twice would + parse the application options as Python options. -.. _init-from-config: + Default: ``1`` in Python mode, ``0`` in isolated mode. -Initialization with PyConfig ----------------------------- + .. versionchanged:: 3.10 + The :c:member:`PyConfig.argv` arguments are now only parsed if + :c:member:`PyConfig.parse_argv` equals to ``1``. -Initializing the interpreter from a populated configuration struct is handled -by calling :c:func:`Py_InitializeFromConfig`. + .. c:member:: int parser_debug -The caller is responsible to handle exceptions (error or exit) using -:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. + Parser debug mode. If greater than ``0``, turn on parser debugging output (for expert only, depending + on compilation options). -If :c:func:`PyImport_FrozenModules`, :c:func:`PyImport_AppendInittab` or -:c:func:`PyImport_ExtendInittab` are used, they must be set or called after -Python preinitialization and before the Python initialization. If Python is -initialized multiple times, :c:func:`PyImport_AppendInittab` or -:c:func:`PyImport_ExtendInittab` must be called before each Python -initialization. + Incremented by the :option:`-d` command line option. Set to the + :envvar:`PYTHONDEBUG` environment variable value. -The current configuration (``PyConfig`` type) is stored in -``PyInterpreterState.config``. + Needs a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro + must be defined). -Example setting the program name:: + Default: ``0``. - void init_python(void) - { - PyStatus status; + .. c:member:: int pathconfig_warnings - PyConfig config; - PyConfig_InitPythonConfig(&config); + If non-zero, calculation of path configuration is allowed to log + warnings into ``stderr``. If equals to ``0``, suppress these warnings. - /* Set the program name. Implicitly preinitialize Python. */ - status = PyConfig_SetString(&config, &config.program_name, - L"/path/to/my_program"); - if (PyStatus_Exception(status)) { - goto exception; - } + Default: ``1`` in Python mode, ``0`` in isolated mode. - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - goto exception; - } - PyConfig_Clear(&config); - return; + Part of the :ref:`Python Path Configuration ` input. - exception: - PyConfig_Clear(&config); - Py_ExitStatusException(status); - } + .. versionchanged:: 3.11 + Now also applies on Windows. -More complete example modifying the default configuration, read the -configuration, and then override some parameters. Note that since -3.11, many parameters are not calculated until initialization, and -so values cannot be read from the configuration structure. Any values -set before initialize is called will be left unchanged by -initialization:: + .. c:member:: wchar_t* prefix - PyStatus init_python(const char *program_name) - { - PyStatus status; + The site-specific directory prefix where the platform independent Python + files are installed: :data:`sys.prefix`. - PyConfig config; - PyConfig_InitPythonConfig(&config); + Default: ``NULL``. - /* Set the program name before reading the configuration - (decode byte string from the locale encoding). + Part of the :ref:`Python Path Configuration ` output. - Implicitly preinitialize Python. */ - status = PyConfig_SetBytesString(&config, &config.program_name, - program_name); - if (PyStatus_Exception(status)) { - goto done; - } + See also :c:member:`PyConfig.base_prefix`. - /* Read all configuration at once */ - status = PyConfig_Read(&config); - if (PyStatus_Exception(status)) { - goto done; - } + .. c:member:: wchar_t* program_name - /* Specify sys.path explicitly */ - /* If you want to modify the default set of paths, finish - initialization first and then use PySys_GetObject("path") */ - config.module_search_paths_set = 1; - status = PyWideStringList_Append(&config.module_search_paths, - L"/path/to/stdlib"); - if (PyStatus_Exception(status)) { - goto done; - } - status = PyWideStringList_Append(&config.module_search_paths, - L"/path/to/more/modules"); - if (PyStatus_Exception(status)) { - goto done; - } + Program name used to initialize :c:member:`~PyConfig.executable` and in + early error messages during Python initialization. - /* Override executable computed by PyConfig_Read() */ - status = PyConfig_SetString(&config, &config.executable, - L"/path/to/my_executable"); - if (PyStatus_Exception(status)) { - goto done; - } + * On macOS, use :envvar:`PYTHONEXECUTABLE` environment variable if set. + * If the ``WITH_NEXT_FRAMEWORK`` macro is defined, use + :envvar:`__PYVENV_LAUNCHER__` environment variable if set. + * Use ``argv[0]`` of :c:member:`~PyConfig.argv` if available and + non-empty. + * Otherwise, use ``L"python"`` on Windows, or ``L"python3"`` on other + platforms. - status = Py_InitializeFromConfig(&config); + Default: ``NULL``. - done: - PyConfig_Clear(&config); - return status; - } + Part of the :ref:`Python Path Configuration ` input. + .. c:member:: wchar_t* pycache_prefix -.. _init-isolated-conf: + Directory where cached ``.pyc`` files are written: + :data:`sys.pycache_prefix`. -Isolated Configuration ----------------------- + Set by the :option:`-X pycache_prefix=PATH <-X>` command line option and + the :envvar:`PYTHONPYCACHEPREFIX` environment variable. + The command-line option takes precedence. -:c:func:`PyPreConfig_InitIsolatedConfig` and -:c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to -isolate Python from the system. For example, to embed Python into an -application. + If ``NULL``, :data:`sys.pycache_prefix` is set to ``None``. -This configuration ignores global configuration variables, environment -variables, command line arguments (:c:member:`PyConfig.argv` is not parsed) -and user site directory. The C standard streams (ex: ``stdout``) and the -LC_CTYPE locale are left unchanged. Signal handlers are not installed. + Default: ``NULL``. -Configuration files are still used with this configuration to determine -paths that are unspecified. Ensure :c:member:`PyConfig.home` is specified -to avoid computing the default path configuration. + .. c:member:: int quiet + Quiet mode. If greater than ``0``, don't display the copyright and version at + Python startup in interactive mode. -.. _init-python-config: + Incremented by the :option:`-q` command line option. -Python Configuration --------------------- + Default: ``0``. -:c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` -functions create a configuration to build a customized Python which behaves as -the regular Python. + .. c:member:: wchar_t* run_command -Environments variables and command line arguments are used to configure -Python, whereas global configuration variables are ignored. + Value of the :option:`-c` command line option. -This function enables C locale coercion (:pep:`538`) -and :ref:`Python UTF-8 Mode ` -(:pep:`540`) depending on the LC_CTYPE locale, :envvar:`PYTHONUTF8` and -:envvar:`PYTHONCOERCECLOCALE` environment variables. + Used by :c:func:`Py_RunMain`. + Default: ``NULL``. -.. _init-path-config: + .. c:member:: wchar_t* run_filename -Python Path Configuration -------------------------- + Filename passed on the command line: trailing command line argument + without :option:`-c` or :option:`-m`. It is used by the + :c:func:`Py_RunMain` function. -:c:type:`PyConfig` contains multiple fields for the path configuration: + For example, it is set to ``script.py`` by the ``python3 script.py arg`` + command line. -* Path configuration inputs: + See also the :c:member:`PyConfig.skip_source_first_line` option. - * :c:member:`PyConfig.home` - * :c:member:`PyConfig.platlibdir` - * :c:member:`PyConfig.pathconfig_warnings` - * :c:member:`PyConfig.program_name` - * :c:member:`PyConfig.pythonpath_env` - * current working directory: to get absolute paths - * ``PATH`` environment variable to get the program full path - (from :c:member:`PyConfig.program_name`) - * ``__PYVENV_LAUNCHER__`` environment variable - * (Windows only) Application paths in the registry under - "Software\Python\PythonCore\X.Y\PythonPath" of HKEY_CURRENT_USER and - HKEY_LOCAL_MACHINE (where X.Y is the Python version). + Default: ``NULL``. -* Path configuration output fields: + .. c:member:: wchar_t* run_module - * :c:member:`PyConfig.base_exec_prefix` - * :c:member:`PyConfig.base_executable` - * :c:member:`PyConfig.base_prefix` - * :c:member:`PyConfig.exec_prefix` - * :c:member:`PyConfig.executable` - * :c:member:`PyConfig.module_search_paths_set`, - :c:member:`PyConfig.module_search_paths` - * :c:member:`PyConfig.prefix` + Value of the :option:`-m` command line option. -If at least one "output field" is not set, Python calculates the path -configuration to fill unset fields. If -:c:member:`~PyConfig.module_search_paths_set` is equal to ``0``, -:c:member:`~PyConfig.module_search_paths` is overridden and -:c:member:`~PyConfig.module_search_paths_set` is set to ``1``. + Used by :c:func:`Py_RunMain`. -It is possible to completely ignore the function calculating the default -path configuration by setting explicitly all path configuration output -fields listed above. A string is considered as set even if it is non-empty. -``module_search_paths`` is considered as set if -``module_search_paths_set`` is set to ``1``. In this case, -``module_search_paths`` will be used without modification. + Default: ``NULL``. -Set :c:member:`~PyConfig.pathconfig_warnings` to ``0`` to suppress warnings when -calculating the path configuration (Unix only, Windows does not log any warning). + .. c:member:: wchar_t* run_presite -If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix` -fields are not set, they inherit their value from :c:member:`~PyConfig.prefix` -and :c:member:`~PyConfig.exec_prefix` respectively. + ``package.module`` path to module that should be imported before + ``site.py`` is run. -:c:func:`Py_RunMain` and :c:func:`Py_Main` modify :data:`sys.path`: + Set by the :option:`-X presite=package.module <-X>` command-line + option and the :envvar:`PYTHON_PRESITE` environment variable. + The command-line option takes precedence. + + Needs a :ref:`debug build of Python ` (the ``Py_DEBUG`` macro + must be defined). -* If :c:member:`~PyConfig.run_filename` is set and is a directory which contains a - ``__main__.py`` script, prepend :c:member:`~PyConfig.run_filename` to - :data:`sys.path`. -* If :c:member:`~PyConfig.isolated` is zero: + Default: ``NULL``. - * If :c:member:`~PyConfig.run_module` is set, prepend the current directory - to :data:`sys.path`. Do nothing if the current directory cannot be read. - * If :c:member:`~PyConfig.run_filename` is set, prepend the directory of the - filename to :data:`sys.path`. - * Otherwise, prepend an empty string to :data:`sys.path`. + .. c:member:: int show_ref_count -If :c:member:`~PyConfig.site_import` is non-zero, :data:`sys.path` can be -modified by the :mod:`site` module. If -:c:member:`~PyConfig.user_site_directory` is non-zero and the user's -site-package directory exists, the :mod:`site` module appends the user's -site-package directory to :data:`sys.path`. + Show total reference count at exit (excluding :term:`immortal` objects)? -The following configuration files are used by the path configuration: + Set to ``1`` by :option:`-X showrefcount <-X>` command line option. -* ``pyvenv.cfg`` -* ``._pth`` file (ex: ``python._pth``) -* ``pybuilddir.txt`` (Unix only) + Needs a :ref:`debug build of Python ` (the ``Py_REF_DEBUG`` + macro must be defined). -If a ``._pth`` file is present: + Default: ``0``. -* Set :c:member:`~PyConfig.isolated` to ``1``. -* Set :c:member:`~PyConfig.use_environment` to ``0``. -* Set :c:member:`~PyConfig.site_import` to ``0``. -* Set :c:member:`~PyConfig.safe_path` to ``1``. + .. c:member:: int site_import -The ``__PYVENV_LAUNCHER__`` environment variable is used to set -:c:member:`PyConfig.base_executable`. + Import the :mod:`site` module at startup? + If equal to zero, disable the import of the module site and the + site-dependent manipulations of :data:`sys.path` that it entails. -PyInitConfig C API -================== + Also disable these manipulations if the :mod:`site` module is explicitly + imported later (call :func:`site.main` if you want them to be triggered). -C API to configure the Python initialization (:pep:`741`). + Set to ``0`` by the :option:`-S` command line option. -.. versionadded:: 3.14 + :data:`sys.flags.no_site ` is set to the inverted value of + :c:member:`~PyConfig.site_import`. -Create Config -------------- + Default: ``1``. -.. c:struct:: PyInitConfig + .. c:member:: int skip_source_first_line - Opaque structure to configure the Python initialization. + If non-zero, skip the first line of the :c:member:`PyConfig.run_filename` + source. + It allows the usage of non-Unix forms of ``#!cmd``. This is intended for + a DOS specific hack only. -.. c:function:: PyInitConfig* PyInitConfig_Create(void) + Set to ``1`` by the :option:`-x` command line option. - Create a new initialization configuration using :ref:`Isolated Configuration - ` default values. + Default: ``0``. - It must be freed by :c:func:`PyInitConfig_Free`. + .. c:member:: wchar_t* stdio_encoding + .. c:member:: wchar_t* stdio_errors - Return ``NULL`` on memory allocation failure. + Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and + :data:`sys.stderr` (but :data:`sys.stderr` always uses + ``"backslashreplace"`` error handler). + Use the :envvar:`PYTHONIOENCODING` environment variable if it is + non-empty. -.. c:function:: void PyInitConfig_Free(PyInitConfig *config) + Default encoding: - Free memory of the initialization configuration *config*. + * ``"UTF-8"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero. + * Otherwise, use the :term:`locale encoding`. - If *config* is ``NULL``, no operation is performed. + Default error handler: + * On Windows: use ``"surrogateescape"``. + * ``"surrogateescape"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero, + or if the LC_CTYPE locale is "C" or "POSIX". + * ``"strict"`` otherwise. -Error Handling --------------- + See also :c:member:`PyConfig.legacy_windows_stdio`. -.. c:function:: int PyInitConfig_GetError(PyInitConfig* config, const char **err_msg) + .. c:member:: int tracemalloc - Get the *config* error message. + Enable tracemalloc? - * Set *\*err_msg* and return ``1`` if an error is set. - * Set *\*err_msg* to ``NULL`` and return ``0`` otherwise. + If non-zero, call :func:`tracemalloc.start` at startup. - An error message is an UTF-8 encoded string. + Set by :option:`-X tracemalloc=N <-X>` command line option and by the + :envvar:`PYTHONTRACEMALLOC` environment variable. - If *config* has an exit code, format the exit code as an error - message. + Default: ``-1`` in Python mode, ``0`` in isolated mode. - The error message remains valid until another ``PyInitConfig`` - function is called with *config*. The caller doesn't have to free the - error message. + .. c:member:: int perf_profiling + Enable the Linux ``perf`` profiler support? -.. c:function:: int PyInitConfig_GetExitCode(PyInitConfig* config, int *exitcode) + If equals to ``1``, enable support for the Linux ``perf`` profiler. - Get the *config* exit code. + If equals to ``2``, enable support for the Linux ``perf`` profiler with + DWARF JIT support. - * Set *\*exitcode* and return ``1`` if *config* has an exit code set. - * Return ``0`` if *config* has no exit code set. + Set to ``1`` by :option:`-X perf <-X>` command-line option and the + :envvar:`PYTHONPERFSUPPORT` environment variable. - Only the ``Py_InitializeFromInitConfig()`` function can set an exit - code if the ``parse_argv`` option is non-zero. + Set to ``2`` by the :option:`-X perf_jit <-X>` command-line option and + the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable. - An exit code can be set when parsing the command line failed (exit - code ``2``) or when a command line option asks to display the command - line help (exit code ``0``). + Default: ``-1``. + .. seealso:: + See :ref:`perf_profiling` for more information. -Get Options ------------ + .. versionadded:: 3.12 -The configuration option *name* parameter must be a non-NULL -null-terminated UTF-8 encoded string. + .. c:member:: wchar_t* stdlib_dir -.. c:function:: int PyInitConfig_HasOption(PyInitConfig *config, const char *name) + Directory of the Python standard library. - Test if the configuration has an option called *name*. + Default: ``NULL``. - Return ``1`` if the option exists, or return ``0`` otherwise. + .. versionadded:: 3.11 + .. c:member:: int use_environment -.. c:function:: int PyInitConfig_GetInt(PyInitConfig *config, const char *name, int64_t *value) + Use :ref:`environment variables `? - Get an integer configuration option. + If equals to zero, ignore the :ref:`environment variables + `. - * Set *\*value*, and return ``0`` on success. - * Set an error in *config* and return ``-1`` on error. + Set to ``0`` by the :option:`-E` environment variable. + Default: ``1`` in Python config and ``0`` in isolated config. -.. c:function:: int PyInitConfig_GetStr(PyInitConfig *config, const char *name, char **value) + .. c:member:: int use_system_logger - Get a string configuration option as a null-terminated UTF-8 - encoded string. + If non-zero, ``stdout`` and ``stderr`` will be redirected to the system + log. - * Set *\*value*, and return ``0`` on success. - * Set an error in *config* and return ``-1`` on error. + Only available on macOS 10.12 and later, and on iOS. - *\*value* can be set to ``NULL`` if the option is an optional string and the - option is unset. + Default: ``0`` (don't use the system log) on macOS; ``1`` on iOS (use the + system log). - On success, the string must be released with ``free(value)`` if it's not - ``NULL``. + .. versionadded:: 3.14 + .. c:member:: int user_site_directory -.. c:function:: int PyInitConfig_GetStrList(PyInitConfig *config, const char *name, size_t *length, char ***items) + If non-zero, add the user site directory to :data:`sys.path`. - Get a string list configuration option as an array of - null-terminated UTF-8 encoded strings. + Set to ``0`` by the :option:`-s` and :option:`-I` command line options. - * Set *\*length* and *\*value*, and return ``0`` on success. - * Set an error in *config* and return ``-1`` on error. + Set to ``0`` by the :envvar:`PYTHONNOUSERSITE` environment variable. - On success, the string list must be released with - ``PyInitConfig_FreeStrList(length, items)``. + Default: ``1`` in Python mode, ``0`` in isolated mode. + .. c:member:: int verbose -.. c:function:: void PyInitConfig_FreeStrList(size_t length, char **items) + Verbose mode. If greater than ``0``, print a message each time a module is + imported, showing the place (filename or built-in module) from which + it is loaded. - Free memory of a string list created by - ``PyInitConfig_GetStrList()``. + If greater than or equal to ``2``, print a message for each file that is + checked for when searching for a module. Also provides information on + module cleanup at exit. + Incremented by the :option:`-v` command line option. -Set Options ------------ + Set by the :envvar:`PYTHONVERBOSE` environment variable value. -The configuration option *name* parameter must be a non-NULL null-terminated -UTF-8 encoded string. + Default: ``0``. -Some configuration options have side effects on other options. This logic is -only implemented when ``Py_InitializeFromInitConfig()`` is called, not by the -"Set" functions below. For example, setting ``dev_mode`` to ``1`` does not set -``faulthandler`` to ``1``. + .. c:member:: PyWideStringList warnoptions -.. c:function:: int PyInitConfig_SetInt(PyInitConfig *config, const char *name, int64_t value) + Options of the :mod:`warnings` module to build warnings filters, lowest + to highest priority: :data:`sys.warnoptions`. - Set an integer configuration option. + The :mod:`warnings` module adds :data:`sys.warnoptions` in the reverse + order: the last :c:member:`PyConfig.warnoptions` item becomes the first + item of :data:`warnings.filters` which is checked first (highest + priority). - * Return ``0`` on success. - * Set an error in *config* and return ``-1`` on error. + The :option:`-W` command line options adds its value to + :c:member:`~PyConfig.warnoptions`, it can be used multiple times. + The :envvar:`PYTHONWARNINGS` environment variable can also be used to add + warning options. Multiple options can be specified, separated by commas + (``,``). -.. c:function:: int PyInitConfig_SetStr(PyInitConfig *config, const char *name, const char *value) + Default: empty list. - Set a string configuration option from a null-terminated UTF-8 - encoded string. The string is copied. + .. c:member:: int write_bytecode - * Return ``0`` on success. - * Set an error in *config* and return ``-1`` on error. + If equal to ``0``, Python won't try to write ``.pyc`` files on the import of + source modules. + Set to ``0`` by the :option:`-B` command line option and the + :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. -.. c:function:: int PyInitConfig_SetStrList(PyInitConfig *config, const char *name, size_t length, char * const *items) + :data:`sys.dont_write_bytecode` is initialized to the inverted value of + :c:member:`~PyConfig.write_bytecode`. - Set a string list configuration option from an array of - null-terminated UTF-8 encoded strings. The string list is copied. + Default: ``1``. - * Return ``0`` on success. - * Set an error in *config* and return ``-1`` on error. + .. c:member:: PyWideStringList xoptions + Values of the :option:`-X` command line options: :data:`sys._xoptions`. -Module ------- + Default: empty list. -.. c:function:: int PyInitConfig_AddModule(PyInitConfig *config, const char *name, PyObject* (*initfunc)(void)) + .. c:member:: int _pystats - Add a built-in extension module to the table of built-in modules. + If non-zero, write performance statistics at Python exit. - The new module can be imported by the name *name*, and uses the function - *initfunc* as the initialization function called on the first attempted - import. + Need a special build with the ``Py_STATS`` macro: + see :option:`--enable-pystats`. - * Return ``0`` on success. - * Set an error in *config* and return ``-1`` on error. + Default: ``0``. - If Python is initialized multiple times, ``PyInitConfig_AddModule()`` must - be called at each Python initialization. +If :c:member:`~PyConfig.parse_argv` is non-zero, :c:member:`~PyConfig.argv` +arguments are parsed the same way the regular Python parses :ref:`command line +arguments `, and Python arguments are stripped from +:c:member:`~PyConfig.argv`. - Similar to the :c:func:`PyImport_AppendInittab` function. +The :c:member:`~PyConfig.xoptions` options are parsed to set other options: see +the :option:`-X` command line option. +.. versionchanged:: 3.9 -Initialize Python ------------------ + The ``show_alloc_count`` field has been removed. -.. c:function:: int Py_InitializeFromInitConfig(PyInitConfig *config) - Initialize Python from the initialization configuration. +.. _init-from-config: - * Return ``0`` on success. - * Set an error in *config* and return ``-1`` on error. - * Set an exit code in *config* and return ``-1`` if Python wants to - exit. +Initialization with PyConfig +---------------------------- - See ``PyInitConfig_GetExitcode()`` for the exit code case. +Initializing the interpreter from a populated configuration struct is handled +by calling :c:func:`Py_InitializeFromConfig`. +The caller is responsible to handle exceptions (error or exit) using +:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. -Example -------- +If :c:func:`PyImport_FrozenModules`, :c:func:`PyImport_AppendInittab` or +:c:func:`PyImport_ExtendInittab` are used, they must be set or called after +Python preinitialization and before the Python initialization. If Python is +initialized multiple times, :c:func:`PyImport_AppendInittab` or +:c:func:`PyImport_ExtendInittab` must be called before each Python +initialization. -Example initializing Python, set configuration options of various types, -return ``-1`` on error: +The current configuration (``PyConfig`` type) is stored in +``PyInterpreterState.config``. -.. code-block:: c +Example setting the program name:: - int init_python(void) + void init_python(void) { - PyInitConfig *config = PyInitConfig_Create(); - if (config == NULL) { - printf("PYTHON INIT ERROR: memory allocation failed\n"); - return -1; - } - - // Set an integer (dev mode) - if (PyInitConfig_SetInt(config, "dev_mode", 1) < 0) { - goto error; - } + PyStatus status; - // Set a list of UTF-8 strings (argv) - char *argv[] = {"my_program", "-c", "pass"}; - if (PyInitConfig_SetStrList(config, "argv", - Py_ARRAY_LENGTH(argv), argv) < 0) { - goto error; - } + PyConfig config; + PyConfig_InitPythonConfig(&config); - // Set a UTF-8 string (program name) - if (PyInitConfig_SetStr(config, "program_name", L"my_program") < 0) { - goto error; + /* Set the program name. Implicitly preinitialize Python. */ + status = PyConfig_SetString(&config, &config.program_name, + L"/path/to/my_program"); + if (PyStatus_Exception(status)) { + goto exception; } - // Initialize Python with the configuration - if (Py_InitializeFromInitConfig(config) < 0) { - goto error; + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto exception; } - PyInitConfig_Free(config); - return 0; - - error: - { - // Display the error message - // This uncommon braces style is used, because you cannot make - // goto targets point to variable declarations. - const char *err_msg; - (void)PyInitConfig_GetError(config, &err_msg); - printf("PYTHON INIT ERROR: %s\n", err_msg); - PyInitConfig_Free(config); + PyConfig_Clear(&config); + return; - return -1; - } + exception: + PyConfig_Clear(&config); + Py_ExitStatusException(status); } +More complete example modifying the default configuration, read the +configuration, and then override some parameters. Note that since +3.11, many parameters are not calculated until initialization, and +so values cannot be read from the configuration structure. Any values +set before initialize is called will be left unchanged by +initialization:: -Runtime Python configuration API -================================ + PyStatus init_python(const char *program_name) + { + PyStatus status; -The configuration option *name* parameter must be a non-NULL null-terminated -UTF-8 encoded string. + PyConfig config; + PyConfig_InitPythonConfig(&config); -Some options are read from the :mod:`sys` attributes. For example, the option -``"argv"`` is read from :data:`sys.argv`. + /* Set the program name before reading the configuration + (decode byte string from the locale encoding). + Implicitly preinitialize Python. */ + status = PyConfig_SetBytesString(&config, &config.program_name, + program_name); + if (PyStatus_Exception(status)) { + goto done; + } -.. c:function:: PyObject* PyConfig_Get(const char *name) + /* Read all configuration at once */ + status = PyConfig_Read(&config); + if (PyStatus_Exception(status)) { + goto done; + } - Get the current runtime value of a configuration option as a Python object. + /* Specify sys.path explicitly */ + /* If you want to modify the default set of paths, finish + initialization first and then use PySys_GetAttrString("path") */ + config.module_search_paths_set = 1; + status = PyWideStringList_Append(&config.module_search_paths, + L"/path/to/stdlib"); + if (PyStatus_Exception(status)) { + goto done; + } + status = PyWideStringList_Append(&config.module_search_paths, + L"/path/to/more/modules"); + if (PyStatus_Exception(status)) { + goto done; + } - * Return a new reference on success. - * Set an exception and return ``NULL`` on error. + /* Override executable computed by PyConfig_Read() */ + status = PyConfig_SetString(&config, &config.executable, + L"/path/to/my_executable"); + if (PyStatus_Exception(status)) { + goto done; + } - The object type depends on the configuration option. It can be: + status = Py_InitializeFromConfig(&config); - * ``bool`` - * ``int`` - * ``str`` - * ``list[str]`` - * ``dict[str, str]`` + done: + PyConfig_Clear(&config); + return status; + } - The caller must hold the GIL. The function cannot be called before - Python initialization nor after Python finalization. - .. versionadded:: 3.14 +.. _init-isolated-conf: +Isolated Configuration +---------------------- -.. c:function:: int PyConfig_GetInt(const char *name, int *value) +:c:func:`PyPreConfig_InitIsolatedConfig` and +:c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to +isolate Python from the system. For example, to embed Python into an +application. - Similar to :c:func:`PyConfig_Get`, but get the value as a C int. +This configuration ignores global configuration variables, environment +variables, command line arguments (:c:member:`PyConfig.argv` is not parsed) +and user site directory. The C standard streams (ex: ``stdout``) and the +LC_CTYPE locale are left unchanged. Signal handlers are not installed. - * Return ``0`` on success. - * Set an exception and return ``-1`` on error. +Configuration files are still used with this configuration to determine +paths that are unspecified. Ensure :c:member:`PyConfig.home` is specified +to avoid computing the default path configuration. - .. versionadded:: 3.14 +.. _init-python-config: -.. c:function:: PyObject* PyConfig_Names(void) +Python Configuration +-------------------- - Get all configuration option names as a ``frozenset``. +:c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` +functions create a configuration to build a customized Python which behaves as +the regular Python. - * Return a new reference on success. - * Set an exception and return ``NULL`` on error. +Environments variables and command line arguments are used to configure +Python, whereas global configuration variables are ignored. - The caller must hold the GIL. The function cannot be called before - Python initialization nor after Python finalization. +This function enables C locale coercion (:pep:`538`) +and :ref:`Python UTF-8 Mode ` +(:pep:`540`) depending on the LC_CTYPE locale, :envvar:`PYTHONUTF8` and +:envvar:`PYTHONCOERCECLOCALE` environment variables. - .. versionadded:: 3.14 +.. _init-path-config: -.. c:function:: int PyConfig_Set(const char *name, PyObject *value) +Python Path Configuration +------------------------- - Set the current runtime value of a configuration option. +:c:type:`PyConfig` contains multiple fields for the path configuration: - * Raise a :exc:`ValueError` if there is no option *name*. - * Raise a :exc:`ValueError` if *value* is an invalid value. - * Raise a :exc:`ValueError` if the option is read-only (cannot be set). - * Raise a :exc:`TypeError` if *value* has not the proper type. +* Path configuration inputs: - The caller must hold the GIL. The function cannot be called before - Python initialization nor after Python finalization. + * :c:member:`PyConfig.home` + * :c:member:`PyConfig.platlibdir` + * :c:member:`PyConfig.pathconfig_warnings` + * :c:member:`PyConfig.program_name` + * :c:member:`PyConfig.pythonpath_env` + * current working directory: to get absolute paths + * ``PATH`` environment variable to get the program full path + (from :c:member:`PyConfig.program_name`) + * ``__PYVENV_LAUNCHER__`` environment variable + * (Windows only) Application paths in the registry under + "Software\Python\PythonCore\X.Y\PythonPath" of HKEY_CURRENT_USER and + HKEY_LOCAL_MACHINE (where X.Y is the Python version). - .. versionadded:: 3.14 +* Path configuration output fields: + * :c:member:`PyConfig.base_exec_prefix` + * :c:member:`PyConfig.base_executable` + * :c:member:`PyConfig.base_prefix` + * :c:member:`PyConfig.exec_prefix` + * :c:member:`PyConfig.executable` + * :c:member:`PyConfig.module_search_paths_set`, + :c:member:`PyConfig.module_search_paths` + * :c:member:`PyConfig.prefix` -Py_GetArgcArgv() -================ +If at least one "output field" is not set, Python calculates the path +configuration to fill unset fields. If +:c:member:`~PyConfig.module_search_paths_set` is equal to ``0``, +:c:member:`~PyConfig.module_search_paths` is overridden and +:c:member:`~PyConfig.module_search_paths_set` is set to ``1``. -.. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv) +It is possible to completely ignore the function calculating the default +path configuration by setting explicitly all path configuration output +fields listed above. A string is considered as set even if it is non-empty. +``module_search_paths`` is considered as set if +``module_search_paths_set`` is set to ``1``. In this case, +``module_search_paths`` will be used without modification. - Get the original command line arguments, before Python modified them. +Set :c:member:`~PyConfig.pathconfig_warnings` to ``0`` to suppress warnings when +calculating the path configuration (Unix only, Windows does not log any warning). - See also :c:member:`PyConfig.orig_argv` member. +If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix` +fields are not set, they inherit their value from :c:member:`~PyConfig.prefix` +and :c:member:`~PyConfig.exec_prefix` respectively. + +:c:func:`Py_RunMain` and :c:func:`Py_Main` modify :data:`sys.path`: +* If :c:member:`~PyConfig.run_filename` is set and is a directory which contains a + ``__main__.py`` script, prepend :c:member:`~PyConfig.run_filename` to + :data:`sys.path`. +* If :c:member:`~PyConfig.isolated` is zero: -Multi-Phase Initialization Private Provisional API -================================================== + * If :c:member:`~PyConfig.run_module` is set, prepend the current directory + to :data:`sys.path`. Do nothing if the current directory cannot be read. + * If :c:member:`~PyConfig.run_filename` is set, prepend the directory of the + filename to :data:`sys.path`. + * Otherwise, prepend an empty string to :data:`sys.path`. -This section is a private provisional API introducing multi-phase -initialization, the core feature of :pep:`432`: +If :c:member:`~PyConfig.site_import` is non-zero, :data:`sys.path` can be +modified by the :mod:`site` module. If +:c:member:`~PyConfig.user_site_directory` is non-zero and the user's +site-package directory exists, the :mod:`site` module appends the user's +site-package directory to :data:`sys.path`. -* "Core" initialization phase, "bare minimum Python": +The following configuration files are used by the path configuration: - * Builtin types; - * Builtin exceptions; - * Builtin and frozen modules; - * The :mod:`sys` module is only partially initialized - (ex: :data:`sys.path` doesn't exist yet). +* ``pyvenv.cfg`` +* ``._pth`` file (ex: ``python._pth``) +* ``pybuilddir.txt`` (Unix only) -* "Main" initialization phase, Python is fully initialized: +If a ``._pth`` file is present: - * Install and configure :mod:`importlib`; - * Apply the :ref:`Path Configuration `; - * Install signal handlers; - * Finish :mod:`sys` module initialization (ex: create :data:`sys.stdout` - and :data:`sys.path`); - * Enable optional features like :mod:`faulthandler` and :mod:`tracemalloc`; - * Import the :mod:`site` module; - * etc. +* Set :c:member:`~PyConfig.isolated` to ``1``. +* Set :c:member:`~PyConfig.use_environment` to ``0``. +* Set :c:member:`~PyConfig.site_import` to ``0``. +* Set :c:member:`~PyConfig.user_site_directory` to ``0`` (since 3.15). +* Set :c:member:`~PyConfig.safe_path` to ``1``. -Private provisional API: +If :c:member:`~PyConfig.home` is not set and a ``pyvenv.cfg`` file is present in +the same directory as :c:member:`~PyConfig.executable`, or its parent, +:c:member:`~PyConfig.prefix` and :c:member:`~PyConfig.exec_prefix` are set that +location. When this happens, :c:member:`~PyConfig.base_prefix` and +:c:member:`~PyConfig.base_exec_prefix` still keep their value, pointing to the +base installation. See :ref:`sys-path-init-virtual-environments` for more +information. -* :c:member:`PyConfig._init_main`: if set to ``0``, - :c:func:`Py_InitializeFromConfig` stops at the "Core" initialization phase. +The ``__PYVENV_LAUNCHER__`` environment variable is used to set +:c:member:`PyConfig.base_executable`. -.. c:function:: PyStatus _Py_InitializeMain(void) +.. versionchanged:: 3.14 - Move to the "Main" initialization phase, finish the Python initialization. + :c:member:`~PyConfig.prefix`, and :c:member:`~PyConfig.exec_prefix`, are now + set to the ``pyvenv.cfg`` directory. This was previously done by :mod:`site`, + therefore affected by :option:`-S`. -No module is imported during the "Core" phase and the ``importlib`` module is -not configured: the :ref:`Path Configuration ` is only -applied during the "Main" phase. It may allow to customize Python in Python to -override or tune the :ref:`Path Configuration `, maybe -install a custom :data:`sys.meta_path` importer or an import hook, etc. -It may become possible to calculate the :ref:`Path Configuration -` in Python, after the Core phase and before the Main phase, -which is one of the :pep:`432` motivation. +.. versionchanged:: 3.15 -The "Core" phase is not properly defined: what should be and what should -not be available at this phase is not specified yet. The API is marked -as private and provisional: the API can be modified or even be removed -anytime until a proper public API is designed. + :c:member:`~PyConfig.user_site_directory` is now set to ``0`` when a + ``._pth`` file is present. -Example running Python code between "Core" and "Main" initialization -phases:: - void init_python(void) - { - PyStatus status; +Py_GetArgcArgv() +================ - PyConfig config; - PyConfig_InitPythonConfig(&config); - config._init_main = 0; +.. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv) - /* ... customize 'config' configuration ... */ + Get the original command line arguments, before Python modified them. - status = Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } + See also :c:member:`PyConfig.orig_argv` member. - /* Use sys.stderr because sys.stdout is only created - by _Py_InitializeMain() */ - int res = PyRun_SimpleString( - "import sys; " - "print('Run Python code before _Py_InitializeMain', " - "file=sys.stderr)"); - if (res < 0) { - exit(1); - } +Delaying main module execution +============================== - /* ... put more configuration code here ... */ +In some embedding use cases, it may be desirable to separate interpreter initialization +from the execution of the main module. - status = _Py_InitializeMain(); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - } +This separation can be achieved by setting ``PyConfig.run_command`` to the empty +string during initialization (to prevent the interpreter from dropping into the +interactive prompt), and then subsequently executing the desired main module +code using ``__main__.__dict__`` as the global namespace. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 8ef463e3f88ca8..6886cd85b09a7d 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -30,6 +30,16 @@ familiar with writing an extension before attempting to embed Python in a real application. +Language version compatibility +============================== + +Python's C API is compatible with C11 and C++11 versions of C and C++. + +This is a lower limit: the C API does not require features from later +C/C++ versions. +You do *not* need to enable your compiler's "c11 mode". + + Coding standards ================ @@ -97,48 +107,70 @@ header files properly declare the entry points to be ``extern "C"``. As a result there is no need to do anything special to use the API from C++. -Useful macros -============= +.. _capi-system-includes: -Several useful macros are defined in the Python header files. Many are -defined closer to where they are useful (e.g. :c:macro:`Py_RETURN_NONE`). -Others of a more general utility are defined here. This is not necessarily a -complete listing. +System includes +--------------- -.. c:macro:: PyMODINIT_FUNC + :file:`Python.h` includes several standard header files. + C extensions should include the standard headers that they use, + and should not rely on these implicit includes. + The implicit includes are: - Declare an extension module ``PyInit`` initialization function. The function - return type is :c:expr:`PyObject*`. The macro declares any special linkage - declarations required by the platform, and for C++ declares the function as - ``extern "C"``. + * ```` + * ```` (on Windows) + * ```` + * ```` + * ```` + * ```` + * ```` + * ```` (if present) - The initialization function must be named :samp:`PyInit_{name}`, where - *name* is the name of the module, and should be the only non-\ ``static`` - item defined in the module file. Example:: + The following are included for backwards compatibility, unless using + :ref:`Limited API ` 3.13 or newer: - static struct PyModuleDef spam_module = { - PyModuleDef_HEAD_INIT, - .m_name = "spam", - ... - }; + * ```` + * ```` (on POSIX) - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModule_Create(&spam_module); - } + The following are included for backwards compatibility, unless using + :ref:`Limited API ` 3.11 or newer: + + * ```` + * ```` + * ```` + * ```` + +.. note:: + + Since Python may define some pre-processor definitions which affect the standard + headers on some systems, you *must* include :file:`Python.h` before any standard + headers are included. + + +Useful macros +============= + +Several useful macros are defined in the Python header files. Many are +defined closer to where they are useful (for example, :c:macro:`Py_RETURN_NONE`, +:c:macro:`PyMODINIT_FUNC`). +Others of a more general utility are defined here. This is not necessarily a +complete listing. .. c:macro:: Py_ABS(x) Return the absolute value of ``x``. + If the result cannot be represented (for example, if ``x`` has + :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is + undefined. + .. versionadded:: 3.3 .. c:macro:: Py_ALWAYS_INLINE Ask the compiler to always inline a static inline function. The compiler can - ignore it and decides to not inline the function. + ignore it and decide to not inline the function. It can be used to inline performance critical static inline functions when building Python in debug mode with function inlining disabled. For example, @@ -179,6 +211,25 @@ complete listing. Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the command line (see :c:member:`PyConfig.use_environment`). +.. c:macro:: Py_LOCAL(type) + + Declare a function returning the specified *type* using a fast-calling + qualifier for functions that are local to the current file. + Semantically, this is equivalent to ``static type``. + +.. c:macro:: Py_LOCAL_INLINE(type) + + Equivalent to :c:macro:`Py_LOCAL` but additionally requests the function + be inlined. + +.. c:macro:: Py_LOCAL_SYMBOL + + Macro used to declare a symbol as local to the shared library (hidden). + On supported platforms, it ensures the symbol is not exported. + + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("hidden")))``. + .. c:macro:: Py_MAX(x, y) Return the maximum value between ``x`` and ``y``. @@ -191,6 +242,14 @@ complete listing. .. versionadded:: 3.6 +.. c:macro:: Py_MEMCPY(dest, src, n) + + This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. + Use :c:func:`!memcpy` directly instead. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. + .. c:macro:: Py_MIN(x, y) Return the minimum value between ``x`` and ``y``. @@ -245,9 +304,32 @@ complete listing. .. versionadded:: 3.4 +.. c:macro:: Py_BUILD_ASSERT(cond) + + Asserts a compile-time condition *cond*, as a statement. + The build will fail if the condition is false or cannot be evaluated at compile time. + + For example:: + + Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(int64_t)); + + .. versionadded:: 3.3 + +.. c:macro:: Py_BUILD_ASSERT_EXPR(cond) + + Asserts a compile-time condition *cond*, as an expression that evaluates to ``0``. + The build will fail if the condition is false or cannot be evaluated at compile time. + + For example:: + + #define foo_to_char(foo) \ + ((char *)(foo) + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + + .. versionadded:: 3.3 + .. c:macro:: PyDoc_STRVAR(name, str) - Creates a variable with name ``name`` that can be used in docstrings. + Creates a variable with name *name* that can be used in docstrings. If Python is built without docstrings, the value will be empty. Use :c:macro:`PyDoc_STRVAR` for docstrings to support building @@ -279,6 +361,60 @@ complete listing. {NULL, NULL} }; +.. c:macro:: PyDoc_VAR(name) + + Declares a static character array variable with the given name *name*. + + For example:: + + PyDoc_VAR(python_doc) = PyDoc_STR("A genus of constricting snakes in the Pythonidae family native " + "to the tropics and subtropics of the Eastern Hemisphere."); + +.. c:macro:: Py_ARRAY_LENGTH(array) + + Compute the length of a statically allocated C array at compile time. + + The *array* argument must be a C array with a size known at compile time. + Passing an array with an unknown size, such as a heap-allocated array, + will result in a compilation error on some compilers, or otherwise produce + incorrect results. + + This is roughly equivalent to:: + + sizeof(array) / sizeof((array)[0]) + + +.. c:macro:: Py_EXPORTED_SYMBOL + + Macro used to declare a symbol (function or data) as exported. + On Windows, this expands to ``__declspec(dllexport)``. + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("default")))``. + This macro is for defining the C API itself; extension modules should not use it. + + +.. c:macro:: Py_IMPORTED_SYMBOL + + Macro used to declare a symbol as imported. + On Windows, this expands to ``__declspec(dllimport)``. + This macro is for defining the C API itself; extension modules should not use it. + + +.. c:macro:: PyAPI_FUNC(type) + + Macro used by CPython to declare a function as part of the C API. + Its expansion depends on the platform and build configuration. + This macro is intended for defining CPython's C API itself; + extension modules should not use it for their own symbols. + + +.. c:macro:: PyAPI_DATA(type) + + Macro used by CPython to declare a public global variable as part of the C API. + Its expansion depends on the platform and build configuration. + This macro is intended for defining CPython's C API itself; + extension modules should not use it for their own symbols. + .. _api-objects: @@ -769,20 +905,11 @@ found along :envvar:`PATH`.) The user can override this behavior by setting the environment variable :envvar:`PYTHONHOME`, or insert additional directories in front of the standard path by setting :envvar:`PYTHONPATH`. -.. index:: - single: Py_GetPath (C function) - single: Py_GetPrefix (C function) - single: Py_GetExecPrefix (C function) - single: Py_GetProgramFullPath (C function) - The embedding application can steer the search by setting :c:member:`PyConfig.program_name` *before* calling :c:func:`Py_InitializeFromConfig`. Note that :envvar:`PYTHONHOME` still overrides this and :envvar:`PYTHONPATH` is still -inserted in front of the standard path. An application that requires total -control has to provide its own implementation of :c:func:`Py_GetPath`, -:c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, and -:c:func:`Py_GetProgramFullPath` (all defined in :file:`Modules/getpath.c`). +inserted in front of the standard path. .. index:: single: Py_IsInitialized (C function) @@ -816,14 +943,17 @@ frequently used builds will be described in the remainder of this section. Compiling the interpreter with the :c:macro:`!Py_DEBUG` macro defined produces what is generally meant by :ref:`a debug build of Python `. -:c:macro:`!Py_DEBUG` is enabled in the Unix build by adding -:option:`--with-pydebug` to the :file:`./configure` command. -It is also implied by the presence of the -not-Python-specific :c:macro:`!_DEBUG` macro. When :c:macro:`!Py_DEBUG` is enabled -in the Unix build, compiler optimization is disabled. + +On Unix, :c:macro:`!Py_DEBUG` can be enabled by adding :option:`--with-pydebug` +to the :file:`./configure` command. This will also disable compiler optimization. + +On Windows, selecting a debug build (e.g., by passing the :option:`-d` option to +:file:`PCbuild/build.bat`) automatically enables :c:macro:`!Py_DEBUG`. +Additionally, the presence of the not-Python-specific :c:macro:`!_DEBUG` macro, +when defined by the compiler, will also implicitly enable :c:macro:`!Py_DEBUG`. In addition to the reference count debugging described below, extra checks are -performed, see :ref:`Python Debug Build `. +performed. See :ref:`Python Debug Build ` for more details. Defining :c:macro:`Py_TRACE_REFS` enables reference tracing (see the :option:`configure --with-trace-refs option <--with-trace-refs>`). @@ -834,3 +964,41 @@ after every statement run by the interpreter.) Please refer to :file:`Misc/SpecialBuilds.txt` in the Python source distribution for more detailed information. + + +.. _c-api-tools: + +Recommended third party tools +============================= + +The following third party tools offer both simpler and more sophisticated +approaches to creating C, C++ and Rust extensions for Python: + +* `Cython `_ +* `cffi `_ +* `HPy `_ +* `nanobind `_ (C++) +* `Numba `_ +* `pybind11 `_ (C++) +* `PyO3 `_ (Rust) +* `SWIG `_ + +Using tools such as these can help avoid writing code that is tightly bound to +a particular version of CPython, avoid reference counting errors, and focus +more on your own code than on using the CPython API. In general, new versions +of Python can be supported by updating the tool, and your code will often use +newer and more efficient APIs automatically. Some tools also support compiling +for other implementations of Python from a single set of sources. + +These projects are not supported by the same people who maintain Python, and +issues need to be raised with the projects directly. Remember to check that the +project is still maintained and supported, as the list above may become +outdated. + +.. seealso:: + + `Python Packaging User Guide: Binary Extensions `_ + The Python Packaging User Guide not only covers several available + tools that simplify the creation of binary extensions, but also + discusses the various reasons why creating an extension module may be + desirable in the first place. diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index bf9df62c6f1706..6cfd24c5ae60c8 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -54,6 +54,6 @@ There are two functions specifically for working with iterators. - ``PYGEN_RETURN`` if iterator returns. Return value is returned via *presult*. - ``PYGEN_NEXT`` if iterator yields. Yielded value is returned via *presult*. - - ``PYGEN_ERROR`` if iterator has raised and exception. *presult* is set to ``NULL``. + - ``PYGEN_ERROR`` if iterator has raised an exception. *presult* is set to ``NULL``. .. versionadded:: 3.10 diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 6b7ba8c9979163..bfbfe3c9279980 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -50,3 +50,72 @@ sentinel value is returned. callable object that can be called with no parameters; each call to it should return the next item in the iteration. When *callable* returns a value equal to *sentinel*, the iteration will be terminated. + + +Range Objects +^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyRange_Type + + The type object for :class:`range` objects. + + +.. c:function:: int PyRange_Check(PyObject *o) + + Return true if the object *o* is an instance of a :class:`range` object. + This function always succeeds. + + +Builtin Iterator Types +^^^^^^^^^^^^^^^^^^^^^^ + +These are built-in iteration types that are included in Python's C API, but +provide no additional functions. They are here for completeness. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * C type + * Python type + * * .. c:var:: PyTypeObject PyEnum_Type + * :py:class:`enumerate` + * * .. c:var:: PyTypeObject PyFilter_Type + * :py:class:`filter` + * * .. c:var:: PyTypeObject PyMap_Type + * :py:class:`map` + * * .. c:var:: PyTypeObject PyReversed_Type + * :py:class:`reversed` + * * .. c:var:: PyTypeObject PyZip_Type + * :py:class:`zip` + + +Other Iterator Objects +^^^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyByteArrayIter_Type +.. c:var:: PyTypeObject PyBytesIter_Type +.. c:var:: PyTypeObject PyListIter_Type +.. c:var:: PyTypeObject PyListRevIter_Type +.. c:var:: PyTypeObject PySetIter_Type +.. c:var:: PyTypeObject PyTupleIter_Type +.. c:var:: PyTypeObject PyRangeIter_Type +.. c:var:: PyTypeObject PyLongRangeIter_Type +.. c:var:: PyTypeObject PyDictIterKey_Type +.. c:var:: PyTypeObject PyDictRevIterKey_Type +.. c:var:: PyTypeObject PyDictIterValue_Type +.. c:var:: PyTypeObject PyDictRevIterValue_Type +.. c:var:: PyTypeObject PyDictIterItem_Type +.. c:var:: PyTypeObject PyDictRevIterItem_Type +.. c:var:: PyTypeObject PyODictIter_Type + + Type objects for iterators of various built-in objects. + + Do not create instances of these directly; prefer calling + :c:func:`PyObject_GetIter` instead. + + Note that there is no guarantee that a given built-in type uses a given iterator + type. For example, iterating over :class:`range` will use one of two iterator + types depending on the size of the range. Other types may start using a + similar scheme in the future, without warning. diff --git a/Doc/c-api/lifecycle.dot b/Doc/c-api/lifecycle.dot new file mode 100644 index 00000000000000..dca9f87e9e0aca --- /dev/null +++ b/Doc/c-api/lifecycle.dot @@ -0,0 +1,156 @@ +digraph "Life Events" { + graph [ + fontnames="svg" + fontsize=12.0 + id="life_events_graph" + layout="dot" + margin="0,0" + ranksep=0.25 + stylesheet="lifecycle.dot.css" + ] + node [ + fontname="Courier" + fontsize=12.0 + ] + edge [ + fontname="Times-Italic" + fontsize=12.0 + ] + + "start" [fontname="Times-Italic" shape=plain label=< start > style=invis] + { + rank="same" + "tp_new" [href="typeobj.html#c.PyTypeObject.tp_new" target="_top"] + "tp_alloc" [href="typeobj.html#c.PyTypeObject.tp_alloc" target="_top"] + } + "tp_init" [href="typeobj.html#c.PyTypeObject.tp_init" target="_top"] + "reachable" [fontname="Times-Italic" shape=box] + "tp_traverse" [ + href="typeobj.html#c.PyTypeObject.tp_traverse" + ordering="in" + target="_top" + ] + "finalized?" [ + fontname="Times-Italic" + label=finalized?> + ordering="in" + shape=diamond + tooltip="marked as finalized?" + ] + "tp_finalize" [ + href="typeobj.html#c.PyTypeObject.tp_finalize" + ordering="in" + target="_top" + ] + "tp_clear" [href="typeobj.html#c.PyTypeObject.tp_clear" target="_top"] + "uncollectable" [ + fontname="Times-Italic" + label=(leaked)> + shape=box + tooltip="uncollectable (leaked)" + ] + "tp_dealloc" [ + href="typeobj.html#c.PyTypeObject.tp_dealloc" + ordering="in" + target="_top" + ] + "tp_free" [href="typeobj.html#c.PyTypeObject.tp_free" target="_top"] + + "start" -> "tp_new" [ + label=< type call > + ] + "tp_new" -> "tp_alloc" [ + label=< direct call > arrowhead=empty + labeltooltip="tp_new to tp_alloc: direct call" + tooltip="tp_new to tp_alloc: direct call" + ] + "tp_new" -> "tp_init" [tooltip="tp_new to tp_init"] + "tp_init" -> "reachable" [tooltip="tp_init to reachable"] + "reachable" -> "tp_traverse" [ + dir="back" + label=< not in a
cyclic
isolate > + labeltooltip="tp_traverse to reachable: not in a cyclic isolate" + tooltip="tp_traverse to reachable: not in a cyclic isolate" + ] + "reachable" -> "tp_traverse" [ + label=< periodic
cyclic isolate
detection > + labeltooltip="reachable to tp_traverse: periodic cyclic isolate detection" + tooltip="reachable to tp_traverse: periodic cyclic isolate detection" + ] + "reachable" -> "tp_init" [tooltip="reachable to tp_init"] + "reachable" -> "tp_finalize" [ + dir="back" + label=< resurrected
(maybe remove
finalized mark) > + labeltooltip="tp_finalize to reachable: resurrected (maybe remove finalized mark)" + tooltip="tp_finalize to reachable: resurrected (maybe remove finalized mark)" + ] + "tp_traverse" -> "finalized?" [ + label=< cyclic
isolate > + labeltooltip="tp_traverse to finalized?: cyclic isolate" + tooltip="tp_traverse to finalized?: cyclic isolate" + ] + "reachable" -> "finalized?" [ + label=< no refs > + labeltooltip="reachable to finalized?: no refs" + tooltip="reachable to finalized?: no refs" + ] + "finalized?" -> "tp_finalize" [ + label=< no (mark
as finalized) > + labeltooltip="finalized? to tp_finalize: no (mark as finalized)" + tooltip="finalized? to tp_finalize: no (mark as finalized)" + ] + "finalized?" -> "tp_clear" [ + label=< yes > + labeltooltip="finalized? to tp_clear: yes" + tooltip="finalized? to tp_clear: yes" + ] + "tp_finalize" -> "tp_clear" [ + label=< no refs or
cyclic isolate > + labeltooltip="tp_finalize to tp_clear: no refs or cyclic isolate" + tooltip="tp_finalize to tp_clear: no refs or cyclic isolate" + ] + "tp_finalize" -> "tp_dealloc" [ + arrowtail=empty + dir="back" + href="lifecycle.html#c.PyObject_CallFinalizerFromDealloc" + style=dashed + label=< recommended
call (see
explanation)> + labeltooltip="tp_dealloc to tp_finalize: recommended call (see explanation)" + target="_top" + tooltip="tp_dealloc to tp_finalize: recommended call (see explanation)" + ] + "tp_finalize" -> "tp_dealloc" [ + label=< no refs > + labeltooltip="tp_finalize to tp_dealloc: no refs" + tooltip="tp_finalize to tp_dealloc: no refs" + ] + "tp_clear" -> "tp_dealloc" [ + label=< no refs > + labeltooltip="tp_clear to tp_dealloc: no refs" + tooltip="tp_clear to tp_dealloc: no refs" + ] + "tp_clear" -> "uncollectable" [ + label=< cyclic
isolate > + labeltooltip="tp_clear to uncollectable: cyclic isolate" + tooltip="tp_clear to uncollectable: cyclic isolate" + ] + "uncollectable" -> "tp_dealloc" [ + style=invis + tooltip="uncollectable to tp_dealloc" + ] + "reachable" -> "uncollectable" [ + label=< cyclic
isolate
(no GC
support) > + labeltooltip="reachable to uncollectable: cyclic isolate (no GC support)" + tooltip="reachable to uncollectable: cyclic isolate (no GC support)" + ] + "reachable" -> "tp_dealloc" [ + label=< no refs> + labeltooltip="reachable to tp_dealloc: no refs" + ] + "tp_dealloc" -> "tp_free" [ + arrowhead=empty + label=< direct call > + labeltooltip="tp_dealloc to tp_free: direct call" + tooltip="tp_dealloc to tp_free: direct call" + ] +} diff --git a/Doc/c-api/lifecycle.dot.css b/Doc/c-api/lifecycle.dot.css new file mode 100644 index 00000000000000..3abf95b74da6ba --- /dev/null +++ b/Doc/c-api/lifecycle.dot.css @@ -0,0 +1,21 @@ +#life_events_graph { + --svg-fgcolor: currentcolor; + --svg-bgcolor: transparent; +} +#life_events_graph a { + color: inherit; +} +#life_events_graph [stroke="black"] { + stroke: var(--svg-fgcolor); +} +#life_events_graph text, +#life_events_graph [fill="black"] { + fill: var(--svg-fgcolor); +} +#life_events_graph [fill="white"] { + fill: var(--svg-bgcolor); +} +#life_events_graph [fill="none"] { + /* On links, setting fill will make the entire shape clickable */ + fill: var(--svg-bgcolor); +} diff --git a/Doc/c-api/lifecycle.dot.pdf b/Doc/c-api/lifecycle.dot.pdf new file mode 100644 index 00000000000000..ed5b5039c83e2c Binary files /dev/null and b/Doc/c-api/lifecycle.dot.pdf differ diff --git a/Doc/c-api/lifecycle.dot.svg b/Doc/c-api/lifecycle.dot.svg new file mode 100644 index 00000000000000..7ace27dfcba113 --- /dev/null +++ b/Doc/c-api/lifecycle.dot.svg @@ -0,0 +1,374 @@ + + + + + + + +Life Events + + + + +tp_new + + +tp_new + + + + + +start->tp_new + + + + + + +    type call   + + + + + +tp_alloc + + +tp_alloc + + + + + +tp_new->tp_alloc + + + + + + +  direct call   + + + + + +tp_init + + +tp_init + + + + + +tp_new->tp_init + + + + + + + + +reachable + +reachable + + + +tp_init->reachable + + + + + + + + +reachable->tp_init + + + + + + + + +tp_traverse + + +tp_traverse + + + + + +reachable->tp_traverse + + + + + + +  not in a   +  cyclic   +  isolate   + + + + + +reachable->tp_traverse + + + + + + +  periodic   +  cyclic isolate    +  detection   + + + + + +finalized? + + +marked as +finalized? + + + + + +reachable->finalized? + + + + + + +  no refs   + + + + + +tp_finalize + + +tp_finalize + + + + + +reachable->tp_finalize + + + + + + +  resurrected   +  (maybe remove   +  finalized mark)   + + + + + +uncollectable + + +uncollectable +(leaked) + + + + + +reachable->uncollectable + + + + + + +  cyclic   +  isolate   +  (no GC   +  support)   + + + + + +tp_dealloc + + +tp_dealloc + + + + + +reachable->tp_dealloc + + + +  no refs + + + + + +tp_traverse->finalized? + + + + + + +  cyclic   +  isolate   + + + + + +finalized?->tp_finalize + + + + + + +  no (mark   +  as finalized)   + + + + + +tp_clear + + +tp_clear + + + + + +finalized?->tp_clear + + + + + + +  yes   + + + + + +tp_finalize->tp_clear + + + + + + +  no refs or    +  cyclic isolate   + + + + + +tp_finalize->tp_dealloc + + + + + + +  recommended +  call (see +  explanation) + + + + + +tp_finalize->tp_dealloc + + + + + + +   no refs   + + + + + +tp_clear->uncollectable + + + + + + +  cyclic   +  isolate   + + + + + +tp_clear->tp_dealloc + + + + + + +  no refs   + + + + + + +tp_free + + +tp_free + + + + + +tp_dealloc->tp_free + + + + + + +    direct call   + + + + + diff --git a/Doc/c-api/lifecycle.rst b/Doc/c-api/lifecycle.rst new file mode 100644 index 00000000000000..531c4080a0131c --- /dev/null +++ b/Doc/c-api/lifecycle.rst @@ -0,0 +1,275 @@ +.. highlight:: c + +.. _life-cycle: + +Object Life Cycle +================= + +This section explains how a type's slots relate to each other throughout the +life of an object. It is not intended to be a complete canonical reference for +the slots; instead, refer to the slot-specific documentation in +:ref:`type-structs` for details about a particular slot. + + +Life Events +----------- + +The figure below illustrates the order of events that can occur throughout an +object's life. An arrow from *A* to *B* indicates that event *B* can occur +after event *A* has occurred, with the arrow's label indicating the condition +that must be true for *B* to occur after *A*. + +.. only:: html and not epub + + .. raw:: html + + + + .. raw:: html + :file: lifecycle.dot.svg + + .. raw:: html + + + +.. only:: epub or not (html or latex) + + .. image:: lifecycle.dot.svg + :align: center + :class: invert-in-dark-mode + :alt: Diagram showing events in an object's life. Explained in detail below. + +.. only:: latex + + .. image:: lifecycle.dot.pdf + :align: center + :class: invert-in-dark-mode + :alt: Diagram showing events in an object's life. Explained in detail below. + +.. container:: + :name: life-events-graph-description + + Explanation: + + * When a new object is constructed by calling its type: + + #. :c:member:`~PyTypeObject.tp_new` is called to create a new object. + #. :c:member:`~PyTypeObject.tp_alloc` is directly called by + :c:member:`~PyTypeObject.tp_new` to allocate the memory for the new + object. + #. :c:member:`~PyTypeObject.tp_init` initializes the newly created object. + :c:member:`!tp_init` can be called again to re-initialize an object, if + desired. The :c:member:`!tp_init` call can also be skipped entirely, + for example by Python code calling :py:meth:`~object.__new__`. + + * After :c:member:`!tp_init` completes, the object is ready to use. + * Some time after the last reference to an object is removed: + + #. If an object is not marked as *finalized*, it might be finalized by + marking it as *finalized* and calling its + :c:member:`~PyTypeObject.tp_finalize` function. Python does + *not* finalize an object when the last reference to it is deleted; use + :c:func:`PyObject_CallFinalizerFromDealloc` to ensure that + :c:member:`~PyTypeObject.tp_finalize` is always called. + #. If the object is marked as finalized, + :c:member:`~PyTypeObject.tp_clear` might be called by the garbage collector + to clear references held by the object. It is *not* called when the + object's reference count reaches zero. + #. :c:member:`~PyTypeObject.tp_dealloc` is called to destroy the object. + To avoid code duplication, :c:member:`~PyTypeObject.tp_dealloc` typically + calls into :c:member:`~PyTypeObject.tp_clear` to free up the object's + references. + #. When :c:member:`~PyTypeObject.tp_dealloc` finishes object destruction, + it directly calls :c:member:`~PyTypeObject.tp_free` (usually set to + :c:func:`PyObject_Free` or :c:func:`PyObject_GC_Del` automatically as + appropriate for the type) to deallocate the memory. + + * The :c:member:`~PyTypeObject.tp_finalize` function is permitted to add a + reference to the object if desired. If it does, the object is + *resurrected*, preventing its pending destruction. (Only + :c:member:`!tp_finalize` is allowed to resurrect an object; + :c:member:`~PyTypeObject.tp_clear` and + :c:member:`~PyTypeObject.tp_dealloc` cannot without calling into + :c:member:`!tp_finalize`.) Resurrecting an object may + or may not cause the object's *finalized* mark to be removed. Currently, + Python does not remove the *finalized* mark from a resurrected object if + it supports garbage collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` + flag is set) but does remove the mark if the object does not support + garbage collection; either or both of these behaviors may change in the + future. + * :c:member:`~PyTypeObject.tp_dealloc` can optionally call + :c:member:`~PyTypeObject.tp_finalize` via + :c:func:`PyObject_CallFinalizerFromDealloc` if it wishes to reuse that + code to help with object destruction. This is recommended because it + guarantees that :c:member:`!tp_finalize` is always called before + destruction. See the :c:member:`~PyTypeObject.tp_dealloc` documentation + for example code. + * If the object is a member of a :term:`cyclic isolate` and either + :c:member:`~PyTypeObject.tp_clear` fails to break the reference cycle or + the cyclic isolate is not detected (perhaps :func:`gc.disable` was called, + or the :c:macro:`Py_TPFLAGS_HAVE_GC` flag was erroneously omitted in one + of the involved types), the objects remain indefinitely uncollectable + (they "leak"). See :data:`gc.garbage`. + + If the object is marked as supporting garbage collection (the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`), the following events are also possible: + + * The garbage collector occasionally calls + :c:member:`~PyTypeObject.tp_traverse` to identify :term:`cyclic isolates + `. + * When the garbage collector discovers a :term:`cyclic isolate`, it + finalizes one of the objects in the group by marking it as *finalized* and + calling its :c:member:`~PyTypeObject.tp_finalize` function, if it has one. + This repeats until the cyclic isolate doesn't exist or all of the objects + have been finalized. + * :c:member:`~PyTypeObject.tp_finalize` is permitted to resurrect the object + by adding a reference from outside the :term:`cyclic isolate`. The new + reference causes the group of objects to no longer form a cyclic isolate + (the reference cycle may still exist, but if it does the objects are no + longer isolated). + * When the garbage collector discovers a :term:`cyclic isolate` and all of + the objects in the group have already been marked as *finalized*, the + garbage collector clears one or more of the uncleared objects in the group + (possibly concurrently) by calling each's + :c:member:`~PyTypeObject.tp_clear` function. This repeats as long as the + cyclic isolate still exists and not all of the objects have been cleared. + + +Cyclic Isolate Destruction +-------------------------- + +Listed below are the stages of life of a hypothetical :term:`cyclic isolate` +that continues to exist after each member object is finalized or cleared. It +is a memory leak if a cyclic isolate progresses through all of these stages; it should +vanish once all objects are cleared, if not sooner. A cyclic isolate can +vanish either because the reference cycle is broken or because the objects are +no longer isolated due to finalizer resurrection (see +:c:member:`~PyTypeObject.tp_finalize`). + +0. **Reachable** (not yet a cyclic isolate): All objects are in their normal, + reachable state. A reference cycle could exist, but an external reference + means the objects are not yet isolated. +#. **Unreachable but consistent:** The final reference from outside the cyclic + group of objects has been removed, causing the objects to become isolated + (thus a cyclic isolate is born). None of the group's objects have been + finalized or cleared yet. The cyclic isolate remains at this stage until + some future run of the garbage collector (not necessarily the next run + because the next run might not scan every object). +#. **Mix of finalized and not finalized:** Objects in a cyclic isolate are + finalized one at a time, which means that there is a period of time when the + cyclic isolate is composed of a mix of finalized and non-finalized objects. + Finalization order is unspecified, so it can appear random. A finalized + object must behave in a sane manner when non-finalized objects interact with + it, and a non-finalized object must be able to tolerate the finalization of + an arbitrary subset of its referents. +#. **All finalized:** All objects in a cyclic isolate are finalized before any + of them are cleared. +#. **Mix of finalized and cleared:** The objects can be cleared serially or + concurrently (but with the :term:`GIL` held); either way, some will finish + before others. A finalized object must be able to tolerate the clearing of + a subset of its referents. :pep:`442` calls this stage "cyclic trash". +#. **Leaked:** If a cyclic isolate still exists after all objects in the group + have been finalized and cleared, then the objects remain indefinitely + uncollectable (see :data:`gc.garbage`). It is a bug if a cyclic isolate + reaches this stage---it means the :c:member:`~PyTypeObject.tp_clear` methods + of the participating objects have failed to break the reference cycle as + required. + +If :c:member:`~PyTypeObject.tp_clear` did not exist, then Python would have no +way to safely break a reference cycle. Simply destroying an object in a cyclic +isolate would result in a dangling pointer, triggering undefined behavior when +an object referencing the destroyed object is itself destroyed. The clearing +step makes object destruction a two-phase process: first +:c:member:`~PyTypeObject.tp_clear` is called to partially destroy the objects +enough to detangle them from each other, then +:c:member:`~PyTypeObject.tp_dealloc` is called to complete the destruction. + +Unlike clearing, finalization is not a phase of destruction. A finalized +object must still behave properly by continuing to fulfill its design +contracts. An object's finalizer is allowed to execute arbitrary Python code, +and is even allowed to prevent the impending destruction by adding a reference. +The finalizer is only related to destruction by call order---if it runs, it runs +before destruction, which starts with :c:member:`~PyTypeObject.tp_clear` (if +called) and concludes with :c:member:`~PyTypeObject.tp_dealloc`. + +The finalization step is not necessary to safely reclaim the objects in a +cyclic isolate, but its existence makes it easier to design types that behave +in a sane manner when objects are cleared. Clearing an object might +necessarily leave it in a broken, partially destroyed state---it might be +unsafe to call any of the cleared object's methods or access any of its +attributes. With finalization, only finalized objects can possibly interact +with cleared objects; non-finalized objects are guaranteed to interact with +only non-cleared (but potentially finalized) objects. + +To summarize the possible interactions: + +* A non-finalized object might have references to or from non-finalized and + finalized objects, but not to or from cleared objects. +* A finalized object might have references to or from non-finalized, finalized, + and cleared objects. +* A cleared object might have references to or from finalized and cleared + objects, but not to or from non-finalized objects. + +Without any reference cycles, an object can be simply destroyed once its last +reference is deleted; the finalization and clearing steps are not necessary to +safely reclaim unused objects. However, it can be useful to automatically call +:c:member:`~PyTypeObject.tp_finalize` and :c:member:`~PyTypeObject.tp_clear` +before destruction anyway because type design is simplified when all objects +always experience the same series of events regardless of whether they +participated in a cyclic isolate. Python currently only calls +:c:member:`~PyTypeObject.tp_finalize` and :c:member:`~PyTypeObject.tp_clear` as +needed to destroy a cyclic isolate; this may change in a future version. + + +Functions +--------- + +To allocate and free memory, see :ref:`allocating-objects`. + + +.. c:function:: void PyObject_CallFinalizer(PyObject *op) + + Finalizes the object as described in :c:member:`~PyTypeObject.tp_finalize`. + Call this function (or :c:func:`PyObject_CallFinalizerFromDealloc`) instead + of calling :c:member:`~PyTypeObject.tp_finalize` directly because this + function may deduplicate multiple calls to :c:member:`!tp_finalize`. + Currently, calls are only deduplicated if the type supports garbage + collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set); this may + change in the future. + + .. versionadded:: 3.4 + + +.. c:function:: int PyObject_CallFinalizerFromDealloc(PyObject *op) + + Same as :c:func:`PyObject_CallFinalizer` but meant to be called at the + beginning of the object's destructor (:c:member:`~PyTypeObject.tp_dealloc`). + There must not be any references to the object. If the object's finalizer + resurrects the object, this function returns -1; no further destruction + should happen. Otherwise, this function returns 0 and destruction can + continue normally. + + .. versionadded:: 3.4 + + .. seealso:: + + :c:member:`~PyTypeObject.tp_dealloc` for example code. diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 9ff3e5265004a1..790ec8da109ba8 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -40,9 +40,11 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Return a new :c:type:`PyLongObject` object from *v*, or ``NULL`` on failure. - The current implementation keeps an array of integer objects for all integers - between ``-5`` and ``256``. When you create an int in that range you actually - just get back a reference to the existing object. + .. impl-detail:: + + CPython keeps an array of integer objects for all integers + between ``-5`` and ``1024``. When you create an int in that range + you actually just get back a reference to the existing object. .. c:function:: PyObject* PyLong_FromUnsignedLong(unsigned long v) @@ -159,6 +161,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_FromPid(pid) + + Macro for creating a Python integer from a process identifier. + + This can be defined as an alias to :c:func:`PyLong_FromLong` or + :c:func:`PyLong_FromLongLong`, depending on the size of the system's + PID type. + + .. versionadded:: 3.2 + + .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: @@ -372,6 +385,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Set *\*value* to a signed C :c:expr:`int32_t` or :c:expr:`int64_t` representation of *obj*. + If *obj* is not an instance of :c:type:`PyLongObject`, first call its + :meth:`~object.__index__` method (if present) to convert it to a + :c:type:`PyLongObject`. + If the *obj* value is out of range, raise an :exc:`OverflowError`. Set *\*value* and return ``0`` on success. @@ -436,10 +453,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Otherwise, returns the number of bytes required to store the value. If this is equal to or less than *n_bytes*, the entire value was copied. - All *n_bytes* of the buffer are written: large buffers are padded with - zeroes. + All *n_bytes* of the buffer are written: remaining bytes filled by + copies of the sign bit. - If the returned value is greater than than *n_bytes*, the value was + If the returned value is greater than *n_bytes*, the value was truncated: as many of the lowest bits of the value as could fit are written, and the higher bits are ignored. This matches the typical behavior of a C-style downcast. @@ -569,6 +586,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_AsPid(pid) + + Macro for converting a Python integer into a process identifier. + + This can be defined as an alias to :c:func:`PyLong_AsLong`, + :c:func:`PyLong_FromLongLong`, or :c:func:`PyLong_AsInt`, depending on the + size of the system's PID type. + + .. versionadded:: 3.2 + + .. c:function:: int PyLong_GetSign(PyObject *obj, int *sign) Get the sign of the integer object *obj*. @@ -582,6 +610,39 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.14 +.. c:function:: int PyLong_IsPositive(PyObject *obj) + + Check if the integer object *obj* is positive (``obj > 0``). + + If *obj* is an instance of :c:type:`PyLongObject` or its subtype, + return ``1`` when it's positive and ``0`` otherwise. Else set an + exception and return ``-1``. + + .. versionadded:: 3.14 + + +.. c:function:: int PyLong_IsNegative(PyObject *obj) + + Check if the integer object *obj* is negative (``obj < 0``). + + If *obj* is an instance of :c:type:`PyLongObject` or its subtype, + return ``1`` when it's negative and ``0`` otherwise. Else set an + exception and return ``-1``. + + .. versionadded:: 3.14 + + +.. c:function:: int PyLong_IsZero(PyObject *obj) + + Check if the integer object *obj* is zero. + + If *obj* is an instance of :c:type:`PyLongObject` or its subtype, + return ``1`` when it's zero and ``0`` otherwise. Else set an + exception and return ``-1``. + + .. versionadded:: 3.14 + + .. c:function:: PyObject* PyLong_GetInfo(void) On success, return a read only :term:`named tuple`, that holds @@ -620,3 +681,205 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.12 + +Export API +^^^^^^^^^^ + +.. versionadded:: 3.14 + +.. c:type:: PyLongLayout + + Layout of an array of "digits" ("limbs" in the GMP terminology), used to + represent absolute value for arbitrary precision integers. + + Use :c:func:`PyLong_GetNativeLayout` to get the native layout of Python + :class:`int` objects, used internally for integers with "big enough" + absolute value. + + See also :data:`sys.int_info` which exposes similar information in Python. + + .. c:member:: uint8_t bits_per_digit + + Bits per digit. For example, a 15 bit digit means that bits 0-14 contain + meaningful information. + + .. c:member:: uint8_t digit_size + + Digit size in bytes. For example, a 15 bit digit will require at least 2 + bytes. + + .. c:member:: int8_t digits_order + + Digits order: + + - ``1`` for most significant digit first + - ``-1`` for least significant digit first + + .. c:member:: int8_t digit_endianness + + Digit endianness: + + - ``1`` for most significant byte first (big endian) + - ``-1`` for least significant byte first (little endian) + + +.. c:function:: const PyLongLayout* PyLong_GetNativeLayout(void) + + Get the native layout of Python :class:`int` objects. + + See the :c:type:`PyLongLayout` structure. + + The function must not be called before Python initialization nor after + Python finalization. The returned layout is valid until Python is + finalized. The layout is the same for all Python sub-interpreters + in a process, and so it can be cached. + + +.. c:type:: PyLongExport + + Export of a Python :class:`int` object. + + There are two cases: + + * If :c:member:`digits` is ``NULL``, only use the :c:member:`value` member. + * If :c:member:`digits` is not ``NULL``, use :c:member:`negative`, + :c:member:`ndigits` and :c:member:`digits` members. + + .. c:member:: int64_t value + + The native integer value of the exported :class:`int` object. + Only valid if :c:member:`digits` is ``NULL``. + + .. c:member:: uint8_t negative + + ``1`` if the number is negative, ``0`` otherwise. + Only valid if :c:member:`digits` is not ``NULL``. + + .. c:member:: Py_ssize_t ndigits + + Number of digits in :c:member:`digits` array. + Only valid if :c:member:`digits` is not ``NULL``. + + .. c:member:: const void *digits + + Read-only array of unsigned digits. Can be ``NULL``. + + +.. c:function:: int PyLong_Export(PyObject *obj, PyLongExport *export_long) + + Export a Python :class:`int` object. + + *export_long* must point to a :c:type:`PyLongExport` structure allocated + by the caller. It must not be ``NULL``. + + On success, fill in *\*export_long* and return ``0``. + On error, set an exception and return ``-1``. + + :c:func:`PyLong_FreeExport` must be called when the export is no longer + needed. + + .. impl-detail:: + This function always succeeds if *obj* is a Python :class:`int` object + or a subclass. + + +.. c:function:: void PyLong_FreeExport(PyLongExport *export_long) + + Release the export *export_long* created by :c:func:`PyLong_Export`. + + .. impl-detail:: + Calling :c:func:`PyLong_FreeExport` is optional if *export_long->digits* + is ``NULL``. + + +PyLongWriter API +^^^^^^^^^^^^^^^^ + +The :c:type:`PyLongWriter` API can be used to import an integer. + +.. versionadded:: 3.14 + +.. c:type:: PyLongWriter + + A Python :class:`int` writer instance. + + The instance must be destroyed by :c:func:`PyLongWriter_Finish` or + :c:func:`PyLongWriter_Discard`. + + +.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) + + Create a :c:type:`PyLongWriter`. + + On success, allocate *\*digits* and return a writer. + On error, set an exception and return ``NULL``. + + *negative* is ``1`` if the number is negative, or ``0`` otherwise. + + *ndigits* is the number of digits in the *digits* array. It must be + greater than 0. + + *digits* must not be NULL. + + After a successful call to this function, the caller should fill in the + array of digits *digits* and then call :c:func:`PyLongWriter_Finish` to get + a Python :class:`int`. + The layout of *digits* is described by :c:func:`PyLong_GetNativeLayout`. + + Digits must be in the range [``0``; ``(1 << bits_per_digit) - 1``] + (where the :c:type:`~PyLongLayout.bits_per_digit` is the number of bits + per digit). + Any unused most significant digits must be set to ``0``. + + Alternately, call :c:func:`PyLongWriter_Discard` to destroy the writer + instance without creating an :class:`~int` object. + + +.. c:function:: PyObject* PyLongWriter_Finish(PyLongWriter *writer) + + Finish a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. + + On success, return a Python :class:`int` object. + On error, set an exception and return ``NULL``. + + The function takes care of normalizing the digits and converts the object + to a compact integer if needed. + + The writer instance and the *digits* array are invalid after the call. + + +.. c:function:: void PyLongWriter_Discard(PyLongWriter *writer) + + Discard a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. + + If *writer* is ``NULL``, no operation is performed. + + The writer instance and the *digits* array are invalid after the call. + + +Deprecated API +^^^^^^^^^^^^^^ + +These macros are :term:`soft deprecated`. They describe parameters +of the internal representation of :c:type:`PyLongObject` instances. + +Use :c:func:`PyLong_GetNativeLayout` instead, along with :c:func:`PyLong_Export` +to read integer data or :c:type:`PyLongWriter` to write it. +These currently use the same layout, but are designed to continue working correctly +even if CPython's internal integer representation changes. + + +.. c:macro:: PyLong_SHIFT + + This is equivalent to :c:member:`~PyLongLayout.bits_per_digit` in + the output of :c:func:`PyLong_GetNativeLayout`. + + +.. c:macro:: PyLong_BASE + + This is currently equivalent to :c:expr:`1 << PyLong_SHIFT`. + + +.. c:macro:: PyLong_MASK + + This is currently equivalent to :c:expr:`(1 << PyLong_SHIFT) - 1` diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 1f55c0aa955c75..2476ebb9b69dce 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -102,7 +102,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions which occur when this calls :meth:`~object.__getitem__` + Exceptions which occur when this calls the :meth:`~object.__getitem__` method are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyWithError`, :c:func:`PyMapping_GetOptionalItem` or :c:func:`PyObject_GetItem()` instead. @@ -116,7 +116,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions that occur when this calls :meth:`~object.__getitem__` + Exceptions that occur when this calls the :meth:`~object.__getitem__` method or while creating the temporary :class:`str` object are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyStringWithError`, diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index b9085ad3ec361d..668a163b2df5a1 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -13,11 +13,12 @@ binary mode. Numeric values are stored with the least significant byte first. -The module supports two versions of the data format: version 0 is the -historical version, version 1 shares interned strings in the file, and upon -unmarshalling. Version 2 uses a binary format for floating-point numbers. -``Py_MARSHAL_VERSION`` indicates the current file format (currently 2). +The module supports several versions of the data format; see +the :py:mod:`Python module documentation ` for details. +.. c:macro:: Py_MARSHAL_VERSION + + The current format version. See :py:data:`marshal.version`. .. c:function:: void PyMarshal_WriteLongToFile(long value, FILE *file, int version) @@ -81,7 +82,7 @@ The following functions allow marshalled values to be read back in. assumes that no further objects will be read from the file, allowing it to aggressively load file data into memory so that the de-serialization can operate from data in memory rather than reading a byte at a time from the - file. Only use these variant if you are certain that you won't be reading + file. Only use this variant if you are certain that you won't be reading anything else from the file. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index aa2ef499bddaf3..58f0de5d0fc541 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -102,7 +102,7 @@ All allocating functions belong to one of three different "domains" (see also strategies and are optimized for different purposes. The specific details on how every domain allocates memory or what internal functions each domain calls is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. +table can be found at :ref:`default-memory-allocators`. The APIs used to allocate and free a block of memory must be from the same domain. For example, :c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. @@ -110,12 +110,12 @@ The three allocation domains are: * Raw domain: intended for allocating memory for general-purpose memory buffers where the allocation *must* go to the system allocator or where the - allocator can operate without the :term:`GIL`. The memory is requested directly - from the system. See :ref:`Raw Memory Interface `. + allocator can operate without an :term:`attached thread state`. The memory + is requested directly from the system. See :ref:`Raw Memory Interface `. * "Mem" domain: intended for allocating memory for Python buffers and general-purpose memory buffers where the allocation must be performed with - the :term:`GIL` held. The memory is taken from the Python private heap. + an :term:`attached thread state`. The memory is taken from the Python private heap. See :ref:`Memory Interface `. * Object domain: intended for allocating memory for Python objects. The @@ -139,8 +139,8 @@ Raw Memory Interface ==================== The following function sets are wrappers to the system allocator. These -functions are thread-safe, the :term:`GIL ` does not -need to be held. +functions are thread-safe, so a :term:`thread state` does not +need to be :term:`attached `. The :ref:`default raw memory allocator ` uses the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` @@ -213,8 +213,7 @@ The :ref:`default memory allocator ` uses the .. warning:: - The :term:`GIL ` must be held when using these - functions. + There must be an :term:`attached thread state` when using these functions. .. versionchanged:: 3.6 @@ -294,17 +293,39 @@ The following type-oriented macros are provided for convenience. Note that Same as :c:func:`PyMem_Free`. -In addition, the following macro sets are provided for calling the Python memory -allocator directly, without involving the C API functions listed above. However, -note that their use does not preserve binary compatibility across Python -versions and is therefore deprecated in extension modules. -* ``PyMem_MALLOC(size)`` -* ``PyMem_NEW(type, size)`` -* ``PyMem_REALLOC(ptr, size)`` -* ``PyMem_RESIZE(ptr, type, size)`` -* ``PyMem_FREE(ptr)`` -* ``PyMem_DEL(ptr)`` +Deprecated aliases +------------------ + +These are :term:`soft deprecated` aliases to existing functions and macros. +They exist solely for backwards compatibility. + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Deprecated alias + * Corresponding function or macro + * * .. c:macro:: PyMem_MALLOC(size) + * :c:func:`PyMem_Malloc` + * * .. c:macro:: PyMem_NEW(type, size) + * :c:macro:`PyMem_New` + * * .. c:macro:: PyMem_REALLOC(ptr, size) + * :c:func:`PyMem_Realloc` + * * .. c:macro:: PyMem_RESIZE(ptr, type, size) + * :c:macro:`PyMem_Resize` + * * .. c:macro:: PyMem_FREE(ptr) + * :c:func:`PyMem_Free` + * * .. c:macro:: PyMem_DEL(ptr) + * :c:func:`PyMem_Free` + +.. versionchanged:: 3.4 + + The macros are now aliases of the corresponding functions and macros. + Previously, their behavior was the same, but their use did not necessarily + preserve binary compatibility across Python versions. + +.. deprecated:: 2.0 .. _objectinterface: @@ -327,8 +348,7 @@ The :ref:`default object allocator ` uses the .. warning:: - The :term:`GIL ` must be held when using these - functions. + There must be an :term:`attached thread state` when using these functions. .. c:function:: void* PyObject_Malloc(size_t n) @@ -378,6 +398,24 @@ The :ref:`default object allocator ` uses the If *p* is ``NULL``, no operation is performed. + Do not call this directly to free an object's memory; call the type's + :c:member:`~PyTypeObject.tp_free` slot instead. + + Do not use this for memory allocated by :c:macro:`PyObject_GC_New` or + :c:macro:`PyObject_GC_NewVar`; use :c:func:`PyObject_GC_Del` instead. + + .. seealso:: + + * :c:func:`PyObject_GC_Del` is the equivalent of this function for memory + allocated by types that support garbage collection. + * :c:func:`PyObject_Malloc` + * :c:func:`PyObject_Realloc` + * :c:func:`PyObject_Calloc` + * :c:macro:`PyObject_New` + * :c:macro:`PyObject_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_free` + .. _default-memory-allocators: @@ -485,12 +523,12 @@ Customize Memory Allocators zero bytes. For the :c:macro:`PYMEM_DOMAIN_RAW` domain, the allocator must be - thread-safe: the :term:`GIL ` is not held when the - allocator is called. + thread-safe: a :term:`thread state` is not :term:`attached ` + when the allocator is called. For the remaining domains, the allocator must also be thread-safe: the allocator may be called in different interpreters that do not - share a ``GIL``. + share a :term:`GIL`. If the new allocator is not a hook (does not call the previous allocator), the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the @@ -507,8 +545,8 @@ Customize Memory Allocators :c:func:`Py_InitializeFromConfig` to install a custom memory allocator. There are no restrictions over the installed allocator other than the ones imposed by the domain (for instance, the Raw - Domain allows the allocator to be called without the GIL held). See - :ref:`the section on allocator domains ` for more + Domain allows the allocator to be called without an :term:`attached thread state`). + See :ref:`the section on allocator domains ` for more information. * If called after Python has finish initializing (after @@ -555,7 +593,7 @@ Runtime checks: called on a memory block allocated by :c:func:`PyMem_Malloc`. - Detect write before the start of the buffer (buffer underflow). - Detect write after the end of the buffer (buffer overflow). -- Check that the :term:`GIL ` is held when +- Check that there is an :term:`attached thread state` when allocator functions of :c:macro:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) and :c:macro:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) domains are called. @@ -620,8 +658,8 @@ PYMEM_CLEANBYTE (meaning uninitialized memory is getting used). The :c:func:`PyMem_SetupDebugHooks` function now also works on Python compiled in release mode. On error, the debug hooks now use :mod:`tracemalloc` to get the traceback where a memory block was allocated. - The debug hooks now also check if the GIL is held when functions of - :c:macro:`PYMEM_DOMAIN_OBJ` and :c:macro:`PYMEM_DOMAIN_MEM` domains are + The debug hooks now also check if there is an :term:`attached thread state` when + functions of :c:macro:`PYMEM_DOMAIN_OBJ` and :c:macro:`PYMEM_DOMAIN_MEM` domains are called. .. versionchanged:: 3.8 @@ -639,7 +677,11 @@ The pymalloc allocator Python has a *pymalloc* allocator optimized for small objects (smaller or equal to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit -platforms. It falls back to :c:func:`PyMem_RawMalloc` and +platforms. When Python is configured with :option:`--with-pymalloc-hugepages`, +the arena size on 64-bit platforms is increased to 2 MiB to match the huge page +size, and arena allocation will attempt to use huge pages (``MAP_HUGETLB`` on +Linux, ``MEM_LARGE_PAGES`` on Windows) with automatic fallback to regular pages. +It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the :ref:`default allocator ` of the @@ -656,6 +698,10 @@ This allocator is disabled if Python is configured with the :option:`--without-pymalloc` option. It can also be disabled at runtime using the :envvar:`PYTHONMALLOC` environment variable (ex: ``PYTHONMALLOC=malloc``). +Typically, it makes sense to disable the pymalloc allocator when building +Python with AddressSanitizer (:option:`--with-address-sanitizer`) which helps +uncover low level bugs within the C code. + Customize pymalloc Arena Allocator ---------------------------------- diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index f6038032805259..e4ac8b57673407 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -13,6 +13,12 @@ A :class:`memoryview` object exposes the C level :ref:`buffer interface any other object. +.. c:var:: PyTypeObject PyMemoryView_Type + + This instance of :c:type:`PyTypeObject` represents the Python memoryview + type. This is the same object as :class:`memoryview` in the Python layer. + + .. c:function:: PyObject *PyMemoryView_FromObject(PyObject *obj) Create a memoryview object from an object that provides the buffer interface. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index f82a050ab75de0..e8a6e09f5554ec 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -3,17 +3,16 @@ .. _moduleobjects: Module Objects --------------- +============== .. index:: pair: object; module - .. c:var:: PyTypeObject PyModule_Type .. index:: single: ModuleType (in module types) This instance of :c:type:`PyTypeObject` represents the Python module type. This - is exposed to Python programs as ``types.ModuleType``. + is exposed to Python programs as :py:class:`types.ModuleType`. .. c:function:: int PyModule_Check(PyObject *p) @@ -71,6 +70,9 @@ Module Objects ``PyObject_*`` functions rather than directly manipulate a module's :attr:`~object.__dict__`. + The returned reference is borrowed from the module; it is valid until + the module is destroyed. + .. c:function:: PyObject* PyModule_GetNameObject(PyObject *module) @@ -90,18 +92,19 @@ Module Objects Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to ``'utf-8'``. -.. c:function:: void* PyModule_GetState(PyObject *module) - - Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or ``NULL``. See - :c:member:`PyModuleDef.m_size`. - + The returned buffer is only valid until the module is renamed or destroyed. + Note that Python code may rename a module by setting its :py:attr:`~module.__name__` + attribute. .. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) Return a pointer to the :c:type:`PyModuleDef` struct from which the module was created, or ``NULL`` if the module wasn't created from a definition. + On error, return ``NULL`` with an exception set. + Use :c:func:`PyErr_Occurred` to tell this case apart from a missing + :c:type:`!PyModuleDef`. + .. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module) @@ -122,243 +125,196 @@ Module Objects Similar to :c:func:`PyModule_GetFilenameObject` but return the filename encoded to 'utf-8'. + The returned buffer is only valid until the module's :py:attr:`~module.__file__` attribute + is reassigned or the module is destroyed. + .. deprecated:: 3.2 :c:func:`PyModule_GetFilename` raises :exc:`UnicodeEncodeError` on unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. -.. _initializing-modules: - -Initializing C modules -^^^^^^^^^^^^^^^^^^^^^^ +.. _pymoduledef_slot: -Modules objects are usually created from extension modules (shared libraries -which export an initialization function), or compiled-in modules -(where the initialization function is added using :c:func:`PyImport_AppendInittab`). -See :ref:`building` or :ref:`extending-with-embedding` for details. +Module definition +----------------- -The initialization function can either pass a module definition instance -to :c:func:`PyModule_Create`, and return the resulting module object, -or request "multi-phase initialization" by returning the definition struct itself. +Modules created using the C API are typically defined using an +array of :dfn:`slots`. +The slots provide a "description" of how a module should be created. -.. c:type:: PyModuleDef - - The module definition struct, which holds all information needed to create - a module object. There is usually only one statically initialized variable - of this type for each module. - - .. c:member:: PyModuleDef_Base m_base - - Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`. +.. versionchanged:: 3.15 - .. c:member:: const char *m_name - - Name for the new module. + Previously, a :c:type:`PyModuleDef` struct was necessary to define modules. + The older way of defining modules is still available: consult either the + :ref:`pymoduledef` section or earlier versions of this documentation + if you plan to support earlier Python versions. - .. c:member:: const char *m_doc +The slots array is usually used to define an extension module's “main” +module object (see :ref:`extension-modules` for details). +It can also be used to +:ref:`create extension modules dynamically `. - Docstring for the module; usually a docstring variable created with - :c:macro:`PyDoc_STRVAR` is used. - - .. c:member:: Py_ssize_t m_size +Unless specified otherwise, the same slot ID may not be repeated +in an array of slots. - Module state may be kept in a per-module memory area that can be - retrieved with :c:func:`PyModule_GetState`, rather than in static globals. - This makes modules safe for use in multiple sub-interpreters. - This memory area is allocated based on *m_size* on module creation, - and freed when the module object is deallocated, after the - :c:member:`~PyModuleDef.m_free` function has been called, if present. +.. c:type:: PyModuleDef_Slot - Setting ``m_size`` to ``-1`` means that the module does not support - sub-interpreters, because it has global state. + .. c:member:: int slot - Setting it to a non-negative value means that the module can be - re-initialized and specifies the additional amount of memory it requires - for its state. Non-negative ``m_size`` is required for multi-phase - initialization. + A slot ID, chosen from the available ``Py_mod_*`` values explained below. - See :PEP:`3121` for more details. + An ID of 0 marks the end of a :c:type:`!PyModuleDef_Slot` array. - .. c:member:: PyMethodDef* m_methods + .. c:member:: void* value - A pointer to a table of module-level functions, described by - :c:type:`PyMethodDef` values. Can be ``NULL`` if no functions are present. + Value of the slot, whose meaning depends on the slot ID. - .. c:member:: PyModuleDef_Slot* m_slots + The value may not be NULL. + To leave a slot out, omit the :c:type:`PyModuleDef_Slot` entry entirely. - An array of slot definitions for multi-phase initialization, terminated by - a ``{0, NULL}`` entry. - When using single-phase initialization, *m_slots* must be ``NULL``. + .. versionadded:: 3.5 - .. versionchanged:: 3.5 - Prior to version 3.5, this member was always set to ``NULL``, - and was defined as: +Metadata slots +.............. - .. c:member:: inquiry m_reload +.. c:macro:: Py_mod_name - .. c:member:: traverseproc m_traverse + :c:type:`Slot ID ` for the name of the new module, + as a NUL-terminated UTF8-encoded ``const char *``. - A traversal function to call during GC traversal of the module object, or - ``NULL`` if not needed. + Note that modules are typically created using a + :py:class:`~importlib.machinery.ModuleSpec`, and when they are, the + name from the spec will be used instead of :c:data:`!Py_mod_name`. + However, it is still recommended to include this slot for introspection + and debugging purposes. - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. + .. versionadded:: 3.15 - .. versionchanged:: 3.9 - No longer called before the module state is allocated. + Use :c:member:`PyModuleDef.m_name` instead to support previous versions. - .. c:member:: inquiry m_clear +.. c:macro:: Py_mod_doc - A clear function to call during GC clearing of the module object, or - ``NULL`` if not needed. + :c:type:`Slot ID ` for the docstring of the new + module, as a NUL-terminated UTF8-encoded ``const char *``. - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. + Usually it is set to a variable created with :c:macro:`PyDoc_STRVAR`. - Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* - called before a module is deallocated. For example, when reference - counting is enough to determine that an object is no longer used, - the cyclic garbage collector is not involved and - :c:member:`~PyModuleDef.m_free` is called directly. + .. versionadded:: 3.15 - .. versionchanged:: 3.9 - No longer called before the module state is allocated. + Use :c:member:`PyModuleDef.m_doc` instead to support previous versions. - .. c:member:: freefunc m_free - A function to call during deallocation of the module object, or ``NULL`` - if not needed. +Feature slots +............. - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. +.. c:macro:: Py_mod_abi - .. versionchanged:: 3.9 - No longer called before the module state is allocated. + :c:type:`Slot ID ` whose value points to + a :c:struct:`PyABIInfo` structure describing the ABI that + the extension is using. -Single-phase initialization -........................... + A suitable :c:struct:`!PyABIInfo` variable can be defined using the + :c:macro:`PyABIInfo_VAR` macro, as in: -The module initialization function may create and return the module object -directly. This is referred to as "single-phase initialization", and uses one -of the following two module creation functions: + .. code-block:: c -.. c:function:: PyObject* PyModule_Create(PyModuleDef *def) + PyABIInfo_VAR(abi_info); - Create a new module object, given the definition in *def*. This behaves - like :c:func:`PyModule_Create2` with *module_api_version* set to - :c:macro:`PYTHON_API_VERSION`. + static PyModuleDef_Slot mymodule_slots[] = { + {Py_mod_abi, &abi_info}, + ... + }; + When creating a module, Python checks the value of this slot + using :c:func:`PyABIInfo_Check`. -.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version) + .. versionadded:: 3.15 - Create a new module object, given the definition in *def*, assuming the - API version *module_api_version*. If that version does not match the version - of the running interpreter, a :exc:`RuntimeWarning` is emitted. +.. c:macro:: Py_mod_multiple_interpreters - Return ``NULL`` with an exception set on error. + :c:type:`Slot ID ` whose value is one of: - .. note:: + .. c:namespace:: NULL - Most uses of this function should be using :c:func:`PyModule_Create` - instead; only use this if you are sure you need it. + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED -Before it is returned from in the initialization function, the resulting module -object is typically populated using functions like :c:func:`PyModule_AddObjectRef`. + The module does not support being imported in subinterpreters. -.. _multi-phase-initialization: + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED -Multi-phase initialization -.......................... + The module supports being imported in subinterpreters, + but only when they share the main interpreter's GIL. + (See :ref:`isolating-extensions-howto`.) -An alternate way to specify extensions is to request "multi-phase initialization". -Extension modules created this way behave more like Python modules: the -initialization is split between the *creation phase*, when the module object -is created, and the *execution phase*, when it is populated. -The distinction is similar to the :py:meth:`!__new__` and :py:meth:`!__init__` methods -of classes. + .. c:macro:: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -Unlike modules created using single-phase initialization, these modules are not -singletons: if the *sys.modules* entry is removed and the module is re-imported, -a new module object is created, and the old module is subject to normal garbage -collection -- as with Python modules. -By default, multiple modules created from the same definition should be -independent: changes to one should not affect the others. -This means that all state should be specific to the module object (using e.g. -using :c:func:`PyModule_GetState`), or its contents (such as the module's -:attr:`~object.__dict__` or individual classes created with :c:func:`PyType_FromSpec`). + The module supports being imported in subinterpreters, + even when they have their own GIL. + (See :ref:`isolating-extensions-howto`.) -All modules created using multi-phase initialization are expected to support -:ref:`sub-interpreters `. Making sure multiple modules -are independent is typically enough to achieve this. + This slot determines whether or not importing this module + in a subinterpreter will fail. -To request multi-phase initialization, the initialization function -(PyInit_modulename) returns a :c:type:`PyModuleDef` instance with non-empty -:c:member:`~PyModuleDef.m_slots`. Before it is returned, the ``PyModuleDef`` -instance must be initialized with the following function: + If ``Py_mod_multiple_interpreters`` is not specified, the import + machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED``. -.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) + .. versionadded:: 3.12 - Ensures a module definition is a properly initialized Python object that - correctly reports its type and reference count. +.. c:macro:: Py_mod_gil - Returns *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. + :c:type:`Slot ID ` whose value is one of: - .. versionadded:: 3.5 + .. c:namespace:: NULL -The *m_slots* member of the module definition must point to an array of -``PyModuleDef_Slot`` structures: + .. c:macro:: Py_MOD_GIL_USED -.. c:type:: PyModuleDef_Slot + The module depends on the presence of the global interpreter lock (GIL), + and may access global state without synchronization. - .. c:member:: int slot + .. c:macro:: Py_MOD_GIL_NOT_USED - A slot ID, chosen from the available values explained below. + The module is safe to run without an active GIL. - .. c:member:: void* value + This slot is ignored by Python builds not configured with + :option:`--disable-gil`. Otherwise, it determines whether or not importing + this module will cause the GIL to be automatically enabled. See + :ref:`whatsnew313-free-threaded-cpython` for more detail. - Value of the slot, whose meaning depends on the slot ID. + If ``Py_mod_gil`` is not specified, the import machinery defaults to + ``Py_MOD_GIL_USED``. - .. versionadded:: 3.5 + .. versionadded:: 3.13 -The *m_slots* array must be terminated by a slot with id 0. -The available slot types are: +Creation and initialization slots +................................. .. c:macro:: Py_mod_create - Specifies a function that is called to create the module object itself. - The *value* pointer of this slot must point to a function of the signature: + :c:type:`Slot ID ` for a function that creates + the module object itself. + The function must have the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) :no-index-entry: :no-contents-entry: - The function receives a :py:class:`~importlib.machinery.ModuleSpec` - instance, as defined in :PEP:`451`, and the module definition. - It should return a new module object, or set an error + The function will be called with: + + - *spec*: a ``ModuleSpec``-like object, meaning that any attributes defined + for :py:class:`importlib.machinery.ModuleSpec` have matching semantics. + However, any of the attributes may be missing. + - *def*: ``NULL``, or the module definition if the module is created from one. + + The function should return a new module object, or set an error and return ``NULL``. This function should be kept minimal. In particular, it should not call arbitrary Python code, as trying to import the same module again may result in an infinite loop. - Multiple ``Py_mod_create`` slots may not be specified in one module - definition. - If ``Py_mod_create`` is not specified, the import machinery will create a normal module object using :c:func:`PyModule_New`. The name is taken from *spec*, not the definition, to allow extension modules to dynamically adjust @@ -366,101 +322,474 @@ The available slot types are: names through symlinks, all while sharing a single module definition. There is no requirement for the returned object to be an instance of - :c:type:`PyModule_Type`. Any type can be used, as long as it supports - setting and getting import-related attributes. - However, only ``PyModule_Type`` instances may be returned if the - ``PyModuleDef`` has non-``NULL`` ``m_traverse``, ``m_clear``, - ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. + :c:type:`PyModule_Type`. + However, some slots may only be used with + :c:type:`!PyModule_Type` instances; in particular: + + - :c:macro:`Py_mod_exec`, + - :ref:`module state slots ` (``Py_mod_state_*``), + - :c:macro:`Py_mod_token`. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.15 + + The *slots* argument may be a ``ModuleSpec``-like object, rather than + a true :py:class:`~importlib.machinery.ModuleSpec` instance. + Note that previous versions of CPython did not enforce this. + + The *def* argument may now be ``NULL``, since modules are not necessarily + made from definitions. .. c:macro:: Py_mod_exec - Specifies a function that is called to *execute* the module. - This is equivalent to executing the code of a Python module: typically, - this function adds classes and constants to the module. + :c:type:`Slot ID ` for a function that will + :dfn:`execute`, or initialize, the module. + This function does the equivalent to executing the code of a Python module: + typically, it adds classes and constants to the module. The signature of the function is: .. c:function:: int exec_module(PyObject* module) :no-index-entry: :no-contents-entry: - If multiple ``Py_mod_exec`` slots are specified, they are processed in the - order they appear in the *m_slots* array. + See the :ref:`capi-module-support-functions` section for some useful + functions to call. -.. c:macro:: Py_mod_multiple_interpreters + For backwards compatibility, the :c:type:`PyModuleDef.m_slots` array may + contain multiple :c:macro:`!Py_mod_exec` slots; these are processed in the + order they appear in the array. + Elsewhere (that is, in arguments to :c:func:`PyModule_FromSlotsAndSpec` + and in return values of :samp:`PyModExport_{}`), repeating the slot + is not allowed. - Specifies one of the following values: + .. versionadded:: 3.5 - .. c:namespace:: NULL + .. versionchanged:: 3.15 - .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED + Repeated ``Py_mod_exec`` slots are disallowed, except in + :c:type:`PyModuleDef.m_slots`. - The module does not support being imported in subinterpreters. +.. c:macro:: Py_mod_methods - .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED + :c:type:`Slot ID ` for a table of module-level + functions, as an array of :c:type:`PyMethodDef` values suitable as the + *functions* argument to :c:func:`PyModule_AddFunctions`. - The module supports being imported in subinterpreters, - but only when they share the main interpreter's GIL. - (See :ref:`isolating-extensions-howto`.) + Like other slot IDs, a slots array may only contain one + :c:macro:`!Py_mod_methods` entry. + To add functions from multiple :c:type:`PyMethodDef` arrays, call + :c:func:`PyModule_AddFunctions` in the :c:macro:`Py_mod_exec` function. - .. c:macro:: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + The table must be statically allocated (or otherwise guaranteed to outlive + the module object). - The module supports being imported in subinterpreters, - even when they have their own GIL. - (See :ref:`isolating-extensions-howto`.) + .. versionadded:: 3.15 - This slot determines whether or not importing this module - in a subinterpreter will fail. + Use :c:member:`PyModuleDef.m_methods` instead to support previous versions. - Multiple ``Py_mod_multiple_interpreters`` slots may not be specified - in one module definition. +.. _ext-module-state: - If ``Py_mod_multiple_interpreters`` is not specified, the import - machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED``. +Module state +------------ - .. versionadded:: 3.12 +Extension modules can have *module state* -- a +piece of memory that is allocated on module creation, +and freed when the module object is deallocated. +The module state is specified using :ref:`dedicated slots `. -.. c:macro:: Py_mod_gil +A typical use of module state is storing an exception type -- or indeed *any* +type object defined by the module -- - Specifies one of the following values: +Unlike the module's Python attributes, Python code cannot replace or delete +data stored in module state. - .. c:namespace:: NULL +Keeping per-module information in attributes and module state, rather than in +static globals, makes module objects *isolated* and safer for use in +multiple sub-interpreters. +It also helps Python do an orderly clean-up when it shuts down. - .. c:macro:: Py_MOD_GIL_USED +Extensions that keep references to Python objects as part of module state must +implement :c:macro:`Py_mod_state_traverse` and :c:macro:`Py_mod_state_clear` +functions to avoid reference leaks. - The module depends on the presence of the global interpreter lock (GIL), - and may access global state without synchronization. +To retrieve the state from a given module, use the following functions: - .. c:macro:: Py_MOD_GIL_NOT_USED +.. c:function:: void* PyModule_GetState(PyObject *module) - The module is safe to run without an active GIL. + Return the "state" of the module, that is, a pointer to the block of memory + allocated at module creation time, or ``NULL``. See + :c:macro:`Py_mod_state_size`. - This slot is ignored by Python builds not configured with - :option:`--disable-gil`. Otherwise, it determines whether or not importing - this module will cause the GIL to be automatically enabled. See - :ref:`whatsnew313-free-threaded-cpython` for more detail. + On error, return ``NULL`` with an exception set. + Use :c:func:`PyErr_Occurred` to tell this case apart from missing + module state. - Multiple ``Py_mod_gil`` slots may not be specified in one module definition. - If ``Py_mod_gil`` is not specified, the import machinery defaults to - ``Py_MOD_GIL_USED``. +.. c:function:: int PyModule_GetStateSize(PyObject *module, Py_ssize_t *result) - .. versionadded:: 3.13 + Set *\*result* to the size of *module*'s state, as specified + using :c:macro:`Py_mod_state_size` (or :c:member:`PyModuleDef.m_size`), + and return 0. + + On error, set *\*result* to -1, and return -1 with an exception set. + + .. versionadded:: 3.15 + + + +.. _ext-module-state-slots: + +Slots for defining module state +............................... + +The following :c:member:`PyModuleDef_Slot.slot` IDs are available for +defining the module state. + +.. c:macro:: Py_mod_state_size + + :c:type:`Slot ID ` for the size of the module state, + in bytes. + + Setting the value to a non-negative value means that the module can be + re-initialized and specifies the additional amount of memory it requires + for its state. + + See :PEP:`3121` for more details. + + Use :c:func:`PyModule_GetStateSize` to retrieve the size of a given module. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_size` instead to support previous versions. + +.. c:macro:: Py_mod_state_traverse + + :c:type:`Slot ID ` for a traversal function to call + during GC traversal of the module object. + + The signature of the function, and meanings of the arguments, + is similar as for :c:member:`PyTypeObject.tp_traverse`: + + .. c:function:: int traverse_module_state(PyObject *module, visitproc visit, void *arg) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_size` instead to support previous versions. + +.. c:macro:: Py_mod_state_clear + + :c:type:`Slot ID ` for a clear function to call + during GC clearing of the module object. + + The signature of the function is: + + .. c:function:: int clear_module_state(PyObject* module) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* + called before a module is deallocated. For example, when reference + counting is enough to determine that an object is no longer used, + the cyclic garbage collector is not involved and + the :c:macro:`Py_mod_state_free` function is called directly. + + .. versionadded:: 3.15 -See :PEP:`489` for more details on multi-phase initialization. + Use :c:member:`PyModuleDef.m_clear` instead to support previous versions. -Low-level module creation functions -................................... +.. c:macro:: Py_mod_state_free -The following functions are called under the hood when using multi-phase -initialization. They can be used directly, for example when creating module -objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and -``PyModule_ExecDef`` must be called to fully initialize a module. + :c:type:`Slot ID ` for a function to call during + deallocation of the module object. + + The signature of the function is: + + .. c:function:: int free_module_state(PyObject* module) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_free` instead to support previous versions. + + +.. _ext-module-token: + +Module token +............ + +Each module may have an associated *token*: a pointer-sized value intended to +identify of the module state's memory layout. +This means that if you have a module object, but you are not sure if it +“belongs” to your extension, you can check using code like this: + +.. code-block:: c + + PyObject *module = + + void *module_token; + if (PyModule_GetToken(module, &module_token) < 0) { + return NULL; + } + if (module_token != your_token) { + PyErr_SetString(PyExc_ValueError, "unexpected module") + return NULL; + } + + // This module's state has the expected memory layout; it's safe to cast + struct my_state state = (struct my_state*)PyModule_GetState(module) + +A module's token -- and the *your_token* value to use in the above code -- is: + +- For modules created with :c:type:`PyModuleDef`: the address of that + :c:type:`PyModuleDef`; +- For modules defined with the :c:macro:`Py_mod_token` slot: the value + of that slot; +- For modules created from an ``PyModExport_*`` + :ref:`export hook `: the slots array that the export + hook returned (unless overridden with :c:macro:`Py_mod_token`). + +.. c:macro:: Py_mod_token + + :c:type:`Slot ID ` for the module token. + + If you use this slot to set the module token (rather than rely on the + default), you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + * If the token points to a :c:type:`PyModuleDef` struct, the module should + behave as if it was created from that :c:type:`PyModuleDef`. + In particular, the module state must have matching layout and semantics. + + Modules created from :c:type:`PyModuleDef` always use the address of + the :c:type:`PyModuleDef` as the token. + This means that :c:macro:`!Py_mod_token` cannot be used in + :c:member:`PyModuleDef.m_slots`. + + .. versionadded:: 3.15 + +.. c:function:: int PyModule_GetToken(PyObject *module, void** result) + + Set *\*result* to the module token for *module* and return 0. + + On error, set *\*result* to NULL, and return -1 with an exception set. + + .. versionadded:: 3.15 + +See also :c:func:`PyType_GetModuleByToken`. + + +.. _module-from-slots: + +Creating extension modules dynamically +-------------------------------------- + +The following functions may be used to create an extension module dynamically, +rather than from an extension's :ref:`export hook `. + +.. c:function:: PyObject *PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) + + Create a new module object, given an array of :ref:`slots ` + and the :py:class:`~importlib.machinery.ModuleSpec` *spec*. + + The *slots* argument must point to an array of :c:type:`PyModuleDef_Slot` + structures, terminated by an entry slot with slot ID of 0 + (typically written as ``{0}`` or ``{0, NULL}`` in C). + The *slots* argument may not be ``NULL``. + + The *spec* argument may be any ``ModuleSpec``-like object, as described + in :c:macro:`Py_mod_create` documentation. + Currently, the *spec* must have a ``name`` attribute. + + On success, return the new module. + On error, return ``NULL`` with an exception set. + + Note that this does not process the module's execution slot + (:c:data:`Py_mod_exec`). + Both :c:func:`!PyModule_FromSlotsAndSpec` and :c:func:`PyModule_Exec` + must be called to fully initialize a module. + (See also :ref:`multi-phase-initialization`.) + + The *slots* array only needs to be valid for the duration of the + :c:func:`!PyModule_FromSlotsAndSpec` call. + In particular, it may be heap-allocated. + + .. versionadded:: 3.15 + +.. c:function:: int PyModule_Exec(PyObject *module) + + Execute the :c:data:`Py_mod_exec` slot(s) of *module*. + + On success, return 0. + On error, return -1 with an exception set. + + For clarity: If *module* has no slots, for example if it uses + :ref:`legacy single-phase initialization `, + this function does nothing and returns 0. + + .. versionadded:: 3.15 + + + +.. _pymoduledef: + +Module definition struct +------------------------ + +Traditionally, extension modules were defined using a *module definition* +as the “description" of how a module should be created. +Rather than using an array of :ref:`slots ` directly, +the definition has dedicated members for most common functionality, +and allows additional slots as an extension mechanism. + +This way of defining modules is still available and there are no plans to +remove it. + +.. c:type:: PyModuleDef + + The module definition struct, which holds information needed to create + a module object. + + This structure must be statically allocated (or be otherwise guaranteed + to be valid while any modules created from it exist). + Usually, there is only one variable of this type for each extension module + defined this way. + + .. c:member:: PyModuleDef_Base m_base + + Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`: + + .. c:namespace:: NULL + + .. c:type:: PyModuleDef_Base + + The type of :c:member:`!PyModuleDef.m_base`. + + .. c:macro:: PyModuleDef_HEAD_INIT + + The required initial value for :c:member:`!PyModuleDef.m_base`. + + .. c:member:: const char *m_name + + Corresponds to the :c:macro:`Py_mod_name` slot. + + .. c:member:: const char *m_doc + + These members correspond to the :c:macro:`Py_mod_doc` slot. + Setting this to NULL is equivalent to omitting the slot. + + .. c:member:: Py_ssize_t m_size + + Corresponds to the :c:macro:`Py_mod_state_size` slot. + Setting this to zero is equivalent to omitting the slot. + + When using :ref:`legacy single-phase initialization ` + or when creating modules dynamically using :c:func:`PyModule_Create` + or :c:func:`PyModule_Create2`, :c:member:`!m_size` may be set to -1. + This indicates that the module does not support sub-interpreters, + because it has global state. + + .. c:member:: PyMethodDef *m_methods + + Corresponds to the :c:macro:`Py_mod_methods` slot. + Setting this to NULL is equivalent to omitting the slot. + + .. c:member:: PyModuleDef_Slot* m_slots + + An array of additional slots, terminated by a ``{0, NULL}`` entry. + + This array may not contain slots corresponding to :c:type:`PyModuleDef` + members. + For example, you cannot use :c:macro:`Py_mod_name` in :c:member:`!m_slots`; + the module name must be given as :c:member:`PyModuleDef.m_name`. + + .. versionchanged:: 3.5 + + Prior to version 3.5, this member was always set to ``NULL``, + and was defined as: + + .. c:member:: inquiry m_reload + + .. c:member:: traverseproc m_traverse + inquiry m_clear + freefunc m_free + + These members correspond to the :c:macro:`Py_mod_state_traverse`, + :c:macro:`Py_mod_state_clear`, and :c:macro:`Py_mod_state_free` slots, + respectively. + + Setting these members to NULL is equivalent to omitting the + corresponding slots. + + .. versionchanged:: 3.9 + + :c:member:`m_traverse`, :c:member:`m_clear` and :c:member:`m_free` + functions are longer called before the module state is allocated. + + +.. _moduledef-dynamic: + +The following API can be used to create modules from a :c:type:`!PyModuleDef` +struct: + +.. c:function:: PyObject* PyModule_Create(PyModuleDef *def) + + Create a new module object, given the definition in *def*. + This is a macro that calls :c:func:`PyModule_Create2` with + *module_api_version* set to :c:macro:`PYTHON_API_VERSION`, or + to :c:macro:`PYTHON_ABI_VERSION` if using the + :ref:`limited API `. + +.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version) + + Create a new module object, given the definition in *def*, assuming the + API version *module_api_version*. If that version does not match the version + of the running interpreter, a :exc:`RuntimeWarning` is emitted. + + Return ``NULL`` with an exception set on error. + + This function does not support slots. + The :c:member:`~PyModuleDef.m_slots` member of *def* must be ``NULL``. + + + .. note:: + + Most uses of this function should be using :c:func:`PyModule_Create` + instead; only use this if you are sure you need it. .. c:function:: PyObject * PyModule_FromDefAndSpec(PyModuleDef *def, PyObject *spec) - Create a new module object, given the definition in *def* and the - ModuleSpec *spec*. This behaves like :c:func:`PyModule_FromDefAndSpec2` - with *module_api_version* set to :c:macro:`PYTHON_API_VERSION`. + This macro calls :c:func:`PyModule_FromDefAndSpec2` with + *module_api_version* set to :c:macro:`PYTHON_API_VERSION`, or + to :c:macro:`PYTHON_ABI_VERSION` if using the + :ref:`limited API `. .. versionadded:: 3.5 @@ -473,6 +802,10 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and Return ``NULL`` with an exception set on error. + Note that this does not process execution slots (:c:data:`Py_mod_exec`). + Both ``PyModule_FromDefAndSpec`` and ``PyModule_ExecDef`` must be called + to fully initialize a module. + .. note:: Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec` @@ -486,35 +819,33 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and .. versionadded:: 3.5 -.. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) +.. c:macro:: PYTHON_API_VERSION + PYTHON_API_STRING - Set the docstring for *module* to *docstring*. - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. + The C API version, as an integer (``1013``) and string (``"1013"``), respectively. + Defined for backwards compatibility. - .. versionadded:: 3.5 + Currently, this constant is not updated in new Python versions, and is not + useful for versioning. This may change in the future. -.. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) +.. c:macro:: PYTHON_ABI_VERSION + PYTHON_ABI_STRING - Add the functions from the ``NULL`` terminated *functions* array to *module*. - Refer to the :c:type:`PyMethodDef` documentation for details on individual - entries (due to the lack of a shared module namespace, module level - "functions" implemented in C typically receive the module as their first - parameter, making them similar to instance methods on Python classes). - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. + Defined as ``3`` and ``"3"``, respectively, for backwards compatibility. + + Currently, this constant is not updated in new Python versions, and is not + useful for versioning. This may change in the future. - .. versionadded:: 3.5 + +.. _capi-module-support-functions: Support functions -................. +----------------- -The module initialization function (if using single phase initialization) or -a function called from a module execution slot (if using multi-phase -initialization), can use the following functions to help initialize the module -state: +The following functions are provided to help initialize a module object. +They are intended for a module's execution slot (:c:data:`Py_mod_exec`), +the initialization function for legacy :ref:`single-phase initialization `, +or code that creates modules dynamically. .. c:function:: int PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) @@ -523,9 +854,6 @@ state: On success, return ``0``. On error, raise an exception and return ``-1``. - Return ``-1`` if *value* is ``NULL``. It must be called with an exception - raised in this case. - Example usage:: static int @@ -540,6 +868,10 @@ state: return res; } + To be convenient, the function accepts ``NULL`` *value* with an exception + set. In this case, return ``-1`` and just leave the raised exception + unchanged. + The example can also be written without checking explicitly if *obj* is ``NULL``:: @@ -662,12 +994,45 @@ state: .. versionadded:: 3.9 +.. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) + + Add the functions from the ``NULL`` terminated *functions* array to *module*. + Refer to the :c:type:`PyMethodDef` documentation for details on individual + entries (due to the lack of a shared module namespace, module level + "functions" implemented in C typically receive the module as their first + parameter, making them similar to instance methods on Python classes). + + This function is called automatically when creating a module from + ``PyModuleDef`` (such as when using :ref:`multi-phase-initialization`, + ``PyModule_Create``, or ``PyModule_FromDefAndSpec``). + Some module authors may prefer defining functions in multiple + :c:type:`PyMethodDef` arrays; in that case they should call this function + directly. + + The *functions* array must be statically allocated (or otherwise guaranteed + to outlive the module object). + + .. versionadded:: 3.5 + +.. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) + + Set the docstring for *module* to *docstring*. + This function is called automatically when creating a module from + ``PyModuleDef`` (such as when using :ref:`multi-phase-initialization`, + ``PyModule_Create``, or ``PyModule_FromDefAndSpec``). + + Return ``0`` on success. + Return ``-1`` with an exception set on error. + + .. versionadded:: 3.5 + .. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) Indicate that *module* does or does not support running without the global interpreter lock (GIL), using one of the values from :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization - function. If this function is not called during module initialization, the + function when using :ref:`single-phase-initialization`. + If this function is not called during module initialization, the import machinery assumes the module does not support running without the GIL. This function is only available in Python builds configured with :option:`--disable-gil`. @@ -676,10 +1041,11 @@ state: .. versionadded:: 3.13 -Module lookup -^^^^^^^^^^^^^ +Module lookup (single-phase initialization) +........................................... -Single-phase initialization creates singleton modules that can be looked up +The legacy :ref:`single-phase initialization ` +initialization scheme creates singleton modules that can be looked up in the context of the current interpreter. This allows the module object to be retrieved later with only a reference to the module definition. @@ -700,7 +1066,8 @@ since multiple such modules can be created from a single definition. Only effective on modules created using single-phase initialization. - Python calls ``PyState_AddModule`` automatically after importing a module, + Python calls ``PyState_AddModule`` automatically after importing a module + that uses :ref:`single-phase initialization `, so it is unnecessary (but harmless) to call it from module initialization code. An explicit call is needed only if the module's own init code subsequently calls ``PyState_FindModule``. @@ -708,7 +1075,10 @@ since multiple such modules can be created from a single definition. mechanisms (either by calling it directly, or by referring to its implementation for details of the required state updates). - The caller must hold the GIL. + If a module was attached previously using the same *def*, it is replaced + by the new *module*. + + The caller must have an :term:`attached thread state`. Return ``-1`` with an exception set on error, ``0`` on success. @@ -719,6 +1089,6 @@ since multiple such modules can be created from a single definition. Removes the module object created from *def* from the interpreter state. Return ``-1`` with an exception set on error, ``0`` on success. - The caller must hold the GIL. + The caller must have an :term:`attached thread state`. .. versionadded:: 3.3 diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index 51d866cfd47469..b0227c2f4faf15 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -75,9 +75,14 @@ See :mod:`sys.monitoring` for descriptions of the events. Fire a ``JUMP`` event. -.. c:function:: int PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) +.. c:function:: int PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) - Fire a ``BRANCH`` event. + Fire a ``BRANCH_LEFT`` event. + + +.. c:function:: int PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``BRANCH_RIGHT`` event. .. c:function:: int PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *retval) @@ -131,7 +136,7 @@ Managing the Monitoring State ----------------------------- Monitoring states can be managed with the help of monitoring scopes. A scope -would typically correspond to a python function. +would typically correspond to a Python function. .. c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) @@ -168,7 +173,8 @@ would typically correspond to a python function. ================================================== ===================================== Macro Event ================================================== ===================================== - .. c:macro:: PY_MONITORING_EVENT_BRANCH :monitoring-event:`BRANCH` + .. c:macro:: PY_MONITORING_EVENT_BRANCH_LEFT :monitoring-event:`BRANCH_LEFT` + .. c:macro:: PY_MONITORING_EVENT_BRANCH_RIGHT :monitoring-event:`BRANCH_RIGHT` .. c:macro:: PY_MONITORING_EVENT_CALL :monitoring-event:`CALL` .. c:macro:: PY_MONITORING_EVENT_C_RAISE :monitoring-event:`C_RAISE` .. c:macro:: PY_MONITORING_EVENT_C_RETURN :monitoring-event:`C_RETURN` @@ -190,3 +196,15 @@ would typically correspond to a python function. .. c:function:: int PyMonitoring_ExitScope(void) Exit the last scope that was entered with :c:func:`!PyMonitoring_EnterScope`. + + +.. c:function:: int PY_MONITORING_IS_INSTRUMENTED_EVENT(uint8_t ev) + + Return true if the event corresponding to the event ID *ev* is + a :ref:`local event `. + + .. versionadded:: 3.13 + + .. deprecated:: 3.14 + + This function is :term:`soft deprecated`. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 630114a4339110..127b50ac479638 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -73,7 +73,7 @@ Object Protocol Flag to be used with multiple functions that print the object (like :c:func:`PyObject_Print` and :c:func:`PyFile_WriteObject`). - If passed, these function would use the :func:`str` of the object + If passed, these functions use the :func:`str` of the object instead of the :func:`repr`. @@ -85,7 +85,36 @@ Object Protocol instead of the :func:`repr`. -.. c:function:: int PyObject_HasAttrWithError(PyObject *o, const char *attr_name) +.. c:function:: void PyObject_Dump(PyObject *op) + + Dump an object *op* to ``stderr``. This should only be used for debugging. + + The output is intended to try dumping objects even after memory corruption: + + * Information is written starting with fields that are the least likely to + crash when accessed. + * This function can be called without an :term:`attached thread state`, but + it's not recommended to do so: it can cause deadlocks. + * An object that does not belong to the current interpreter may be dumped, + but this may also cause crashes or unintended behavior. + * Implement a heuristic to detect if the object memory has been freed. Don't + display the object contents in this case, only its memory address. + * The output format may change at any time. + + Example of output: + + .. code-block:: output + + object address : 0x7f80124702c0 + object refcount : 2 + object type : 0x9902e0 + object type name: str + object repr : 'abcdef' + + .. versionadded:: 3.15 + + +.. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name) Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This is equivalent to the Python expression ``hasattr(o, attr_name)``. @@ -111,7 +140,8 @@ Object Protocol .. note:: Exceptions that occur when this calls :meth:`~object.__getattr__` and - :meth:`~object.__getattribute__` methods are silently ignored. + :meth:`~object.__getattribute__` methods aren't propagated, + but instead given to :func:`sys.unraisablehook`. For proper error handling, use :c:func:`PyObject_HasAttrWithError`, :c:func:`PyObject_GetOptionalAttr` or :c:func:`PyObject_GetAttr` instead. @@ -196,6 +226,13 @@ Object Protocol in favour of using :c:func:`PyObject_DelAttr`, but there are currently no plans to remove it. + The function must not be called with a ``NULL`` *v* and an exception set. + This case can arise from forgetting ``NULL`` checks and would delete the + attribute. + + .. versionchanged:: 3.15 + Must not be called with NULL value if an exception is set. + .. c:function:: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v) @@ -206,6 +243,10 @@ Object Protocol If *v* is ``NULL``, the attribute is deleted, but this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. + The function must not be called with a ``NULL`` *v* and an exception set. + This case can arise from forgetting ``NULL`` checks and would delete the + attribute. + The number of different attribute names passed to this function should be kept small, usually by using a statically allocated string as *attr_name*. @@ -214,6 +255,10 @@ Object Protocol For more details, see :c:func:`PyUnicode_InternFromString`, which may be used internally to create a key object. + .. versionchanged:: 3.15 + Must not be called with NULL value if an exception is set. + + .. c:function:: int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) Generic attribute setter and deleter function that is meant @@ -492,6 +537,13 @@ Object Protocol on failure. This is equivalent to the Python statement ``del o[key]``. +.. c:function:: int PyObject_DelItemString(PyObject *o, const char *key) + + This is the same as :c:func:`PyObject_DelItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. c:function:: PyObject* PyObject_Dir(PyObject *o) This is equivalent to the Python expression ``dir(o)``, returning a (possibly @@ -509,6 +561,12 @@ Object Protocol iterated. +.. c:function:: PyObject* PyObject_SelfIter(PyObject *obj) + + This is equivalent to the Python ``__iter__(self): return self`` method. + It is intended for :term:`iterator` types, to be used in the :c:member:`PyTypeObject.tp_iter` slot. + + .. c:function:: PyObject* PyObject_GetAIter(PyObject *o) This is the equivalent to the Python expression ``aiter(o)``. Takes an @@ -571,7 +629,175 @@ Object Protocol Clear the managed dictionary of *obj*. - This function must only be called in a traverse function of the type which + This function must only be called in a clear function of the type which has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_Object_EnableDeferredRefcount(PyObject *obj) + + Enable `deferred reference counting `_ on *obj*, + if supported by the runtime. In the :term:`free-threaded ` build, + this allows the interpreter to avoid reference count adjustments to *obj*, + which may improve multi-threaded performance. The tradeoff is + that *obj* will only be deallocated by the tracing garbage collector, and + not when the interpreter no longer has any references to it. + + This function returns ``1`` if deferred reference counting is enabled on *obj*, + and ``0`` if deferred reference counting is not supported or if the hint was + ignored by the interpreter, such as when deferred reference counting is already + enabled on *obj*. This function is thread-safe, and cannot fail. + + This function does nothing on builds with the :term:`GIL` enabled, which do + not support deferred reference counting. This also does nothing if *obj* is not + an object tracked by the garbage collector (see :func:`gc.is_tracked` and + :c:func:`PyObject_GC_IsTracked`). + + This function is intended to be used soon after *obj* is created, + by the code that creates it, such as in the object's :c:member:`~PyTypeObject.tp_new` + slot. + + .. versionadded:: 3.14 + +.. c:function:: int PyUnstable_Object_IsUniqueReferencedTemporary(PyObject *obj) + + Check if *obj* is a unique temporary object. + Returns ``1`` if *obj* is known to be a unique temporary object, + and ``0`` otherwise. This function cannot fail, but the check is + conservative, and may return ``0`` in some cases even if *obj* is a unique + temporary object. + + If an object is a unique temporary, it is guaranteed that the current code + has the only reference to the object. For arguments to C functions, this + should be used instead of checking if the reference count is ``1``. Starting + with Python 3.14, the interpreter internally avoids some reference count + modifications when loading objects onto the operands stack by + :term:`borrowing ` references when possible, which means + that a reference count of ``1`` by itself does not guarantee that a function + argument uniquely referenced. + + In the example below, ``my_func`` is called with a unique temporary object + as its argument:: + + my_func([1, 2, 3]) + + In the example below, ``my_func`` is **not** called with a unique temporary + object as its argument, even if its refcount is ``1``:: + + my_list = [1, 2, 3] + my_func(my_list) + + See also the function :c:func:`Py_REFCNT`. + + .. versionadded:: 3.14 + +.. c:function:: int PyUnstable_IsImmortal(PyObject *obj) + + This function returns non-zero if *obj* is :term:`immortal`, and zero + otherwise. This function cannot fail. + + .. note:: + + Objects that are immortal in one CPython version are not guaranteed to + be immortal in another. + + .. versionadded:: 3.14 + +.. c:function:: int PyUnstable_TryIncRef(PyObject *obj) + + Increments the reference count of *obj* if it is not zero. Returns ``1`` + if the object's reference count was successfully incremented. Otherwise, + this function returns ``0``. + + :c:func:`PyUnstable_EnableTryIncRef` must have been called + earlier on *obj* or this function may spuriously return ``0`` in the + :term:`free threading` build. + + This function is logically equivalent to the following C code, except that + it behaves atomically in the :term:`free threading` build:: + + if (Py_REFCNT(op) > 0) { + Py_INCREF(op); + return 1; + } + return 0; + + This is intended as a building block for managing weak references + without the overhead of a Python :ref:`weak reference object `. + + Typically, correct use of this function requires support from *obj*'s + deallocator (:c:member:`~PyTypeObject.tp_dealloc`). + For example, the following sketch could be adapted to implement a + "weakmap" that works like a :py:class:`~weakref.WeakValueDictionary` + for a specific type: + + .. code-block:: c + + PyMutex mutex; + + PyObject * + add_entry(weakmap_key_type *key, PyObject *value) + { + PyUnstable_EnableTryIncRef(value); + weakmap_type weakmap = ...; + PyMutex_Lock(&mutex); + weakmap_add_entry(weakmap, key, value); + PyMutex_Unlock(&mutex); + Py_RETURN_NONE; + } + + PyObject * + get_value(weakmap_key_type *key) + { + weakmap_type weakmap = ...; + PyMutex_Lock(&mutex); + PyObject *result = weakmap_find(weakmap, key); + if (PyUnstable_TryIncRef(result)) { + // `result` is safe to use + PyMutex_Unlock(&mutex); + return result; + } + // if we get here, `result` is starting to be garbage-collected, + // but has not been removed from the weakmap yet + PyMutex_Unlock(&mutex); + return NULL; + } + + // tp_dealloc function for weakmap values + void + value_dealloc(PyObject *value) + { + weakmap_type weakmap = ...; + PyMutex_Lock(&mutex); + weakmap_remove_value(weakmap, value); + + ... + PyMutex_Unlock(&mutex); + } + + .. versionadded:: 3.14 + +.. c:function:: void PyUnstable_EnableTryIncRef(PyObject *obj) + + Enables subsequent uses of :c:func:`PyUnstable_TryIncRef` on *obj*. The + caller must hold a :term:`strong reference` to *obj* when calling this. + + .. versionadded:: 3.14 + +.. c:function:: int PyUnstable_Object_IsUniquelyReferenced(PyObject *op) + + Determine if *op* only has one reference. + + On GIL-enabled builds, this function is equivalent to + :c:expr:`Py_REFCNT(op) == 1`. + + On a :term:`free threaded ` build, this checks if *op*'s + :term:`reference count` is equal to one and additionally checks if *op* + is only used by this thread. :c:expr:`Py_REFCNT(op) == 1` is **not** + thread-safe on free threaded builds; prefer this function. + + The caller must hold an :term:`attached thread state`, despite the fact + that this function doesn't call into the Python interpreter. This function + cannot fail. + + .. versionadded:: 3.14 diff --git a/Doc/c-api/objimpl.rst b/Doc/c-api/objimpl.rst index 8bd8c107c98bdf..83de4248039949 100644 --- a/Doc/c-api/objimpl.rst +++ b/Doc/c-api/objimpl.rst @@ -12,6 +12,7 @@ object types. .. toctree:: allocation.rst + lifecycle.rst structures.rst typeobj.rst gcsupport.rst diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index 3d44d2eb6bf41d..76a1e9f528dc70 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -5,18 +5,19 @@ Support for Perf Maps ---------------------- -On supported platforms (as of this writing, only Linux), the runtime can take +On supported platforms (Linux and macOS), the runtime can take advantage of *perf map files* to make Python functions visible to an external -profiling tool (such as `perf `_). -A running process may create a file in the ``/tmp`` directory, which contains entries -that can map a section of executable code to a name. This interface is described in the +profiling tool (such as `perf `_ or +`samply `_). A running process may create a +file in the ``/tmp`` directory, which contains entries that can map a section +of executable code to a name. This interface is described in the `documentation of the Linux Perf tool `_. In Python, these helper APIs can be used by libraries and features that rely on generating machine code on the fly. -Note that holding the Global Interpreter Lock (GIL) is not required for these APIs. +Note that holding an :term:`attached thread state` is not required for these APIs. .. c:function:: int PyUnstable_PerfMapState_Init(void) diff --git a/Doc/c-api/picklebuffer.rst b/Doc/c-api/picklebuffer.rst new file mode 100644 index 00000000000000..9e2d92341b0f93 --- /dev/null +++ b/Doc/c-api/picklebuffer.rst @@ -0,0 +1,59 @@ +.. highlight:: c + +.. _picklebuffer-objects: + +.. index:: + pair: object; PickleBuffer + +Pickle buffer objects +--------------------- + +.. versionadded:: 3.8 + +A :class:`pickle.PickleBuffer` object wraps a :ref:`buffer-providing object +` for out-of-band data transfer with the :mod:`pickle` module. + + +.. c:var:: PyTypeObject PyPickleBuffer_Type + + This instance of :c:type:`PyTypeObject` represents the Python pickle buffer type. + This is the same object as :class:`pickle.PickleBuffer` in the Python layer. + + +.. c:function:: int PyPickleBuffer_Check(PyObject *op) + + Return true if *op* is a pickle buffer instance. + This function always succeeds. + + +.. c:function:: PyObject *PyPickleBuffer_FromObject(PyObject *obj) + + Create a pickle buffer from the object *obj*. + + This function will fail if *obj* doesn't support the :ref:`buffer protocol `. + + On success, return a new pickle buffer instance. + On failure, set an exception and return ``NULL``. + + Analogous to calling :class:`pickle.PickleBuffer` with *obj* in Python. + + +.. c:function:: const Py_buffer *PyPickleBuffer_GetBuffer(PyObject *picklebuf) + + Get a pointer to the underlying :c:type:`Py_buffer` that the pickle buffer wraps. + + The returned pointer is valid as long as *picklebuf* is alive and has not been + released. The caller must not modify or free the returned :c:type:`Py_buffer`. + If the pickle buffer has been released, raise :exc:`ValueError`. + + On success, return a pointer to the buffer view. + On failure, set an exception and return ``NULL``. + + +.. c:function:: int PyPickleBuffer_Release(PyObject *picklebuf) + + Release the underlying buffer held by the pickle buffer. + + Return ``0`` on success. On failure, set an exception and return ``-1``. + + Analogous to calling :meth:`pickle.PickleBuffer.release` in Python. diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index d75dad737bc992..57a0728d4e9af4 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -23,6 +23,15 @@ of Python objects. Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. + .. note:: + + On :term:`free threaded ` builds of Python, returning 1 + isn't sufficient to determine if it's safe to treat *o* as having no + access by other threads. Use :c:func:`PyUnstable_Object_IsUniquelyReferenced` + for that instead. + + See also the function :c:func:`PyUnstable_Object_IsUniqueReferencedTemporary()`. + .. versionchanged:: 3.10 :c:func:`Py_REFCNT()` is changed to the inline static function. @@ -201,7 +210,7 @@ of Python objects. Py_SETREF(dst, src); - That arranges to set *dst* to *src* _before_ releasing the reference + That arranges to set *dst* to *src* *before* releasing the reference to the old value of *dst*, so that any code triggered as a side-effect of *dst* getting torn down no longer believes *dst* points to a valid object. diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 038e6977104560..54fd5a064aa2ac 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -55,7 +55,7 @@ Reflection .. c:function:: PyFrameObject* PyEval_GetFrame(void) - Return the current thread state's frame, which is ``NULL`` if no frame is + Return the :term:`attached thread state`'s frame, which is ``NULL`` if no frame is currently executing. See also :c:func:`PyThreadState_GetFrame`. diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index ce28839f5ba739..df5bf6b64a93a0 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -105,6 +105,15 @@ Sequence Protocol equivalent to the Python expression ``value in o``. +.. c:function:: int PySequence_In(PyObject *o, PyObject *value) + + Alias for :c:func:`PySequence_Contains`. + + .. deprecated:: 3.14 + The function is :term:`soft deprecated` and should no longer be used to + write new code. + + .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) Return the first index *i* for which ``o[i] == value``. On error, return diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index cba823aa027bd6..b74859dd669c54 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -147,7 +147,7 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. @@ -166,3 +166,20 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Empty an existing set of all elements. Return ``0`` on success. Return ``-1`` and raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + + +Deprecated API +^^^^^^^^^^^^^^ + +.. c:macro:: PySet_MINSIZE + + A :term:`soft deprecated` constant representing the size of an internal + preallocated table inside :c:type:`PySetObject` instances. + + This is documented solely for completeness, as there are no guarantees + that a given version of CPython uses preallocated tables with a fixed + size. + In code that does not deal with unstable set internals, + :c:macro:`!PySet_MINSIZE` can be replaced with a small constant like ``8``. + + If looking for the size of a set, use :c:func:`PySet_Size` instead. diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 8adf6a961378a3..c6d761fe7fd1c9 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -118,6 +118,12 @@ Ellipsis Object ^^^^^^^^^^^^^^^ +.. c:var:: PyTypeObject PyEllipsis_Type + + The type of Python :const:`Ellipsis` object. Same as :class:`types.EllipsisType` + in the Python layer. + + .. c:var:: PyObject *Py_Ellipsis The Python ``Ellipsis`` object. This object has no methods. Like diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 5b9e43874c7f2b..f5e6b7ad157e99 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -2,9 +2,9 @@ .. _stable: -*************** -C API Stability -*************** +*********************** +C API and ABI Stability +*********************** Unless documented otherwise, Python's C API is covered by the Backwards Compatibility Policy, :pep:`387`. @@ -51,6 +51,7 @@ It is generally intended for specialized, low-level tools like debuggers. Projects that use this API are expected to follow CPython development and spend extra effort adjusting to changes. +.. _stable-application-binary-interface: Stable Application Binary Interface =================================== @@ -66,7 +67,7 @@ Limited C API Python 3.2 introduced the *Limited API*, a subset of Python's C API. Extensions that only use the Limited API can be -compiled once and work with multiple versions of Python. +compiled once and be loaded on multiple versions of Python. Contents of the Limited API are :ref:`listed below `. .. c:macro:: Py_LIMITED_API @@ -76,7 +77,7 @@ Contents of the Limited API are :ref:`listed below `. Define ``Py_LIMITED_API`` to the value of :c:macro:`PY_VERSION_HEX` corresponding to the lowest Python version your extension supports. - The extension will work without recompilation with all Python 3 releases + The extension will be ABI-compatible with all Python 3 releases from the specified one onward, and can use Limited API introduced up to that version. @@ -94,7 +95,15 @@ Stable ABI ---------- To enable this, Python provides a *Stable ABI*: a set of symbols that will -remain compatible across Python 3.x versions. +remain ABI-compatible across Python 3.x versions. + +.. note:: + + The Stable ABI prevents ABI issues, like linker errors due to missing + symbols or data corruption due to changes in structure layouts or function + signatures. + However, other changes in Python can change the *behavior* of extensions. + See Python's Backwards Compatibility Policy (:pep:`387`) for details. The Stable ABI contains symbols exposed in the :ref:`Limited API `, but also other ones – for example, functions necessary to @@ -190,6 +199,162 @@ This is the case with Windows and macOS releases from ``python.org`` and many third-party distributors. +ABI Checking +============ + +.. versionadded:: 3.15 + +Python includes a rudimentary check for ABI compatibility. + +This check is not comprehensive. +It only guards against common cases of incompatible modules being +installed for the wrong interpreter. +It also does not take :ref:`platform incompatibilities ` +into account. +It can only be done after an extension is successfully loaded. + +Despite these limitations, it is recommended that extension modules use this +mechanism, so that detectable incompatibilities raise exceptions rather than +crash. + +Most modules can use this check via the :c:data:`Py_mod_abi` +slot and the :c:macro:`PyABIInfo_VAR` macro, for example like this: + +.. code-block:: c + + PyABIInfo_VAR(abi_info); + + static PyModuleDef_Slot mymodule_slots[] = { + {Py_mod_abi, &abi_info}, + ... + }; + + +The full API is described below for advanced use cases. + +.. c:function:: int PyABIInfo_Check(PyABIInfo *info, const char *module_name) + + Verify that the given *info* is compatible with the currently running + interpreter. + + Return 0 on success. On failure, raise an exception and return -1. + + If the ABI is incompatible, the raised exception will be :py:exc:`ImportError`. + + The *module_name* argument can be ``NULL``, or point to a NUL-terminated + UTF-8-encoded string used for error messages. + + Note that if *info* describes the ABI that the current code uses (as defined + by :c:macro:`PyABIInfo_VAR`, for example), using any other Python C API + may lead to crashes. + In particular, it is not safe to examine the raised exception. + + .. versionadded:: 3.15 + +.. c:macro:: PyABIInfo_VAR(NAME) + + Define a static :c:struct:`PyABIInfo` variable with the given *NAME* that + describes the ABI that the current code will use. + This macro expands to: + + .. code-block:: c + + static PyABIInfo NAME = { + 1, 0, + PyABIInfo_DEFAULT_FLAGS, + PY_VERSION_HEX, + PyABIInfo_DEFAULT_ABI_VERSION + } + + .. versionadded:: 3.15 + +.. c:type:: PyABIInfo + + .. c:member:: uint8_t abiinfo_major_version + + The major version of :c:struct:`PyABIInfo`. Can be set to: + + * ``0`` to skip all checking, or + * ``1`` to specify this version of :c:struct:`!PyABIInfo`. + + .. c:member:: uint8_t abiinfo_minor_version + + The minor version of :c:struct:`PyABIInfo`. + Must be set to ``0``; larger values are reserved for backwards-compatible + future versions of :c:struct:`!PyABIInfo`. + + .. c:member:: uint16_t flags + + .. c:namespace:: NULL + + This field is usually set to the following macro: + + .. c:macro:: PyABIInfo_DEFAULT_FLAGS + + Default flags, based on current values of macros such as + :c:macro:`Py_LIMITED_API` and :c:macro:`Py_GIL_DISABLED`. + + Alternately, the field can be set to the following flags, combined + by bitwise OR. + Unused bits must be set to zero. + + ABI variant -- one of: + + .. c:macro:: PyABIInfo_STABLE + + Specifies that the stable ABI is used. + + .. c:macro:: PyABIInfo_INTERNAL + + Specifies ABI specific to a particular build of CPython. + Internal use only. + + Free-threading compatibility -- one of: + + .. c:macro:: PyABIInfo_FREETHREADED + + Specifies ABI compatible with free-threading builds of CPython. + (That is, ones compiled with :option:`--disable-gil`; with ``t`` + in :py:data:`sys.abiflags`) + + .. c:macro:: PyABIInfo_GIL + + Specifies ABI compatible with non-free-threading builds of CPython + (ones compiled *without* :option:`--disable-gil`). + + .. c:member:: uint32_t build_version + + The version of the Python headers used to build the code, in the format + used by :c:macro:`PY_VERSION_HEX`. + + This can be set to ``0`` to skip any checks related to this field. + This option is meant mainly for projects that do not use the CPython + headers directly, and do not emulate a specific version of them. + + .. c:member:: uint32_t abi_version + + The ABI version. + + For the Stable ABI, this field should be the value of + :c:macro:`Py_LIMITED_API` + (except if :c:macro:`Py_LIMITED_API` is ``3``; use + :c:expr:`Py_PACK_VERSION(3, 2)` in that case). + + Otherwise, it should be set to :c:macro:`PY_VERSION_HEX`. + + It can also be set to ``0`` to skip any checks related to this field. + + .. c:namespace:: NULL + + .. c:macro:: PyABIInfo_DEFAULT_ABI_VERSION + + The value that should be used for this field, based on current + values of macros such as :c:macro:`Py_LIMITED_API`, + :c:macro:`PY_VERSION_HEX` and :c:macro:`Py_GIL_DISABLED`. + + .. versionadded:: 3.15 + + .. _limited-api-list: Contents of Limited API diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index d333df397782e0..62f45def04f746 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -28,18 +28,52 @@ under :ref:`reference counting `. object. In a normal "release" build, it contains only the object's reference count and a pointer to the corresponding type object. Nothing is actually declared to be a :c:type:`PyObject`, but every pointer - to a Python object can be cast to a :c:expr:`PyObject*`. Access to the - members must be done by using the macros :c:macro:`Py_REFCNT` and - :c:macro:`Py_TYPE`. + to a Python object can be cast to a :c:expr:`PyObject*`. + + The members must not be accessed directly; instead use macros such as + :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + + .. c:member:: Py_ssize_t ob_refcnt + + The object's reference count, as returned by :c:macro:`Py_REFCNT`. + Do not use this field directly; instead use functions and macros such as + :c:macro:`!Py_REFCNT`, :c:func:`Py_INCREF` and :c:func:`Py_DecRef`. + + The field type may be different from ``Py_ssize_t``, depending on + build configuration and platform. + + .. c:member:: PyTypeObject* ob_type + + The object's type. + Do not use this field directly; use :c:macro:`Py_TYPE` and + :c:func:`Py_SET_TYPE` instead. .. c:type:: PyVarObject - This is an extension of :c:type:`PyObject` that adds the :c:member:`~PyVarObject.ob_size` - field. This is only used for objects that have some notion of *length*. - This type does not often appear in the Python/C API. - Access to the members must be done by using the macros - :c:macro:`Py_REFCNT`, :c:macro:`Py_TYPE`, and :c:macro:`Py_SIZE`. + An extension of :c:type:`PyObject` that adds the + :c:member:`~PyVarObject.ob_size` field. + This is intended for objects that have some notion of *length*. + + As with :c:type:`!PyObject`, the members must not be accessed directly; + instead use macros such as :c:macro:`Py_SIZE`, :c:macro:`Py_REFCNT` and + :c:macro:`Py_TYPE`. + + .. c:member:: Py_ssize_t ob_size + + A size field, whose contents should be considered an object's internal + implementation detail. + + Do not use this field directly; use :c:macro:`Py_SIZE` instead. + + Object creation functions such as :c:func:`PyObject_NewVar` will + generally set this field to the requested size (number of items). + After creation, arbitrary values can be stored in :c:member:`!ob_size` + using :c:macro:`Py_SET_SIZE`. + + To get an object's publicly exposed length, as returned by + the Python function :py:func:`len`, use :c:func:`PyObject_Length` + instead. .. c:macro:: PyObject_HEAD @@ -63,6 +97,11 @@ under :ref:`reference counting `. See documentation of :c:type:`PyVarObject` above. +.. c:var:: PyTypeObject PyBaseObject_Type + + The base class of all other objects, the same as :class:`object` in Python. + + .. c:function:: int Py_Is(PyObject *x, PyObject *y) Test if the *x* object is the *y* object, the same as ``x is y`` in Python. @@ -98,9 +137,8 @@ under :ref:`reference counting `. Get the type of the Python object *o*. - Return a :term:`borrowed reference`. - - Use the :c:func:`Py_SET_TYPE` function to set an object type. + The returned reference is :term:`borrowed ` from *o*. + Do not release it with :c:func:`Py_DECREF` or similar. .. versionchanged:: 3.11 :c:func:`Py_TYPE()` is changed to an inline static function. @@ -117,16 +155,26 @@ under :ref:`reference counting `. .. c:function:: void Py_SET_TYPE(PyObject *o, PyTypeObject *type) - Set the object *o* type to *type*. + Set the type of object *o* to *type*, without any checking or reference + counting. + + This is a very low-level operation. + Consider instead setting the Python attribute :attr:`~object.__class__` + using :c:func:`PyObject_SetAttrString` or similar. + + Note that assigning an incompatible type can lead to undefined behavior. + + If *type* is a :ref:`heap type `, the caller must create a + new reference to it. + Similarly, if the old type of *o* is a heap type, the caller must release + a reference to that type. .. versionadded:: 3.9 .. c:function:: Py_ssize_t Py_SIZE(PyVarObject *o) - Get the size of the Python object *o*. - - Use the :c:func:`Py_SET_SIZE` function to set an object size. + Get the :c:member:`~PyVarObject.ob_size` field of *o*. .. versionchanged:: 3.11 :c:func:`Py_SIZE()` is changed to an inline static function. @@ -135,7 +183,7 @@ under :ref:`reference counting `. .. c:function:: void Py_SET_SIZE(PyVarObject *o, Py_ssize_t size) - Set the object *o* size to *size*. + Set the :c:member:`~PyVarObject.ob_size` field of *o* to *size*. .. versionadded:: 3.9 @@ -232,6 +280,8 @@ Implementing functions and methods Name of the method. + A ``NULL`` *ml_name* marks the end of a :c:type:`!PyMethodDef` array. + .. c:member:: PyCFunction ml_meth Pointer to the C implementation. @@ -399,6 +449,25 @@ definition with the same method name. slot. This is helpful because calls to PyCFunctions are optimized more than wrapper object calls. + +.. c:var:: PyTypeObject PyCMethod_Type + + The type object corresponding to Python C method objects. This is + available as :class:`types.BuiltinMethodType` in the Python layer. + + +.. c:function:: int PyCMethod_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCMethod_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCMethod_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCMethod_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *cls) Turn *ml* into a Python :term:`callable` object. @@ -424,6 +493,24 @@ definition with the same method name. .. versionadded:: 3.9 +.. c:var:: PyTypeObject PyCFunction_Type + + The type object corresponding to Python C function objects. This is + available as :class:`types.BuiltinFunctionType` in the Python layer. + + +.. c:function:: int PyCFunction_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCFunction_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCFunction_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCFunction_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) Equivalent to ``PyCMethod_New(ml, self, module, NULL)``. @@ -434,6 +521,62 @@ definition with the same method name. Equivalent to ``PyCMethod_New(ml, self, NULL, NULL)``. +.. c:function:: int PyCFunction_GetFlags(PyObject *func) + + Get the function's flags on *func* as they were passed to + :c:member:`~PyMethodDef.ml_flags`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function's flags on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FLAGS(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFlags`, but without error + or type checking. + + +.. c:function:: PyCFunction PyCFunction_GetFunction(PyObject *func) + + Get the function pointer on *func* as it was passed to + :c:member:`~PyMethodDef.ml_meth`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function pointer on success, and ``NULL`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FUNCTION(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFunction`, but without error + or type checking. + + +.. c:function:: PyObject *PyCFunction_GetSelf(PyObject *func) + + Get the "self" object on *func*. This is the object that would be passed + to the first argument of a :c:type:`PyCFunction`. For C function objects + created through a :c:type:`PyMethodDef` on a :c:type:`PyModuleDef`, this + is the resulting module object. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns a :term:`borrowed reference` to the "self" object + on success, and ``NULL`` with an exception set on failure. + + +.. c:function:: PyObject *PyCFunction_GET_SELF(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetSelf`, but without error or + type checking. + + Accessing attributes of extension types --------------------------------------- @@ -557,14 +700,12 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: entry indicates an offset from the subclass-specific data, rather than from ``PyObject``. - Can only be used as part of :c:member:`Py_tp_members ` + Can only be used as part of the :c:data:`Py_tp_members` :c:type:`slot ` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - - This flag is only used in :c:type:`PyType_Slot`. - When setting :c:member:`~PyTypeObject.tp_members` during - class creation, Python clears it and sets + When setting :c:member:`~PyTypeObject.tp_members` from the slot during + class creation, Python clears the flag and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index d6fca1a0b0a219..ee73c1c8adaa7b 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -123,6 +123,24 @@ Operating System Utilities This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! + +.. c:function:: int PyOS_InterruptOccurred(void) + + Check if a :c:macro:`!SIGINT` signal has been received. + + Returns ``1`` if a :c:macro:`!SIGINT` has occurred and clears the signal flag, + or ``0`` otherwise. + + In most cases, you should prefer :c:func:`PyErr_CheckSignals` over this function. + :c:func:`!PyErr_CheckSignals` invokes the appropriate signal handlers + for all pending signals, allowing Python code to handle the signal properly. + This function only detects :c:macro:`!SIGINT` and does not invoke any Python + signal handlers. + + This function is async-signal-safe and this function cannot fail. + The caller must hold an :term:`attached thread state`. + + .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) .. warning:: @@ -216,6 +234,38 @@ Operating System Utilities The function now uses the UTF-8 encoding on Windows if :c:member:`PyPreConfig.legacy_windows_fs_encoding` is zero. +.. c:function:: FILE* Py_fopen(PyObject *path, const char *mode) + + Similar to :c:func:`!fopen`, but *path* is a Python object and + an exception is set on error. + + *path* must be a :class:`str` object, a :class:`bytes` object, + or a :term:`path-like object`. + + On success, return the new file pointer. + On error, set an exception and return ``NULL``. + + The file must be closed by :c:func:`Py_fclose` rather than calling directly + :c:func:`!fclose`. + + The file descriptor is created non-inheritable (:pep:`446`). + + The caller must have an :term:`attached thread state`. + + .. versionadded:: 3.14 + + +.. c:function:: int Py_fclose(FILE *file) + + Close a file that was opened by :c:func:`Py_fopen`. + + On success, return ``0``. + On error, return ``EOF`` and ``errno`` is set to indicate the error. + In either case, any further access (including another call to + :c:func:`Py_fclose`) to the stream results in undefined behavior. + + .. versionadded:: 3.14 + .. _systemfunctions: @@ -226,10 +276,57 @@ These are utility functions that make functionality from the :mod:`sys` module accessible to C code. They all work with the current interpreter thread's :mod:`sys` module's dict, which is contained in the internal thread state structure. +.. c:function:: PyObject *PySys_GetAttr(PyObject *name) + + Get the attribute *name* of the :mod:`sys` module. + Return a :term:`strong reference`. + Raise :exc:`RuntimeError` and return ``NULL`` if it does not exist or + if the :mod:`sys` module cannot be found. + + If the non-existing object should not be treated as a failure, you can use + :c:func:`PySys_GetOptionalAttr` instead. + + .. versionadded:: 3.15 + +.. c:function:: PyObject *PySys_GetAttrString(const char *name) + + This is the same as :c:func:`PySys_GetAttr`, but *name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + If the non-existing object should not be treated as a failure, you can use + :c:func:`PySys_GetOptionalAttrString` instead. + + .. versionadded:: 3.15 + +.. c:function:: int PySys_GetOptionalAttr(PyObject *name, PyObject **result) + + Variant of :c:func:`PySys_GetAttr` which doesn't raise + exception if the object does not exist. + + * Set *\*result* to a new :term:`strong reference` to the object and + return ``1`` if the object exists. + * Set *\*result* to ``NULL`` and return ``0`` without setting an exception + if the object does not exist. + * Set an exception, set *\*result* to ``NULL``, and return ``-1``, + if an error occurred. + + .. versionadded:: 3.15 + +.. c:function:: int PySys_GetOptionalAttrString(const char *name, PyObject **result) + + This is the same as :c:func:`PySys_GetOptionalAttr`, but *name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. versionadded:: 3.15 + .. c:function:: PyObject *PySys_GetObject(const char *name) - Return the object *name* from the :mod:`sys` module or ``NULL`` if it does - not exist, without setting an exception. + Similar to :c:func:`PySys_GetAttrString`, but return a :term:`borrowed + reference` and return ``NULL`` *without* setting exception on failure. + + Preserves exception that was set before the call. .. c:function:: int PySys_SetObject(const char *name, PyObject *v) @@ -237,14 +334,6 @@ accessible to C code. They all work with the current interpreter thread's case *name* is deleted from the sys module. Returns ``0`` on success, ``-1`` on error. -.. c:function:: void PySys_ResetWarnOptions() - - Reset :data:`sys.warnoptions` to an empty list. This function may be - called prior to :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - .. c:function:: void PySys_WriteStdout(const char *format, ...) Write the output string described by *format* to :data:`sys.stdout`. No @@ -346,8 +435,8 @@ accessible to C code. They all work with the current interpreter thread's silently abort the operation by raising an error subclassed from :class:`Exception` (other errors will not be silenced). - The hook function is always called with the GIL held by the Python - interpreter that raised the event. + The hook function is always called with an :term:`attached thread state` by + the Python interpreter that raised the event. See :pep:`578` for a detailed description of auditing. Functions in the runtime and standard library that raise events are listed in the @@ -426,3 +515,7 @@ Process Control function registered last is called first. Each cleanup function will be called at most once. Since Python's internal finalization will have completed before the cleanup function, no Python APIs should be called by *func*. + + .. seealso:: + + :c:func:`PyUnstable_AtExit` for passing a ``void *data`` argument. diff --git a/Doc/c-api/time.rst b/Doc/c-api/time.rst index 7032cc48aa6913..31d6adbc225439 100644 --- a/Doc/c-api/time.rst +++ b/Doc/c-api/time.rst @@ -56,7 +56,7 @@ range. system time.) As any other C API (unless otherwise specified), the functions must be called -with the :term:`GIL` held. +with an :term:`attached thread state`. .. c:function:: int PyTime_Monotonic(PyTime_t *result) @@ -78,29 +78,29 @@ Raw Clock Functions ------------------- Similar to clock functions, but don't set an exception on error and don't -require the caller to hold the GIL. +require the caller to have an :term:`attached thread state`. On success, the functions return ``0``. On failure, they set ``*result`` to ``0`` and return ``-1``, *without* setting -an exception. To get the cause of the error, acquire the GIL and call the -regular (non-``Raw``) function. Note that the regular function may succeed after +an exception. To get the cause of the error, :term:`attach ` a :term:`thread state`, +and call the regular (non-``Raw``) function. Note that the regular function may succeed after the ``Raw`` one failed. .. c:function:: int PyTime_MonotonicRaw(PyTime_t *result) Similar to :c:func:`PyTime_Monotonic`, - but don't set an exception on error and don't require holding the GIL. + but don't set an exception on error and don't require an :term:`attached thread state`. .. c:function:: int PyTime_PerfCounterRaw(PyTime_t *result) Similar to :c:func:`PyTime_PerfCounter`, - but don't set an exception on error and don't require holding the GIL. + but don't set an exception on error and don't require an :term:`attached thread state`. .. c:function:: int PyTime_TimeRaw(PyTime_t *result) Similar to :c:func:`PyTime_Time`, - but don't set an exception on error and don't require holding the GIL. + but don't set an exception on error and don't require an :term:`attached thread state`. Conversion functions diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 815afddad19df1..3e3752696c46d8 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -37,6 +37,19 @@ Tuple Objects or ``NULL`` with an exception set on failure. +.. c:function:: PyObject* PyTuple_FromArray(PyObject *const *array, Py_ssize_t size) + + Create a tuple of *size* items and copy references from *array* to the new + tuple. + + *array* can be NULL if *size* is ``0``. + + On success, return a new reference. + On error, set an exception and return ``NULL``. + + .. versionadded:: 3.15 + + .. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) Return a new tuple object of size *n*, @@ -48,7 +61,7 @@ Tuple Objects .. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) Take a pointer to a tuple object, and return the size of that tuple. - On error, return ``-1`` and with an exception set. + On error, return ``-1`` with an exception set. .. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) @@ -135,8 +148,11 @@ Tuple Objects Struct Sequence Objects ----------------------- -Struct sequence objects are the C equivalent of :func:`~collections.namedtuple` -objects, i.e. a sequence whose items can also be accessed through attributes. +A struct sequence object is a :term:`named tuple`, that is, a sequence +whose items can also be accessed through attributes. +It is similar to :func:`collections.namedtuple`, but provides a slightly +different interface. + To create a struct sequence, you first have to create a specific struct sequence type. diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 86d3967d9fb577..8cadf26cee3027 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -82,6 +82,9 @@ Type Objects error (e.g. no more watcher IDs available), return ``-1`` and set an exception. + In free-threaded builds, :c:func:`PyType_AddWatcher` is not thread-safe, + so it must be called at start up (before spawning the first thread). + .. versionadded:: 3.12 @@ -113,6 +116,20 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_Unwatch(int watcher_id, PyObject *type) + + Mark *type* as not watched. This undoes a previous call to + :c:func:`PyType_Watch`. *type* must not be ``NULL``. + + An extension should never call this function with a *watcher_id* that was + not returned to it by a previous call to :c:func:`PyType_AddWatcher`. + + On success, this function returns ``0``. On failure, this function returns + ``-1`` with an exception set. + + .. versionadded:: 3.12 + + .. c:type:: int (*PyType_WatchCallback)(PyObject *type) Type of a type-watcher callback function. @@ -130,6 +147,18 @@ Type Objects Type features are denoted by single bit flags. +.. c:function:: int PyType_FastSubclass(PyTypeObject *type, int flag) + + Return non-zero if the type object *type* sets the subclass flag *flag*. + Subclass flags are denoted by + :c:macro:`Py_TPFLAGS_*_SUBCLASS `. + This function is used by many ``_Check`` functions for common types. + + .. seealso:: + :c:func:`PyObject_TypeCheck`, which is used as a slower alternative in + ``_Check`` functions for types that don't come with subclass flags. + + .. c:function:: int PyType_IS_GC(PyTypeObject *o) Return true if the type object includes support for the cycle detector; this @@ -148,14 +177,31 @@ Type Objects .. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems) - Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type object. Use - Python's default memory allocation mechanism to allocate a new instance and - initialize all its contents to ``NULL``. + Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type + object. Uses Python's default memory allocation mechanism to allocate memory + for a new instance, zeros the memory, then initializes the memory as if by + calling :c:func:`PyObject_Init` or :c:func:`PyObject_InitVar`. + + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + For types that support garbage collection (i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set), this function behaves like + :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar` (except the + memory is guaranteed to be zeroed before initialization), and should be + paired with :c:func:`PyObject_GC_Del` in :c:member:`~PyTypeObject.tp_free`. + Otherwise, it behaves like :c:macro:`PyObject_New` or + :c:macro:`PyObject_NewVar` (except the memory is guaranteed to be zeroed + before initialization) and should be paired with :c:func:`PyObject_Free` in + :c:member:`~PyTypeObject.tp_free`. + .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) - Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Create a - new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot. + Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type + object. Creates a new instance using the type's + :c:member:`~PyTypeObject.tp_alloc` slot and returns the resulting object. + .. c:function:: int PyType_Ready(PyTypeObject *type) @@ -173,6 +219,7 @@ Type Objects GC protocol itself by at least implementing the :c:member:`~PyTypeObject.tp_traverse` handle. + .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) Return the type's name. Equivalent to getting the type's @@ -180,6 +227,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the @@ -195,6 +243,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) Return the type's module name. Equivalent to getting the @@ -202,6 +251,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -218,6 +268,7 @@ Type Objects :c:func:`PyType_GetSlot` can now accept all types. Previously, it was limited to :ref:`heap types `. + .. c:function:: PyObject* PyType_GetModule(PyTypeObject *type) Return the module object associated with the given type when the type was @@ -232,11 +283,12 @@ Type Objects ``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses are not necessarily defined in the same module as their superclass. See :c:type:`PyCMethod` to get the class that defines the method. - See :c:func:`PyType_GetModuleByDef` for cases when :c:type:`!PyCMethod` cannot - be used. + See :c:func:`PyType_GetModuleByToken` for cases when :c:type:`!PyCMethod` + cannot be used. .. versionadded:: 3.9 + .. c:function:: void* PyType_GetModuleState(PyTypeObject *type) Return the state of the module object associated with the given type. @@ -251,10 +303,11 @@ Type Objects .. versionadded:: 3.9 -.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) - Find the first superclass whose module was created from - the given :c:type:`PyModuleDef` *def*, and return that module. +.. c:function:: PyObject* PyType_GetModuleByToken(PyTypeObject *type, const void *mod_token) + + Find the first superclass whose module has the given + :ref:`module token `, and return that module. If no module is found, raises a :py:class:`TypeError` and returns ``NULL``. @@ -264,12 +317,34 @@ Type Objects and other places where a method's defining class cannot be passed using the :c:type:`PyCMethod` calling convention. + .. versionadded:: 3.15 + + +.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) + + Find the first superclass whose module was created from the given + :c:type:`PyModuleDef` *def*, or whose :ref:`module token ` + is equal to *def*, and return that module. + + Note that modules created from a :c:type:`PyModuleDef` always have their + token set to the :c:type:`PyModuleDef`'s address. + In other words, this function is equivalent to + :c:func:`PyType_GetModuleByToken`, except that it: + + - returns a borrowed reference, and + - has a non-``void*`` argument type (which is a cosmetic difference in C). + + The returned reference is :term:`borrowed ` from *type*, + and will be valid as long as you hold a reference to *type*. + Do not release it with :c:func:`Py_DECREF` or similar. + .. versionadded:: 3.11 -.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) + +.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *tp_token, PyTypeObject **result) Find the first superclass in *type*'s :term:`method resolution order` whose - :c:macro:`Py_tp_token` token is equal to the given one. + :c:macro:`Py_tp_token` token is equal to *tp_token*. * If found, set *\*result* to a new :term:`strong reference` to it and return ``1``. @@ -280,10 +355,11 @@ Type Objects The *result* argument may be ``NULL``, in which case *\*result* is not set. Use this if you need only the return value. - The *token* argument may not be ``NULL``. + The *tp_token* argument may not be ``NULL``. .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -294,6 +370,16 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) + + Return true if instances of *type* support creating weak references, false + otherwise. This function always succeeds. *type* must not be ``NULL``. + + .. seealso:: + * :ref:`weakrefobjects` + * :py:mod:`weakref` + + Creating Heap-Allocated Types ............................. @@ -311,15 +397,11 @@ The following functions and structs are used to create Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not supported, except if ``tp_new`` is ``NULL``. - (For backwards compatibility, other ``PyType_From*`` functions allow - such metaclasses. They ignore ``tp_new``, which may result in incomplete - initialization. This is deprecated and in Python 3.14+ such metaclasses will - not be supported.) The *bases* argument can be used to specify base classes; it can either be only one class or a tuple of classes. - If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. - If that also is ``NULL``, the *Py_tp_base* slot is used instead. + If *bases* is ``NULL``, the :c:data:`Py_tp_bases` slot is used instead. + If that also is ``NULL``, the :c:data:`Py_tp_base` slot is used instead. If that also is ``NULL``, the new type derives from :class:`object`. The *module* argument can be used to record the module in which the new @@ -346,6 +428,7 @@ The following functions and structs are used to create .. versionadded:: 3.12 + .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, module, spec, bases)``. @@ -372,6 +455,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, bases)``. @@ -393,6 +477,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``. @@ -413,6 +498,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: int PyType_Freeze(PyTypeObject *type) Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. @@ -456,6 +542,9 @@ The following functions and structs are used to create class need *in addition* to the superclass. Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific memory reserved this way. + For negative :c:member:`!basicsize`, Python will insert padding when + needed to meet :c:member:`~PyTypeObject.tp_basicsize`'s alignment + requirements. .. versionchanged:: 3.12 @@ -518,9 +607,9 @@ The following functions and structs are used to create :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. For example, use: - * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` - * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` - * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + * :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` + * :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` + * :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` An additional slot is supported that does not correspond to a :c:type:`!PyTypeObject` struct field: @@ -529,19 +618,19 @@ The following functions and structs are used to create The following “offset” fields cannot be set using :c:type:`PyType_Slot`: - * :c:member:`~PyTypeObject.tp_weaklistoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) - * :c:member:`~PyTypeObject.tp_dictoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) - * :c:member:`~PyTypeObject.tp_vectorcall_offset` - (use ``"__vectorcalloffset__"`` in - :ref:`PyMemberDef `) - - If it is not possible to switch to a ``MANAGED`` flag (for example, - for vectorcall or to support Python older than 3.12), specify the - offset in :c:member:`Py_tp_members `. - See :ref:`PyMemberDef documentation ` - for details. + * :c:member:`~PyTypeObject.tp_weaklistoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) + * :c:member:`~PyTypeObject.tp_dictoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) + * :c:member:`~PyTypeObject.tp_vectorcall_offset` + (use ``"__vectorcalloffset__"`` in + :ref:`PyMemberDef `) + + If it is not possible to switch to a ``MANAGED`` flag (for example, + for vectorcall or to support Python older than 3.12), specify the + offset in :c:data:`Py_tp_members`. + See :ref:`PyMemberDef documentation ` + for details. The following internal fields cannot be set at all when creating a heap type: @@ -557,20 +646,18 @@ The following functions and structs are used to create To avoid issues, use the *bases* argument of :c:func:`PyType_FromSpecWithBases` instead. - .. versionchanged:: 3.9 - - Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + .. versionchanged:: 3.9 + Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. - .. versionchanged:: 3.11 - :c:member:`~PyBufferProcs.bf_getbuffer` and - :c:member:`~PyBufferProcs.bf_releasebuffer` are now available - under the :ref:`limited API `. + .. versionchanged:: 3.11 + :c:member:`~PyBufferProcs.bf_getbuffer` and + :c:member:`~PyBufferProcs.bf_releasebuffer` are now available + under the :ref:`limited API `. - .. versionchanged:: 3.14 - - The field :c:member:`~PyTypeObject.tp_vectorcall` can now set - using ``Py_tp_vectorcall``. See the field's documentation - for details. + .. versionchanged:: 3.14 + The field :c:member:`~PyTypeObject.tp_vectorcall` can now be set + using :c:data:`Py_tp_vectorcall`. See the field's documentation + for details. .. c:member:: void *pfunc @@ -579,10 +666,11 @@ The following functions and structs are used to create *pfunc* values may not be ``NULL``, except for the following slots: - * ``Py_tp_doc`` + * :c:data:`Py_tp_doc` * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` rather than ``NULL``) + .. c:macro:: Py_tp_token A :c:member:`~PyType_Slot.slot` that records a static memory layout ID diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 0c59b3da0795cb..a33da367e6071f 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -2,8 +2,8 @@ .. _type-structs: -Type Objects -============ +Type Object Structures +====================== Perhaps one of the most important structures of the Python object system is the structure that defines a new type: the :c:type:`PyTypeObject` structure. Type @@ -79,7 +79,7 @@ Quick Reference | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | | | | __delattr__ | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | + | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | :ref:`sub-slots` | | | | % | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ @@ -325,9 +325,10 @@ sub-slots +---------------------------------------------------------+-----------------------------------+---------------+ | | +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | | + | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | __buffer__ | +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | | + | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | __release_\ | + | | | buffer\__ | +---------------------------------------------------------+-----------------------------------+---------------+ .. _slot-typedefs-table: @@ -355,7 +356,7 @@ slot typedefs +-----------------------------+-----------------------------+----------------------+ | :c:type:`newfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | -| | :c:type:`PyObject` * | | +| | :c:type:`PyTypeObject` * | | | | :c:type:`PyObject` * | | | | :c:type:`PyObject` * | | +-----------------------------+-----------------------------+----------------------+ @@ -473,7 +474,7 @@ PyTypeObject Definition ----------------------- The structure definition for :c:type:`PyTypeObject` can be found in -:file:`Include/object.h`. For convenience of reference, this repeats the +:file:`Include/cpython/object.h`. For convenience of reference, this repeats the definition found there: .. XXX Drop this? @@ -491,9 +492,9 @@ metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that it type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. -.. c:member:: Py_ssize_t PyObject.ob_refcnt +:c:member:`PyObject.ob_refcnt` - This is the type object's reference count, initialized to ``1`` by the + The type object's reference count is initialized to ``1`` by the ``PyObject_HEAD_INIT`` macro. Note that for :ref:`statically allocated type objects `, the type's instances (objects whose :c:member:`~PyObject.ob_type` points back to the type) do *not* count as references. But for @@ -505,7 +506,7 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. This field is not inherited by subtypes. -.. c:member:: PyTypeObject* PyObject.ob_type +:c:member:`PyObject.ob_type` This is the type's type, in other words its metatype. It is initialized by the argument to the ``PyObject_HEAD_INIT`` macro, and its value should normally be @@ -531,12 +532,14 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. PyVarObject Slots ----------------- -.. c:member:: Py_ssize_t PyVarObject.ob_size +:c:member:`PyVarObject.ob_size` For :ref:`statically allocated type objects `, this should be initialized to zero. For :ref:`dynamically allocated type objects `, this field has a special internal meaning. + This field should be accessed using the :c:func:`Py_SIZE()` macro. + **Inheritance:** This field is not inherited by subtypes. @@ -587,119 +590,230 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: Py_ssize_t PyTypeObject.tp_basicsize - Py_ssize_t PyTypeObject.tp_itemsize + Py_ssize_t PyTypeObject.tp_itemsize These fields allow calculating the size in bytes of instances of the type. There are two kinds of types: types with fixed-length instances have a zero - :c:member:`~PyTypeObject.tp_itemsize` field, types with variable-length instances have a non-zero - :c:member:`~PyTypeObject.tp_itemsize` field. For a type with fixed-length instances, all - instances have the same size, given in :c:member:`~PyTypeObject.tp_basicsize`. + :c:member:`!tp_itemsize` field, types with variable-length instances have a non-zero + :c:member:`!tp_itemsize` field. For a type with fixed-length instances, all + instances have the same size, given in :c:member:`!tp_basicsize`. + (Exceptions to this rule can be made using + :c:func:`PyUnstable_Object_GC_NewWithExtraData`.) For a type with variable-length instances, the instances must have an - :c:member:`~PyVarObject.ob_size` field, and the instance size is :c:member:`~PyTypeObject.tp_basicsize` plus N - times :c:member:`~PyTypeObject.tp_itemsize`, where N is the "length" of the object. The value of - N is typically stored in the instance's :c:member:`~PyVarObject.ob_size` field. There are - exceptions: for example, ints use a negative :c:member:`~PyVarObject.ob_size` to indicate a - negative number, and N is ``abs(ob_size)`` there. Also, the presence of an - :c:member:`~PyVarObject.ob_size` field in the instance layout doesn't mean that the instance - structure is variable-length (for example, the structure for the list type has - fixed-length instances, yet those instances have a meaningful :c:member:`~PyVarObject.ob_size` - field). - - The basic size includes the fields in the instance declared by the macro - :c:macro:`PyObject_HEAD` or :c:macro:`PyObject_VAR_HEAD` (whichever is used to - declare the instance struct) and this in turn includes the :c:member:`~PyObject._ob_prev` and - :c:member:`~PyObject._ob_next` fields if they are present. This means that the only correct - way to get an initializer for the :c:member:`~PyTypeObject.tp_basicsize` is to use the - ``sizeof`` operator on the struct used to declare the instance layout. - The basic size does not include the GC header size. + :c:member:`~PyVarObject.ob_size` field, and the instance size is + :c:member:`!tp_basicsize` plus N times :c:member:`!tp_itemsize`, + where N is the "length" of the object. + + Functions like :c:func:`PyObject_NewVar` will take the value of N as an + argument, and store in the instance's :c:member:`~PyVarObject.ob_size` field. + Note that the :c:member:`~PyVarObject.ob_size` field may later be used for + other purposes. For example, :py:type:`int` instances use the bits of + :c:member:`~PyVarObject.ob_size` in an implementation-defined + way; the underlying storage and its size should be accessed using + :c:func:`PyLong_Export`. + + .. note:: + + The :c:member:`~PyVarObject.ob_size` field should be accessed using + the :c:func:`Py_SIZE()` and :c:func:`Py_SET_SIZE()` macros. - A note about alignment: if the variable items require a particular alignment, - this should be taken care of by the value of :c:member:`~PyTypeObject.tp_basicsize`. Example: - suppose a type implements an array of ``double``. :c:member:`~PyTypeObject.tp_itemsize` is - ``sizeof(double)``. It is the programmer's responsibility that - :c:member:`~PyTypeObject.tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the - alignment requirement for ``double``). + Also, the presence of an :c:member:`~PyVarObject.ob_size` field in the + instance layout doesn't mean that the instance structure is variable-length. + For example, the :py:type:`list` type has fixed-length instances, yet those + instances have a :c:member:`~PyVarObject.ob_size` field. + (As with :py:type:`int`, avoid reading lists' :c:member:`!ob_size` directly. + Call :c:func:`PyList_Size` instead.) - For any type with variable-length instances, this field must not be ``NULL``. + The :c:member:`!tp_basicsize` includes size needed for data of the type's + :c:member:`~PyTypeObject.tp_base`, plus any extra data needed + by each instance. + + The correct way to set :c:member:`!tp_basicsize` is to use the + ``sizeof`` operator on the struct used to declare the instance layout. + This struct must include the struct used to declare the base type. + In other words, :c:member:`!tp_basicsize` must be greater than or equal + to the base's :c:member:`!tp_basicsize`. + + Since every type is a subtype of :py:type:`object`, this struct must + include :c:type:`PyObject` or :c:type:`PyVarObject` (depending on + whether :c:member:`~PyVarObject.ob_size` should be included). These are + usually defined by the macro :c:macro:`PyObject_HEAD` or + :c:macro:`PyObject_VAR_HEAD`, respectively. + + The basic size does not include the GC header size, as that header is not + part of :c:macro:`PyObject_HEAD`. + + For cases where struct used to declare the base type is unknown, + see :c:member:`PyType_Spec.basicsize` and :c:func:`PyType_FromMetaclass`. + + Notes about alignment: + + - :c:member:`!tp_basicsize` must be a multiple of ``_Alignof(PyObject)``. + When using ``sizeof`` on a ``struct`` that includes + :c:macro:`PyObject_HEAD`, as recommended, the compiler ensures this. + When not using a C ``struct``, or when using compiler + extensions like ``__attribute__((packed))``, it is up to you. + - If the variable items require a particular alignment, + :c:member:`!tp_basicsize` and :c:member:`!tp_itemsize` must each be a + multiple of that alignment. + For example, if a type's variable part stores a ``double``, it is + your responsibility that both fields are a multiple of + ``_Alignof(double)``. **Inheritance:** - These fields are inherited separately by subtypes. If the base type has a - non-zero :c:member:`~PyTypeObject.tp_itemsize`, it is generally not safe to set + These fields are inherited separately by subtypes. + (That is, if the field is set to zero, :c:func:`PyType_Ready` will copy + the value from the base type, indicating that the instances do not + need additional storage.) + + If the base type has a non-zero :c:member:`~PyTypeObject.tp_itemsize`, it is generally not safe to set :c:member:`~PyTypeObject.tp_itemsize` to a different non-zero value in a subtype (though this depends on the implementation of the base type). .. c:member:: destructor PyTypeObject.tp_dealloc - A pointer to the instance destructor function. This function must be defined - unless the type guarantees that its instances will never be deallocated (as is - the case for the singletons ``None`` and ``Ellipsis``). The function signature is:: + .. corresponding-type-slot:: Py_tp_dealloc + + A pointer to the instance destructor function. The function signature is:: void tp_dealloc(PyObject *self); - The destructor function is called by the :c:func:`Py_DECREF` and - :c:func:`Py_XDECREF` macros when the new reference count is zero. At this point, - the instance is still in existence, but there are no references to it. The - destructor function should free all references which the instance owns, free all - memory buffers owned by the instance (using the freeing function corresponding - to the allocation function used to allocate the buffer), and call the type's - :c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable - (doesn't have the :c:macro:`Py_TPFLAGS_BASETYPE` flag bit set), it is - permissible to call the object deallocator directly instead of via - :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the - instance; this is normally :c:func:`PyObject_Free` if the instance was allocated - using :c:macro:`PyObject_New` or :c:macro:`PyObject_NewVar`, or - :c:func:`PyObject_GC_Del` if the instance was allocated using - :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. - - If the type supports garbage collection (has the :c:macro:`Py_TPFLAGS_HAVE_GC` - flag bit set), the destructor should call :c:func:`PyObject_GC_UnTrack` - before clearing any member fields. + The destructor function should remove all references which the instance owns + (e.g., call :c:func:`Py_CLEAR`), free all memory buffers owned by the + instance, and call the type's :c:member:`~PyTypeObject.tp_free` function to + free the object itself. + + If you may call functions that may set the error indicator, you must use + :c:func:`PyErr_GetRaisedException` and :c:func:`PyErr_SetRaisedException` + to ensure you don't clobber a preexisting error indicator (the deallocation + could have occurred while processing a different error): .. code-block:: c - static void foo_dealloc(foo_object *self) { + static void + foo_dealloc(foo_object *self) + { + PyObject *et, *ev, *etb; + PyObject *exc = PyErr_GetRaisedException(); + ... + PyErr_SetRaisedException(exc); + } + + The dealloc handler itself must not raise an exception; if it hits an error + case it should call :c:func:`PyErr_FormatUnraisable` to log (and clear) an + unraisable exception. + + No guarantees are made about when an object is destroyed, except: + + * Python will destroy an object immediately or some time after the final + reference to the object is deleted, unless its finalizer + (:c:member:`~PyTypeObject.tp_finalize`) subsequently resurrects the + object. + * An object will not be destroyed while it is being automatically finalized + (:c:member:`~PyTypeObject.tp_finalize`) or automatically cleared + (:c:member:`~PyTypeObject.tp_clear`). + + CPython currently destroys an object immediately from :c:func:`Py_DECREF` + when the new reference count is zero, but this may change in a future + version. + + It is recommended to call :c:func:`PyObject_CallFinalizerFromDealloc` at the + beginning of :c:member:`!tp_dealloc` to guarantee that the object is always + finalized before destruction. + + If the type supports garbage collection (the :c:macro:`Py_TPFLAGS_HAVE_GC` + flag is set), the destructor should call :c:func:`PyObject_GC_UnTrack` + before clearing any member fields. + + It is permissible to call :c:member:`~PyTypeObject.tp_clear` from + :c:member:`!tp_dealloc` to reduce code duplication and to guarantee that the + object is always cleared before destruction. Beware that + :c:member:`!tp_clear` might have already been called. + + If the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the + deallocator should release the owned reference to its type object (via + :c:func:`Py_DECREF`) after calling the type deallocator. See the example + code below.:: + + static void + foo_dealloc(PyObject *op) + { + foo_object *self = (foo_object *) op; PyObject_GC_UnTrack(self); Py_CLEAR(self->ref); - Py_TYPE(self)->tp_free((PyObject *)self); - } + Py_TYPE(self)->tp_free(self); + } - Finally, if the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the - deallocator should release the owned reference to its type object - (via :c:func:`Py_DECREF`) after - calling the type deallocator. In order to avoid dangling pointers, the - recommended way to achieve this is: + :c:member:`!tp_dealloc` must leave the exception status unchanged. If it + needs to call something that might raise an exception, the exception state + must be backed up first and restored later (after logging any exceptions + with :c:func:`PyErr_WriteUnraisable`). - .. code-block:: c + Example:: - static void foo_dealloc(foo_object *self) { - PyTypeObject *tp = Py_TYPE(self); - // free references and buffers here - tp->tp_free(self); - Py_DECREF(tp); - } + static void + foo_dealloc(PyObject *self) + { + PyObject *exc = PyErr_GetRaisedException(); - .. warning:: + if (PyObject_CallFinalizerFromDealloc(self) < 0) { + // self was resurrected. + goto done; + } - In a garbage collected Python, :c:member:`!tp_dealloc` may be called from - any Python thread, not just the thread which created the object (if the - object becomes part of a refcount cycle, that cycle might be collected by - a garbage collection on any thread). This is not a problem for Python - API calls, since the thread on which :c:member:`!tp_dealloc` is called - will own the Global Interpreter Lock (GIL). However, if the object being - destroyed in turn destroys objects from some other C or C++ library, care - should be taken to ensure that destroying those objects on the thread - which called :c:member:`!tp_dealloc` will not violate any assumptions of - the library. + PyTypeObject *tp = Py_TYPE(self); + + if (tp->tp_flags & Py_TPFLAGS_HAVE_GC) { + PyObject_GC_UnTrack(self); + } + + // Optional, but convenient to avoid code duplication. + if (tp->tp_clear && tp->tp_clear(self) < 0) { + PyErr_WriteUnraisable(self); + } + + // Any additional destruction goes here. + + tp->tp_free(self); + self = NULL; // In case PyErr_WriteUnraisable() is called below. + + if (tp->tp_flags & Py_TPFLAGS_HEAPTYPE) { + Py_CLEAR(tp); + } + + done: + // Optional, if something was called that might have raised an + // exception. + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(self); + } + PyErr_SetRaisedException(exc); + } + + :c:member:`!tp_dealloc` may be called from + any Python thread, not just the thread which created the object (if the + object becomes part of a refcount cycle, that cycle might be collected by + a garbage collection on any thread). This is not a problem for Python + API calls, since the thread on which :c:member:`!tp_dealloc` is called + with an :term:`attached thread state`. However, if the object being + destroyed in turn destroys objects from some other C library, care + should be taken to ensure that destroying those objects on the thread + which called :c:member:`!tp_dealloc` will not violate any assumptions of + the library. **Inheritance:** This field is inherited by subtypes. + .. seealso:: + + :ref:`life-cycle` for details about how this slot relates to other slots. + .. c:member:: Py_ssize_t PyTypeObject.tp_vectorcall_offset @@ -748,6 +862,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrfunc PyTypeObject.tp_getattr + .. corresponding-type-slot:: Py_tp_getattr + An optional pointer to the get-attribute-string function. This field is deprecated. When it is defined, it should point to a function @@ -765,6 +881,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrfunc PyTypeObject.tp_setattr + .. corresponding-type-slot:: Py_tp_setattr + An optional pointer to the function for setting and deleting attributes. This field is deprecated. When it is defined, it should point to a function @@ -797,6 +915,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_repr + .. corresponding-type-slot:: Py_tp_repr + .. index:: pair: built-in function; repr An optional pointer to a function that implements the built-in function @@ -862,6 +982,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: hashfunc PyTypeObject.tp_hash + .. corresponding-type-slot:: Py_tp_hash + .. index:: pair: built-in function; hash An optional pointer to a function that implements the built-in function @@ -903,6 +1025,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: ternaryfunc PyTypeObject.tp_call + .. corresponding-type-slot:: Py_tp_call + An optional pointer to a function that implements calling the object. This should be ``NULL`` if the object is not callable. The signature is the same as for :c:func:`PyObject_Call`:: @@ -916,6 +1040,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_str + .. corresponding-type-slot:: Py_tp_str + An optional pointer to a function that implements the built-in operation :func:`str`. (Note that :class:`str` is a type now, and :func:`str` calls the constructor for that type. This constructor calls :c:func:`PyObject_Str` to do @@ -941,6 +1067,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrofunc PyTypeObject.tp_getattro + .. corresponding-type-slot:: Py_tp_getattro + An optional pointer to the get-attribute function. The signature is the same as for :c:func:`PyObject_GetAttr`:: @@ -965,6 +1093,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrofunc PyTypeObject.tp_setattro + .. corresponding-type-slot:: Py_tp_setattro + An optional pointer to the function for setting and deleting attributes. The signature is the same as for :c:func:`PyObject_SetAttr`:: @@ -1023,6 +1153,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have ``NULL`` values. + .. XXX are most flag bits *really* inherited individually? **Default:** @@ -1089,11 +1220,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_HAVE_GC This bit is set when the object supports garbage collection. If this bit - is set, instances must be created using :c:macro:`PyObject_GC_New` and - destroyed using :c:func:`PyObject_GC_Del`. More information in section - :ref:`supporting-cycle-detection`. This bit also implies that the - GC-related fields :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` are present in - the type object. + is set, memory for new instances (see :c:member:`~PyTypeObject.tp_alloc`) + must be allocated using :c:macro:`PyObject_GC_New` or + :c:func:`PyType_GenericAlloc` and deallocated (see + :c:member:`~PyTypeObject.tp_free`) using :c:func:`PyObject_GC_Del`. More + information in section :ref:`supporting-cycle-detection`. **Inheritance:** @@ -1144,10 +1275,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_MANAGED_DICT - This bit indicates that instances of the class have a `~object.__dict__` + This bit indicates that instances of the class have a :attr:`~object.__dict__` attribute, and that the space for the dictionary is managed by the VM. - If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. + If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` must also be set. The type traverse function must call :c:func:`PyObject_VisitManagedDict` and its clear function must call :c:func:`PyObject_ClearManagedDict`. @@ -1165,6 +1296,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) This bit indicates that instances of the class should be weakly referenceable. + If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` must also be set. + .. versionadded:: 3.12 **Inheritance:** @@ -1173,6 +1306,19 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_weaklistoffset` field is set in a superclass. + .. c:macro:: Py_TPFLAGS_PREHEADER + + These bits indicate that the VM will manage some fields by storing them + before the object. Currently, this macro is equivalent to + :c:expr:`Py_TPFLAGS_MANAGED_DICT | Py_TPFLAGS_MANAGED_WEAKREF`. + + This macro value relies on the implementation of the VM, so its value is not + stable and may change in a future version. Prefer using individual + flags instead. + + .. versionadded:: 3.12 + + .. c:macro:: Py_TPFLAGS_ITEMS_AT_END Only usable with variable-size types, i.e. ones with non-zero @@ -1205,8 +1351,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_BASE_EXC_SUBCLASS .. c:macro:: Py_TPFLAGS_TYPE_SUBCLASS - These flags are used by functions such as - :c:func:`PyLong_Check` to quickly determine if a type is a subclass + Functions such as :c:func:`PyLong_Check` will call :c:func:`PyType_FastSubclass` + with one of these flags to quickly determine if a type is a subclass of a built-in type; such specific checks are faster than a generic check, like :c:func:`PyObject_IsInstance`. Custom types that inherit from built-ins should have their :c:member:`~PyTypeObject.tp_flags` @@ -1227,6 +1373,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) type structure. + .. c:macro:: _Py_TPFLAGS_HAVE_VECTORCALL + :no-typesetting: + .. c:macro:: Py_TPFLAGS_HAVE_VECTORCALL This bit is set when the class implements @@ -1238,7 +1387,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) This bit is inherited if :c:member:`~PyTypeObject.tp_call` is also inherited. - .. versionadded:: 3.9 + .. versionadded:: 3.8 as ``_Py_TPFLAGS_HAVE_VECTORCALL`` + + .. versionchanged:: 3.9 + + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. versionchanged:: 3.12 @@ -1347,6 +1501,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: const char* PyTypeObject.tp_doc + .. corresponding-type-slot:: Py_tp_doc + An optional pointer to a NUL-terminated C string giving the docstring for this type object. This is exposed as the :attr:`~type.__doc__` attribute on the type and instances of the type. @@ -1358,6 +1514,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: traverseproc PyTypeObject.tp_traverse + .. corresponding-type-slot:: Py_tp_traverse + An optional pointer to a traversal function for the garbage collector. This is only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: @@ -1373,8 +1531,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) :mod:`!_thread` extension module:: static int - local_traverse(localobject *self, visitproc visit, void *arg) + local_traverse(PyObject *op, visitproc visit, void *arg) { + localobject *self = (localobject *) op; Py_VISIT(self->args); Py_VISIT(self->kw); Py_VISIT(self->dict); @@ -1418,6 +1577,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) but the instance has no strong reference to the elements inside it, as they are allowed to be removed even if the instance is still alive). + .. warning:: + The traversal function must not have any side effects. It must not + modify the reference counts of any Python objects nor create or destroy + any Python objects. + Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to :c:func:`!local_traverse` to have these specific names; don't name them just anything. @@ -1429,6 +1593,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) heap-allocated superclass). If they do not, the type object may not be garbage-collected. + .. note:: + + The :c:member:`~PyTypeObject.tp_traverse` function can be called from any + thread. + .. versionchanged:: 3.9 Heap-allocated types are expected to visit ``Py_TYPE(self)`` in @@ -1448,28 +1617,112 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_clear - An optional pointer to a clear function for the garbage collector. This is only - used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: + .. corresponding-type-slot:: Py_tp_clear + + An optional pointer to a clear function. The signature is:: int tp_clear(PyObject *); - The :c:member:`~PyTypeObject.tp_clear` member function is used to break reference cycles in cyclic - garbage detected by the garbage collector. Taken together, all :c:member:`~PyTypeObject.tp_clear` - functions in the system must combine to break all reference cycles. This is - subtle, and if in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For example, - the tuple type does not implement a :c:member:`~PyTypeObject.tp_clear` function, because it's - possible to prove that no reference cycle can be composed entirely of tuples. - Therefore the :c:member:`~PyTypeObject.tp_clear` functions of other types must be sufficient to - break any cycle containing a tuple. This isn't immediately obvious, and there's - rarely a good reason to avoid implementing :c:member:`~PyTypeObject.tp_clear`. + The purpose of this function is to break reference cycles that are causing a + :term:`cyclic isolate` so that the objects can be safely destroyed. A + cleared object is a partially destroyed object; the object is not obligated + to satisfy design invariants held during normal use. + + :c:member:`!tp_clear` does not need to delete references to objects that + can't participate in reference cycles, such as Python strings or Python + integers. However, it may be convenient to clear all references, and write + the type's :c:member:`~PyTypeObject.tp_dealloc` function to invoke + :c:member:`!tp_clear` to avoid code duplication. (Beware that + :c:member:`!tp_clear` might have already been called. Prefer calling + idempotent functions like :c:func:`Py_CLEAR`.) + + Any non-trivial cleanup should be performed in + :c:member:`~PyTypeObject.tp_finalize` instead of :c:member:`!tp_clear`. + + .. note:: + + If :c:member:`!tp_clear` fails to break a reference cycle then the + objects in the :term:`cyclic isolate` may remain indefinitely + uncollectable ("leak"). See :data:`gc.garbage`. + + .. note:: + + Referents (direct and indirect) might have already been cleared; they are + not guaranteed to be in a consistent state. + + .. note:: + + The :c:member:`~PyTypeObject.tp_clear` function can be called from any + thread. + + .. note:: + + An object is not guaranteed to be automatically cleared before its + destructor (:c:member:`~PyTypeObject.tp_dealloc`) is called. + + This function differs from the destructor + (:c:member:`~PyTypeObject.tp_dealloc`) in the following ways: + + * The purpose of clearing an object is to remove references to other objects + that might participate in a reference cycle. The purpose of the + destructor, on the other hand, is a superset: it must release *all* + resources it owns, including references to objects that cannot participate + in a reference cycle (e.g., integers) as well as the object's own memory + (by calling :c:member:`~PyTypeObject.tp_free`). + * When :c:member:`!tp_clear` is called, other objects might still hold + references to the object being cleared. Because of this, + :c:member:`!tp_clear` must not deallocate the object's own memory + (:c:member:`~PyTypeObject.tp_free`). The destructor, on the other hand, + is only called when no (strong) references exist, and as such, must + safely destroy the object itself by deallocating it. + * :c:member:`!tp_clear` might never be automatically called. An object's + destructor, on the other hand, will be automatically called some time + after the object becomes unreachable (i.e., either there are no references + to the object or the object is a member of a :term:`cyclic isolate`). + + No guarantees are made about when, if, or how often Python automatically + clears an object, except: + + * Python will not automatically clear an object if it is reachable, i.e., + there is a reference to it and it is not a member of a :term:`cyclic + isolate`. + * Python will not automatically clear an object if it has not been + automatically finalized (see :c:member:`~PyTypeObject.tp_finalize`). (If + the finalizer resurrected the object, the object may or may not be + automatically finalized again before it is cleared.) + * If an object is a member of a :term:`cyclic isolate`, Python will not + automatically clear it if any member of the cyclic isolate has not yet + been automatically finalized (:c:member:`~PyTypeObject.tp_finalize`). + * Python will not destroy an object until after any automatic calls to its + :c:member:`!tp_clear` function have returned. This ensures that the act + of breaking a reference cycle does not invalidate the ``self`` pointer + while :c:member:`!tp_clear` is still executing. + * Python will not automatically call :c:member:`!tp_clear` multiple times + concurrently. + + CPython currently only automatically clears objects as needed to break + reference cycles in a :term:`cyclic isolate`, but future versions might + clear objects regularly before their destruction. + + Taken together, all :c:member:`~PyTypeObject.tp_clear` functions in the + system must combine to break all reference cycles. This is subtle, and if + in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For + example, the tuple type does not implement a + :c:member:`~PyTypeObject.tp_clear` function, because it's possible to prove + that no reference cycle can be composed entirely of tuples. Therefore the + :c:member:`~PyTypeObject.tp_clear` functions of other types are responsible + for breaking any cycle containing a tuple. This isn't immediately obvious, + and there's rarely a good reason to avoid implementing + :c:member:`~PyTypeObject.tp_clear`. Implementations of :c:member:`~PyTypeObject.tp_clear` should drop the instance's references to those of its members that may be Python objects, and set its pointers to those members to ``NULL``, as in the following example:: static int - local_clear(localobject *self) + local_clear(PyObject *op) { + localobject *self = (localobject *) op; Py_CLEAR(self->key); Py_CLEAR(self->args); Py_CLEAR(self->kw); @@ -1490,23 +1743,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:func:`Py_CLEAR` macro performs the operations in a safe order. If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the - :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:member:`~PyTypeObject.tp_flags` field, the clear function must call :c:func:`PyObject_ClearManagedDict` like this:: PyObject_ClearManagedDict((PyObject*)self); - Note that :c:member:`~PyTypeObject.tp_clear` is not *always* called - before an instance is deallocated. For example, when reference counting - is enough to determine that an object is no longer used, the cyclic garbage - collector is not involved and :c:member:`~PyTypeObject.tp_dealloc` is - called directly. - - Because the goal of :c:member:`~PyTypeObject.tp_clear` functions is to break reference cycles, - it's not necessary to clear contained objects like Python strings or Python - integers, which can't participate in reference cycles. On the other hand, it may - be convenient to clear all contained Python objects, and write the type's - :c:member:`~PyTypeObject.tp_dealloc` function to invoke :c:member:`~PyTypeObject.tp_clear`. - More information about Python's garbage collection scheme can be found in section :ref:`supporting-cycle-detection`. @@ -1519,9 +1760,15 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in the subtype. + .. seealso:: + + :ref:`life-cycle` for details about how this slot relates to other slots. + .. c:member:: richcmpfunc PyTypeObject.tp_richcompare + .. corresponding-type-slot:: Py_tp_richcompare + An optional pointer to the rich comparison function, whose signature is:: PyObject *tp_richcompare(PyObject *self, PyObject *other, int op); @@ -1624,6 +1871,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getiterfunc PyTypeObject.tp_iter + .. corresponding-type-slot:: Py_tp_iter + An optional pointer to a function that returns an :term:`iterator` for the object. Its presence normally signals that the instances of this type are :term:`iterable` (although sequences may be iterable without this function). @@ -1639,6 +1888,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: iternextfunc PyTypeObject.tp_iternext + .. corresponding-type-slot:: Py_tp_iternext + An optional pointer to a function that returns the next item in an :term:`iterator`. The signature is:: @@ -1662,6 +1913,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMethodDef* PyTypeObject.tp_methods + .. corresponding-type-slot:: Py_tp_methods + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMethodDef` structures, declaring regular methods of this type. @@ -1676,6 +1929,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMemberDef* PyTypeObject.tp_members + .. corresponding-type-slot:: Py_tp_members + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMemberDef` structures, declaring regular data members (fields or slots) of instances of this type. @@ -1691,6 +1946,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyGetSetDef* PyTypeObject.tp_getset + .. corresponding-type-slot:: Py_tp_getset + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyGetSetDef` structures, declaring computed attributes of instances of this type. @@ -1705,6 +1962,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyTypeObject* PyTypeObject.tp_base + .. corresponding-type-slot:: Py_tp_base + An optional pointer to a base type from which type properties are inherited. At this level, only single inheritance is supported; multiple inheritance require dynamically creating a type object by calling the metatype. @@ -1777,6 +2036,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrgetfunc PyTypeObject.tp_descr_get + .. corresponding-type-slot:: Py_tp_descr_get + An optional pointer to a "descriptor get" function. The function signature is:: @@ -1792,6 +2053,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrsetfunc PyTypeObject.tp_descr_set + .. corresponding-type-slot:: Py_tp_descr_set + An optional pointer to a function for setting and deleting a descriptor's value. @@ -1829,7 +2092,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) dictionary, so it is may be more efficient to call :c:func:`PyObject_GetAttr` when accessing an attribute on the object. - It is an error to set both the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit and + It is an error to set both the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit and :c:member:`~PyTypeObject.tp_dictoffset`. **Inheritance:** @@ -1852,6 +2115,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: initproc PyTypeObject.tp_init + .. corresponding-type-slot:: Py_tp_init + An optional pointer to an instance initialization function. This function corresponds to the :meth:`~object.__init__` method of classes. Like @@ -1887,6 +2152,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: allocfunc PyTypeObject.tp_alloc + .. corresponding-type-slot:: Py_tp_alloc + An optional pointer to an instance allocation function. The function signature is:: @@ -1895,22 +2162,23 @@ and :c:data:`PyType_Type` effectively act as defaults.) **Inheritance:** - This field is inherited by static subtypes, but not by dynamic - subtypes (subtypes created by a class statement). + Static subtypes inherit this slot, which will be + :c:func:`PyType_GenericAlloc` if inherited from :class:`object`. + + :ref:`Heap subtypes ` do not inherit this slot. **Default:** - For dynamic subtypes, this field is always set to - :c:func:`PyType_GenericAlloc`, to force a standard heap - allocation strategy. + For heap subtypes, this field is always set to + :c:func:`PyType_GenericAlloc`. - For static subtypes, :c:data:`PyBaseObject_Type` uses - :c:func:`PyType_GenericAlloc`. That is the recommended value - for all statically defined types. + For static subtypes, this slot is inherited (see above). .. c:member:: newfunc PyTypeObject.tp_new + .. corresponding-type-slot:: Py_tp_new + An optional pointer to an instance creation function. The function signature is:: @@ -1950,28 +2218,39 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: freefunc PyTypeObject.tp_free + .. corresponding-type-slot:: Py_tp_free + An optional pointer to an instance deallocation function. Its signature is:: void tp_free(void *self); - An initializer that is compatible with this signature is :c:func:`PyObject_Free`. + This function must free the memory allocated by + :c:member:`~PyTypeObject.tp_alloc`. **Inheritance:** - This field is inherited by static subtypes, but not by dynamic - subtypes (subtypes created by a class statement) + Static subtypes inherit this slot, which will be :c:func:`PyObject_Free` if + inherited from :class:`object`. Exception: If the type supports garbage + collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`) and it would inherit + :c:func:`PyObject_Free`, then this slot is not inherited but instead defaults + to :c:func:`PyObject_GC_Del`. + + :ref:`Heap subtypes ` do not inherit this slot. **Default:** - In dynamic subtypes, this field is set to a deallocator suitable to - match :c:func:`PyType_GenericAlloc` and the value of the - :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit. + For :ref:`heap subtypes `, this slot defaults to a deallocator suitable to match + :c:func:`PyType_GenericAlloc` and the value of the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag. - For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_Free`. + For static subtypes, this slot is inherited (see above). .. c:member:: inquiry PyTypeObject.tp_is_gc + .. corresponding-type-slot:: Py_tp_is_gc + An optional pointer to a function called by the garbage collector. The garbage collector needs to know whether a particular object is collectible @@ -2000,12 +2279,14 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyObject* PyTypeObject.tp_bases + .. corresponding-type-slot:: Py_tp_bases + Tuple of base types. This field should be set to ``NULL`` and treated as read-only. Python will fill it in when the type is :c:func:`initialized `. - For dynamically created classes, the ``Py_tp_bases`` + For dynamically created classes, the :c:data:`Py_tp_bases` :c:type:`slot ` can be used instead of the *bases* argument of :c:func:`PyType_FromSpecWithBases`. The argument form is preferred. @@ -2080,6 +2361,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_del + .. corresponding-type-slot:: Py_tp_del + This field is deprecated. Use :c:member:`~PyTypeObject.tp_finalize` instead. @@ -2094,32 +2377,141 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_finalize - An optional pointer to an instance finalization function. Its signature is:: + .. corresponding-type-slot:: Py_tp_finalize + + An optional pointer to an instance finalization function. This is the C + implementation of the :meth:`~object.__del__` special method. Its signature + is:: void tp_finalize(PyObject *self); - If :c:member:`~PyTypeObject.tp_finalize` is set, the interpreter calls it once when - finalizing an instance. It is called either from the garbage - collector (if the instance is part of an isolated reference cycle) or - just before the object is deallocated. Either way, it is guaranteed - to be called before attempting to break reference cycles, ensuring - that it finds the object in a sane state. + The primary purpose of finalization is to perform any non-trivial cleanup + that must be performed before the object is destroyed, while the object and + any other objects it directly or indirectly references are still in a + consistent state. The finalizer is allowed to execute + arbitrary Python code. + + Before Python automatically finalizes an object, some of the object's direct + or indirect referents might have themselves been automatically finalized. + However, none of the referents will have been automatically cleared + (:c:member:`~PyTypeObject.tp_clear`) yet. + + Other non-finalized objects might still be using a finalized object, so the + finalizer must leave the object in a sane state (e.g., invariants are still + met). + + .. note:: + + After Python automatically finalizes an object, Python might start + automatically clearing (:c:member:`~PyTypeObject.tp_clear`) the object + and its referents (direct and indirect). Cleared objects are not + guaranteed to be in a consistent state; a finalized object must be able + to tolerate cleared referents. + + .. note:: - :c:member:`~PyTypeObject.tp_finalize` should not mutate the current exception status; - therefore, a recommended way to write a non-trivial finalizer is:: + An object is not guaranteed to be automatically finalized before its + destructor (:c:member:`~PyTypeObject.tp_dealloc`) is called. It is + recommended to call :c:func:`PyObject_CallFinalizerFromDealloc` at the + beginning of :c:member:`!tp_dealloc` to guarantee that the object is + always finalized before destruction. + + .. note:: + + The :c:member:`~PyTypeObject.tp_finalize` function can be called from any + thread, although the :term:`GIL` will be held. + + .. note:: + + The :c:member:`!tp_finalize` function can be called during shutdown, + after some global variables have been deleted. See the documentation of + the :meth:`~object.__del__` method for details. + + When Python finalizes an object, it behaves like the following algorithm: + + #. Python might mark the object as *finalized*. Currently, Python always + marks objects whose type supports garbage collection (i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`) and never marks other types of + objects; this might change in a future version. + #. If the object is not marked as *finalized* and its + :c:member:`!tp_finalize` finalizer function is non-``NULL``, the + finalizer function is called. + #. If the finalizer function was called and the finalizer made the object + reachable (i.e., there is a reference to the object and it is not a + member of a :term:`cyclic isolate`), then the finalizer is said to have + *resurrected* the object. It is unspecified whether the finalizer can + also resurrect the object by adding a new reference to the object that + does not make it reachable, i.e., the object is (still) a member of a + cyclic isolate. + #. If the finalizer resurrected the object, the object's pending destruction + is canceled and the object's *finalized* mark might be removed if + present. Currently, Python never removes the *finalized* mark; this + might change in a future version. + + *Automatic finalization* refers to any finalization performed by Python + except via calls to :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc`. No guarantees are made about + when, if, or how often an object is automatically finalized, except: + + * Python will not automatically finalize an object if it is reachable, i.e., + there is a reference to it and it is not a member of a :term:`cyclic + isolate`. + * Python will not automatically finalize an object if finalizing it would + not mark the object as *finalized*. Currently, this applies to objects + whose type does not support garbage collection, i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is not set. Such objects can still be + manually finalized by calling :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc`. + * Python will not automatically finalize any two members of a :term:`cyclic + isolate` concurrently. + * Python will not automatically finalize an object after it has + automatically cleared (:c:member:`~PyTypeObject.tp_clear`) the object. + * If an object is a member of a :term:`cyclic isolate`, Python will not + automatically finalize it after automatically clearing (see + :c:member:`~PyTypeObject.tp_clear`) any other member. + * Python will automatically finalize every member of a :term:`cyclic + isolate` before it automatically clears (see + :c:member:`~PyTypeObject.tp_clear`) any of them. + * If Python is going to automatically clear an object + (:c:member:`~PyTypeObject.tp_clear`), it will automatically finalize the + object first. + + Python currently only automatically finalizes objects that are members of a + :term:`cyclic isolate`, but future versions might finalize objects regularly + before their destruction. + + To manually finalize an object, do not call this function directly; call + :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc` instead. + + :c:member:`~PyTypeObject.tp_finalize` should leave the current exception + status unchanged. The recommended way to write a non-trivial finalizer is + to back up the exception at the beginning by calling + :c:func:`PyErr_GetRaisedException` and restore the exception at the end by + calling :c:func:`PyErr_SetRaisedException`. If an exception is encountered + in the middle of the finalizer, log and clear it with + :c:func:`PyErr_WriteUnraisable` or :c:func:`PyErr_FormatUnraisable`. For + example:: static void - local_finalize(PyObject *self) + foo_finalize(PyObject *self) { - PyObject *error_type, *error_value, *error_traceback; + // Save the current exception, if any. + PyObject *exc = PyErr_GetRaisedException(); - /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + // ... - /* ... */ + if (do_something_that_might_raise() != success_indicator) { + PyErr_WriteUnraisable(self); + goto done; + } - /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + done: + // Restore the saved exception. This silently discards any exception + // raised above, so be sure to call PyErr_WriteUnraisable first if + // necessary. + PyErr_SetRaisedException(exc); } **Inheritance:** @@ -2134,11 +2526,19 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:macro:`Py_TPFLAGS_HAVE_FINALIZE` flags bit in order for this field to be used. This is no longer required. - .. seealso:: "Safe object finalization" (:pep:`442`) + .. seealso:: + + * :pep:`442`: "Safe object finalization" + * :ref:`life-cycle` for details about how this slot relates to other + slots. + * :c:func:`PyObject_CallFinalizer` + * :c:func:`PyObject_CallFinalizerFromDealloc` .. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall + .. corresponding-type-slot:: Py_tp_vectorcall + A :ref:`vectorcall function ` to use for calls of this type object (rather than instances). In other words, ``tp_vectorcall`` can be used to optimize ``type.__call__``, @@ -2230,7 +2630,7 @@ This is done by filling a :c:type:`PyType_Spec` structure and calling .. _number-structs: Number Object Structures -======================== +------------------------ .. sectionauthor:: Amaury Forgeot d'Arc @@ -2304,47 +2704,153 @@ Number Object Structures Python 3.0.1. .. c:member:: binaryfunc PyNumberMethods.nb_add + + .. corresponding-type-slot:: Py_nb_add + .. c:member:: binaryfunc PyNumberMethods.nb_subtract + + .. corresponding-type-slot:: Py_nb_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_multiply + + .. corresponding-type-slot:: Py_nb_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_remainder + + .. corresponding-type-slot:: Py_nb_remainder + .. c:member:: binaryfunc PyNumberMethods.nb_divmod + + .. corresponding-type-slot:: Py_nb_divmod + .. c:member:: ternaryfunc PyNumberMethods.nb_power + + .. corresponding-type-slot:: Py_nb_power + .. c:member:: unaryfunc PyNumberMethods.nb_negative + + .. corresponding-type-slot:: Py_nb_negative + .. c:member:: unaryfunc PyNumberMethods.nb_positive + + .. corresponding-type-slot:: Py_nb_positive + .. c:member:: unaryfunc PyNumberMethods.nb_absolute + + .. corresponding-type-slot:: Py_nb_absolute + .. c:member:: inquiry PyNumberMethods.nb_bool + + .. corresponding-type-slot:: Py_nb_bool + .. c:member:: unaryfunc PyNumberMethods.nb_invert + + .. corresponding-type-slot:: Py_nb_invert + .. c:member:: binaryfunc PyNumberMethods.nb_lshift + + .. corresponding-type-slot:: Py_nb_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_rshift + + .. corresponding-type-slot:: Py_nb_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_and + + .. corresponding-type-slot:: Py_nb_and + .. c:member:: binaryfunc PyNumberMethods.nb_xor + + .. corresponding-type-slot:: Py_nb_xor + .. c:member:: binaryfunc PyNumberMethods.nb_or + + .. corresponding-type-slot:: Py_nb_or + .. c:member:: unaryfunc PyNumberMethods.nb_int + + .. corresponding-type-slot:: Py_nb_int + .. c:member:: void *PyNumberMethods.nb_reserved + .. c:member:: unaryfunc PyNumberMethods.nb_float + + .. corresponding-type-slot:: Py_nb_float + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_add + + .. corresponding-type-slot:: Py_nb_inplace_add + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_subtract + + .. corresponding-type-slot:: Py_nb_inplace_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_multiply + + .. corresponding-type-slot:: Py_nb_inplace_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_remainder + + .. corresponding-type-slot:: Py_nb_inplace_remainder + .. c:member:: ternaryfunc PyNumberMethods.nb_inplace_power + + .. corresponding-type-slot:: Py_nb_inplace_power + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_lshift + + .. corresponding-type-slot:: Py_nb_inplace_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_rshift + + .. corresponding-type-slot:: Py_nb_inplace_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_and + + .. corresponding-type-slot:: Py_nb_inplace_and + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_xor + + .. corresponding-type-slot:: Py_nb_inplace_xor + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_or + + .. corresponding-type-slot:: Py_nb_inplace_or + .. c:member:: binaryfunc PyNumberMethods.nb_floor_divide + + .. corresponding-type-slot:: Py_nb_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_true_divide + + .. corresponding-type-slot:: Py_nb_true_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_floor_divide + + .. corresponding-type-slot:: Py_nb_inplace_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_true_divide + + .. corresponding-type-slot:: Py_nb_inplace_true_divide + .. c:member:: unaryfunc PyNumberMethods.nb_index + + .. corresponding-type-slot:: Py_nb_index + .. c:member:: binaryfunc PyNumberMethods.nb_matrix_multiply + + .. corresponding-type-slot:: Py_nb_matrix_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_matrix_multiply + .. corresponding-type-slot:: Py_nb_inplace_matrix_multiply + + .. _mapping-structs: Mapping Object Structures -========================= +------------------------- .. sectionauthor:: Amaury Forgeot d'Arc @@ -2356,12 +2862,16 @@ Mapping Object Structures .. c:member:: lenfunc PyMappingMethods.mp_length + .. corresponding-type-slot:: Py_mp_length + This function is used by :c:func:`PyMapping_Size` and :c:func:`PyObject_Size`, and has the same signature. This slot may be set to ``NULL`` if the object has no defined length. .. c:member:: binaryfunc PyMappingMethods.mp_subscript + .. corresponding-type-slot:: Py_mp_subscript + This function is used by :c:func:`PyObject_GetItem` and :c:func:`PySequence_GetSlice`, and has the same signature as :c:func:`!PyObject_GetItem`. This slot must be filled for the @@ -2370,6 +2880,8 @@ Mapping Object Structures .. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript + .. corresponding-type-slot:: Py_mp_ass_subscript + This function is used by :c:func:`PyObject_SetItem`, :c:func:`PyObject_DelItem`, :c:func:`PySequence_SetSlice` and :c:func:`PySequence_DelSlice`. It has the same signature as @@ -2381,7 +2893,7 @@ Mapping Object Structures .. _sequence-structs: Sequence Object Structures -========================== +-------------------------- .. sectionauthor:: Amaury Forgeot d'Arc @@ -2393,6 +2905,8 @@ Sequence Object Structures .. c:member:: lenfunc PySequenceMethods.sq_length + .. corresponding-type-slot:: Py_sq_length + This function is used by :c:func:`PySequence_Size` and :c:func:`PyObject_Size`, and has the same signature. It is also used for handling negative indices via the :c:member:`~PySequenceMethods.sq_item` @@ -2400,18 +2914,24 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_concat + .. corresponding-type-slot:: Py_sq_concat + This function is used by :c:func:`PySequence_Concat` and has the same signature. It is also used by the ``+`` operator, after trying the numeric addition via the :c:member:`~PyNumberMethods.nb_add` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_repeat + .. corresponding-type-slot:: Py_sq_repeat + This function is used by :c:func:`PySequence_Repeat` and has the same signature. It is also used by the ``*`` operator, after trying numeric multiplication via the :c:member:`~PyNumberMethods.nb_multiply` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_item + .. corresponding-type-slot:: Py_sq_item + This function is used by :c:func:`PySequence_GetItem` and has the same signature. It is also used by :c:func:`PyObject_GetItem`, after trying the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot. @@ -2425,6 +2945,8 @@ Sequence Object Structures .. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item + .. corresponding-type-slot:: Py_sq_ass_item + This function is used by :c:func:`PySequence_SetItem` and has the same signature. It is also used by :c:func:`PyObject_SetItem` and :c:func:`PyObject_DelItem`, after trying the item assignment and deletion @@ -2434,6 +2956,8 @@ Sequence Object Structures .. c:member:: objobjproc PySequenceMethods.sq_contains + .. corresponding-type-slot:: Py_sq_contains + This function may be used by :c:func:`PySequence_Contains` and has the same signature. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_Contains` simply traverses the sequence until it @@ -2441,6 +2965,8 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_inplace_concat + .. corresponding-type-slot:: Py_sq_inplace_concat + This function is used by :c:func:`PySequence_InPlaceConcat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceConcat` @@ -2450,6 +2976,8 @@ Sequence Object Structures .. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat + .. corresponding-type-slot:: Py_sq_inplace_repeat + This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceRepeat` @@ -2461,7 +2989,7 @@ Sequence Object Structures .. _buffer-structs: Buffer Object Structures -======================== +------------------------ .. sectionauthor:: Greg J. Stein .. sectionauthor:: Benjamin Peterson @@ -2475,6 +3003,8 @@ Buffer Object Structures .. c:member:: getbufferproc PyBufferProcs.bf_getbuffer + .. corresponding-type-slot:: Py_bf_getbuffer + The signature of this function is:: int (PyObject *exporter, Py_buffer *view, int flags); @@ -2524,6 +3054,8 @@ Buffer Object Structures .. c:member:: releasebufferproc PyBufferProcs.bf_releasebuffer + .. corresponding-type-slot:: Py_bf_releasebuffer + The signature of this function is:: void (PyObject *exporter, Py_buffer *view); @@ -2556,7 +3088,7 @@ Buffer Object Structures Async Object Structures -======================= +----------------------- .. sectionauthor:: Yury Selivanov @@ -2578,6 +3110,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_await + .. corresponding-type-slot:: Py_am_await + The signature of this function is:: PyObject *am_await(PyObject *self); @@ -2589,6 +3123,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_aiter + .. corresponding-type-slot:: Py_am_aiter + The signature of this function is:: PyObject *am_aiter(PyObject *self); @@ -2601,6 +3137,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_anext + .. corresponding-type-slot:: Py_am_anext + The signature of this function is:: PyObject *am_anext(PyObject *self); @@ -2611,6 +3149,8 @@ Async Object Structures .. c:member:: sendfunc PyAsyncMethods.am_send + .. corresponding-type-slot:: Py_am_send + The signature of this function is:: PySendResult am_send(PyObject *self, PyObject *arg, PyObject **result); @@ -2624,7 +3164,7 @@ Async Object Structures .. _slot-typedefs: Slot Type typedefs -================== +------------------ .. c:type:: PyObject *(*allocfunc)(PyTypeObject *cls, Py_ssize_t nitems) @@ -2647,7 +3187,7 @@ Slot Type typedefs See :c:member:`~PyTypeObject.tp_free`. -.. c:type:: PyObject *(*newfunc)(PyObject *, PyObject *, PyObject *) +.. c:type:: PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *) See :c:member:`~PyTypeObject.tp_new`. @@ -2733,7 +3273,7 @@ Slot Type typedefs .. _typedef-examples: Examples -======== +-------- The following are simple examples of Python type definitions. They include common usage you may encounter. Some demonstrate tricky corner diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 59bd7661965d93..d2b6643c700e88 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -31,6 +31,18 @@ Unicode Type These are the basic Unicode object types used for the Unicode implementation in Python: +.. c:var:: PyTypeObject PyUnicode_Type + + This instance of :c:type:`PyTypeObject` represents the Python Unicode type. + It is exposed to Python code as :py:class:`str`. + + +.. c:var:: PyTypeObject PyUnicodeIter_Type + + This instance of :c:type:`PyTypeObject` represents the Python Unicode + iterator type. It is used to iterate over Unicode string objects. + + .. c:type:: Py_UCS4 Py_UCS2 Py_UCS1 @@ -42,19 +54,6 @@ Python: .. versionadded:: 3.3 -.. c:type:: Py_UNICODE - - This is a typedef of :c:type:`wchar_t`, which is a 16-bit type or 32-bit type - depending on the platform. - - .. versionchanged:: 3.3 - In previous versions, this was a 16-bit type or a 32-bit type depending on - whether you selected a "narrow" or "wide" Unicode version of Python at - build time. - - .. deprecated-removed:: 3.13 3.15 - - .. c:type:: PyASCIIObject PyCompactUnicodeObject PyUnicodeObject @@ -66,10 +65,25 @@ Python: .. versionadded:: 3.3 -.. c:var:: PyTypeObject PyUnicode_Type + The structure of a particular object can be determined using the following + macros. + The macros cannot fail; their behavior is undefined if their argument + is not a Python Unicode object. + + .. c:namespace:: NULL - This instance of :c:type:`PyTypeObject` represents the Python Unicode type. It - is exposed to Python code as ``str``. + .. c:macro:: PyUnicode_IS_COMPACT(o) + + True if *o* uses the :c:struct:`PyCompactUnicodeObject` structure. + + .. versionadded:: 3.3 + + + .. c:macro:: PyUnicode_IS_COMPACT_ASCII(o) + + True if *o* uses the :c:struct:`PyASCIIObject` structure. + + .. versionadded:: 3.3 The following APIs are C macros and static inlined functions for fast checks and @@ -87,16 +101,6 @@ access to internal read-only data of Unicode objects: subtype. This function always succeeds. -.. c:function:: int PyUnicode_READY(PyObject *unicode) - - Returns ``0``. This API is kept only for backward compatibility. - - .. versionadded:: 3.3 - - .. deprecated:: 3.10 - This API does nothing since Python 3.12. - - .. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *unicode) Return the length of the Unicode string, in code points. *unicode* has to be a @@ -149,12 +153,16 @@ access to internal read-only data of Unicode objects: .. c:function:: void PyUnicode_WRITE(int kind, void *data, \ Py_ssize_t index, Py_UCS4 value) - Write into a canonical representation *data* (as obtained with - :c:func:`PyUnicode_DATA`). This function performs no sanity checks, and is - intended for usage in loops. The caller should cache the *kind* value and - *data* pointer as obtained from other calls. *index* is the index in - the string (starts at 0) and *value* is the new code point value which should - be written to that location. + Write the code point *value* to the given zero-based *index* in a string. + + The *kind* value and *data* pointer must have been obtained from a + string using :c:func:`PyUnicode_KIND` and :c:func:`PyUnicode_DATA` + respectively. You must hold a reference to that string while calling + :c:func:`!PyUnicode_WRITE`. All requirements of + :c:func:`PyUnicode_WriteChar` also apply. + + The function performs no checks for any of its requirements, + and is intended for usage in loops. .. versionadded:: 3.3 @@ -196,6 +204,30 @@ access to internal read-only data of Unicode objects: is not ready. +.. c:function:: unsigned int PyUnicode_IS_ASCII(PyObject *unicode) + + Return true if the string only contains ASCII characters. + Equivalent to :py:meth:`str.isascii`. + + .. versionadded:: 3.2 + + +.. c:function:: Py_hash_t PyUnstable_Unicode_GET_CACHED_HASH(PyObject *str) + + If the hash of *str*, as returned by :c:func:`PyObject_Hash`, has been + cached and is immediately available, return it. + Otherwise, return ``-1`` *without* setting an exception. + + If *str* is not a string (that is, if ``PyUnicode_Check(obj)`` + is false), the behavior is undefined. + + This function never fails with an exception. + + Note that there are no guarantees on when an object's hash is cached, + and the (non-)existence of a cached hash does not imply that the string has + any other properties. + + Unicode Character Properties """""""""""""""""""""""""""" @@ -256,13 +288,8 @@ the Python configuration. .. c:function:: int Py_UNICODE_ISPRINTABLE(Py_UCS4 ch) - Return ``1`` or ``0`` depending on whether *ch* is a printable character. - Nonprintable characters are those characters defined in the Unicode character - database as "Other" or "Separator", excepting the ASCII space (0x20) which is - considered printable. (Note that printable characters in this context are - those which should not be escaped when :func:`repr` is invoked on a string. - It has no bearing on the handling of strings written to :data:`sys.stdout` or - :data:`sys.stderr`.) + Return ``1`` or ``0`` depending on whether *ch* is a printable character, + in the sense of :meth:`str.isprintable`. These APIs can be used for fast direct character conversions: @@ -315,12 +342,22 @@ These APIs can be used to work with surrogates: Check if *ch* is a low surrogate (``0xDC00 <= ch <= 0xDFFF``). +.. c:function:: Py_UCS4 Py_UNICODE_HIGH_SURROGATE(Py_UCS4 ch) + + Return the high UTF-16 surrogate (``0xD800`` to ``0xDBFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + +.. c:function:: Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) + + Return the low UTF-16 surrogate (``0xDC00`` to ``0xDFFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a - surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must - be in the range [0xDC00; 0xDFFF]. + surrogate pair. *high* must be in the range ``[0xD800; 0xDBFF]`` and *low* must + be in the range ``[0xDC00; 0xDFFF]``. Creating and accessing Unicode strings @@ -335,11 +372,29 @@ APIs: to be placed in the string. As an approximation, it can be rounded up to the nearest value in the sequence 127, 255, 65535, 1114111. - This is the recommended way to allocate a new Unicode object. Objects - created using this function are not resizable. - On error, set an exception and return ``NULL``. + After creation, the string can be filled by :c:func:`PyUnicode_WriteChar`, + :c:func:`PyUnicode_CopyCharacters`, :c:func:`PyUnicode_Fill`, + :c:func:`PyUnicode_WRITE` or similar. + Since strings are supposed to be immutable, take care to not “use” the + result while it is being modified. In particular, before it's filled + with its final contents, a string: + + - must not be hashed, + - must not be :c:func:`converted to UTF-8 `, + or another non-"canonical" representation, + - must not have its reference count changed, + - must not be shared with code that might do one of the above. + + This list is not exhaustive. Avoiding these uses is your responsibility; + Python does not always check these requirements. + + To avoid accidentally exposing a partially-written string object, prefer + using the :c:type:`PyUnicodeWriter` API, or one of the ``PyUnicode_From*`` + functions below. + + .. versionadded:: 3.3 @@ -594,6 +649,14 @@ APIs: Objects other than Unicode or its subtypes will cause a :exc:`TypeError`. +.. c:function:: PyObject* PyUnicode_FromOrdinal(int ordinal) + + Create a Unicode Object from the given Unicode code point *ordinal*. + + The ordinal must be in ``range(0x110000)``. A :exc:`ValueError` is + raised in the case it is not. + + .. c:function:: PyObject* PyUnicode_FromEncodedObject(PyObject *obj, \ const char *encoding, const char *errors) @@ -612,6 +675,43 @@ APIs: decref'ing the returned objects. +.. c:function:: void PyUnicode_Append(PyObject **p_left, PyObject *right) + + Append the string *right* to the end of *p_left*. + *p_left* must point to a :term:`strong reference` to a Unicode object; + :c:func:`!PyUnicode_Append` releases ("steals") this reference. + + On error, set *\*p_left* to ``NULL`` and set an exception. + + On success, set *\*p_left* to a new strong reference to the result. + + +.. c:function:: void PyUnicode_AppendAndDel(PyObject **p_left, PyObject *right) + + The function is similar to :c:func:`PyUnicode_Append`, with the only + difference being that it decrements the reference count of *right* by one. + + +.. c:function:: PyObject* PyUnicode_BuildEncodingMap(PyObject* string) + + Return a mapping suitable for decoding a custom single-byte encoding. + Given a Unicode string *string* of up to 256 characters representing an encoding + table, returns either a compact internal mapping object or a dictionary + mapping character ordinals to byte values. Raises a :exc:`TypeError` and + return ``NULL`` on invalid input. + + .. versionadded:: 3.2 + + +.. c:function:: const char* PyUnicode_GetDefaultEncoding(void) + + Return the name of the default string encoding, ``"utf-8"``. + See :func:`sys.getdefaultencoding`. + + The returned string does not need to be freed, and is valid + until interpreter shutdown. + + .. c:function:: Py_ssize_t PyUnicode_GetLength(PyObject *unicode) Return the length of the Unicode object, in code points. @@ -632,9 +732,27 @@ APIs: possible. Returns ``-1`` and sets an exception on error, otherwise returns the number of copied characters. + The string must not have been “used” yet. + See :c:func:`PyUnicode_New` for details. + .. versionadded:: 3.3 +.. c:function:: int PyUnicode_Resize(PyObject **unicode, Py_ssize_t length); + + Resize a Unicode object *\*unicode* to the new *length* in code points. + + Try to resize the string in place (which is usually faster than allocating + a new string and copying characters), or create a new string. + + *\*unicode* is modified to point to the new (resized) object and ``0`` is + returned on success. Otherwise, ``-1`` is returned and an exception is set, + and *\*unicode* is left untouched. + + The function doesn't check string content, the result may not be a + string in canonical representation. + + .. c:function:: Py_ssize_t PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, \ Py_ssize_t length, Py_UCS4 fill_char) @@ -644,6 +762,9 @@ APIs: Fail if *fill_char* is bigger than the string maximum character, or if the string has more than 1 reference. + The string must not have been “used” yet. + See :c:func:`PyUnicode_New` for details. + Return the number of written character, or return ``-1`` and raise an exception on error. @@ -653,15 +774,16 @@ APIs: .. c:function:: int PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, \ Py_UCS4 character) - Write a character to a string. The string must have been created through - :c:func:`PyUnicode_New`. Since Unicode strings are supposed to be immutable, - the string must not be shared, or have been hashed yet. + Write a *character* to the string *unicode* at the zero-based *index*. + Return ``0`` on success, ``-1`` on error with an exception set. This function checks that *unicode* is a Unicode object, that the index is - not out of bounds, and that the object can be modified safely (i.e. that it - its reference count is one). + not out of bounds, and that the object's reference count is one. + See :c:func:`PyUnicode_WRITE` for a version that skips these checks, + making them your responsibility. - Return ``0`` on success, ``-1`` on error with an exception set. + The string must not have been “used” yet. + See :c:func:`PyUnicode_New` for details. .. versionadded:: 3.3 @@ -786,16 +908,25 @@ Functions encoding to and decoding from the :term:`filesystem encoding and error handler` (:pep:`383` and :pep:`529`). To encode file names to :class:`bytes` during argument parsing, the ``"O&"`` -converter should be used, passing :c:func:`PyUnicode_FSConverter` as the +converter should be used, passing :c:func:`!PyUnicode_FSConverter` as the conversion function: .. c:function:: int PyUnicode_FSConverter(PyObject* obj, void* result) - ParseTuple converter: encode :class:`str` objects -- obtained directly or + :ref:`PyArg_Parse\* converter `: encode :class:`str` objects -- obtained directly or through the :class:`os.PathLike` interface -- to :class:`bytes` using :c:func:`PyUnicode_EncodeFSDefault`; :class:`bytes` objects are output as-is. - *result* must be a :c:expr:`PyBytesObject*` which must be released when it is - no longer used. + *result* must be an address of a C variable of type :c:expr:`PyObject*` + (or :c:expr:`PyBytesObject*`). + On success, set the variable to a new :term:`strong reference` to + a :ref:`bytes object ` which must be released + when it is no longer used and return a non-zero value + (:c:macro:`Py_CLEANUP_SUPPORTED`). + Embedded null bytes are not allowed in the result. + On failure, return ``0`` with an exception set. + + If *obj* is ``NULL``, the function releases a strong reference + stored in the variable referred by *result* and returns ``1``. .. versionadded:: 3.1 @@ -803,16 +934,26 @@ conversion function: Accepts a :term:`path-like object`. To decode file names to :class:`str` during argument parsing, the ``"O&"`` -converter should be used, passing :c:func:`PyUnicode_FSDecoder` as the +converter should be used, passing :c:func:`!PyUnicode_FSDecoder` as the conversion function: .. c:function:: int PyUnicode_FSDecoder(PyObject* obj, void* result) - ParseTuple converter: decode :class:`bytes` objects -- obtained either + :ref:`PyArg_Parse\* converter `: decode :class:`bytes` objects -- obtained either directly or indirectly through the :class:`os.PathLike` interface -- to :class:`str` using :c:func:`PyUnicode_DecodeFSDefaultAndSize`; :class:`str` - objects are output as-is. *result* must be a :c:expr:`PyUnicodeObject*` which - must be released when it is no longer used. + objects are output as-is. + *result* must be an address of a C variable of type :c:expr:`PyObject*` + (or :c:expr:`PyUnicodeObject*`). + On success, set the variable to a new :term:`strong reference` to + a :ref:`Unicode object ` which must be released + when it is no longer used and return a non-zero value + (:c:macro:`Py_CLEANUP_SUPPORTED`). + Embedded null characters are not allowed in the result. + On failure, return ``0`` with an exception set. + + If *obj* is ``NULL``, release the strong reference + to the object referred to by *result* and return ``1``. .. versionadded:: 3.2 @@ -949,6 +1090,17 @@ generic ones are documented for simplicity. Generic Codecs """""""""""""" +The following macro is provided: + + +.. c:macro:: Py_UNICODE_REPLACEMENT_CHARACTER + + The Unicode code point ``U+FFFD`` (replacement character). + + This Unicode character is used as the replacement character during + decoding if the *errors* argument is set to "replace". + + These are the generic codec APIs: @@ -1035,6 +1187,15 @@ These are the UTF-8 codec APIs: As :c:func:`PyUnicode_AsUTF8AndSize`, but does not store the size. + .. warning:: + + This function does not have any special behavior for + `null characters `_ embedded within + *unicode*. As a result, strings containing null characters will remain in the returned + string, which some C functions might interpret as the end of the string, leading to + truncation. If truncation is an issue, it is recommended to use :c:func:`PyUnicode_AsUTF8AndSize` + instead. + .. versionadded:: 3.3 .. versionchanged:: 3.7 @@ -1324,6 +1485,13 @@ the user settings on the machine running the codec. in *consumed*. +.. c:function:: PyObject* PyUnicode_DecodeCodePageStateful(int code_page, const char *str, \ + Py_ssize_t size, const char *errors, Py_ssize_t *consumed) + + Similar to :c:func:`PyUnicode_DecodeMBCSStateful`, except uses the code page + specified by *code_page*. + + .. c:function:: PyObject* PyUnicode_AsMBCSString(PyObject *unicode) Encode a Unicode object using MBCS and return the result as Python bytes @@ -1340,10 +1508,6 @@ the user settings on the machine running the codec. .. versionadded:: 3.3 -Methods & Slots -""""""""""""""" - - .. _unicodemethodsandslots: Methods and Slot Functions @@ -1368,6 +1532,20 @@ They all return ``NULL`` or ``-1`` if an exception occurs. separator. At most *maxsplit* splits will be done. If negative, no limit is set. Separators are not included in the resulting list. + On error, return ``NULL`` with an exception set. + + Equivalent to :py:meth:`str.split`. + + +.. c:function:: PyObject* PyUnicode_RSplit(PyObject *unicode, PyObject *sep, Py_ssize_t maxsplit) + + Similar to :c:func:`PyUnicode_Split`, but splitting will be done beginning + at the end of the string. + + On error, return ``NULL`` with an exception set. + + Equivalent to :py:meth:`str.rsplit`. + .. c:function:: PyObject* PyUnicode_Splitlines(PyObject *unicode, int keepends) @@ -1376,6 +1554,33 @@ They all return ``NULL`` or ``-1`` if an exception occurs. characters are not included in the resulting strings. +.. c:function:: PyObject* PyUnicode_Partition(PyObject *unicode, PyObject *sep) + + Split a Unicode string at the first occurrence of *sep*, and return + a 3-tuple containing the part before the separator, the separator itself, + and the part after the separator. If the separator is not found, + return a 3-tuple containing the string itself, followed by two empty strings. + + *sep* must not be empty. + + On error, return ``NULL`` with an exception set. + + Equivalent to :py:meth:`str.partition`. + + +.. c:function:: PyObject* PyUnicode_RPartition(PyObject *unicode, PyObject *sep) + + Similar to :c:func:`PyUnicode_Partition`, but split a Unicode string at the + last occurrence of *sep*. If the separator is not found, return a 3-tuple + containing two empty strings, followed by the string itself. + + *sep* must not be empty. + + On error, return ``NULL`` with an exception set. + + Equivalent to :py:meth:`str.rpartition`. + + .. c:function:: PyObject* PyUnicode_Join(PyObject *separator, PyObject *seq) Join a sequence of strings using the given *separator* and return the resulting @@ -1564,9 +1769,19 @@ They all return ``NULL`` or ``-1`` if an exception occurs. from user input, prefer calling :c:func:`PyUnicode_FromString` and :c:func:`PyUnicode_InternInPlace` directly. + +.. c:function:: unsigned int PyUnicode_CHECK_INTERNED(PyObject *str) + + Return a non-zero value if *str* is interned, zero if not. + The *str* argument must be a string; this is not checked. + This function always succeeds. + .. impl-detail:: - Strings interned this way are made :term:`immortal`. + A non-zero return value may carry additional information + about *how* the string is interned. + The meaning of such non-zero values, as well as each specific string's + intern-related details, may change between CPython versions. PyUnicodeWriter @@ -1588,6 +1803,11 @@ object. Create a Unicode writer instance. + *length* must be greater than or equal to ``0``. + + If *length* is greater than ``0``, preallocate an internal buffer of + *length* characters. + Set an exception and return ``NULL`` on error. .. c:function:: PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) @@ -1596,12 +1816,16 @@ object. Set an exception and return ``NULL`` on error. + The writer instance is invalid after this call. + .. c:function:: void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) Discard the internal Unicode buffer and destroy the writer instance. If *writer* is ``NULL``, no operation is performed. + The writer instance is invalid after this call. + .. c:function:: int PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) Write the single Unicode character *ch* into *writer*. @@ -1621,9 +1845,24 @@ object. See also :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. +.. c:function:: int PyUnicodeWriter_WriteASCII(PyUnicodeWriter *writer, const char *str, Py_ssize_t size) + + Write the ASCII string *str* into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + *str* must only contain ASCII characters. The behavior is undefined if + *str* contains non-ASCII characters. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + + .. versionadded:: 3.14 + .. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) - Writer the wide string *str* into *writer*. + Write the wide string *str* into *writer*. *size* is a number of wide characters. If *size* is equal to ``-1``, call ``wcslen(str)`` to get the string length. @@ -1680,8 +1919,8 @@ object. *size* is the string length in bytes. If *size* is equal to ``-1``, call ``strlen(str)`` to get the string length. - *errors* is an error handler name, such as ``"replace"``. If *errors* is - ``NULL``, use the strict error handler. + *errors* is an :ref:`error handler ` name, such as + ``"replace"``. If *errors* is ``NULL``, use the strict error handler. If *consumed* is not ``NULL``, set *\*consumed* to the number of decoded bytes on success. @@ -1692,3 +1931,49 @@ object. On error, set an exception, leave the writer unchanged, and return ``-1``. See also :c:func:`PyUnicodeWriter_WriteUTF8`. + +Deprecated API +^^^^^^^^^^^^^^ + +The following API is deprecated. + +.. c:type:: Py_UNICODE + + This is a typedef of :c:type:`wchar_t`, which is a 16-bit type or 32-bit type + depending on the platform. + Please use :c:type:`wchar_t` directly instead. + + .. versionchanged:: 3.3 + In previous versions, this was a 16-bit type or a 32-bit type depending on + whether you selected a "narrow" or "wide" Unicode version of Python at + build time. + + .. deprecated-removed:: 3.13 3.15 + + +.. c:function:: int PyUnicode_READY(PyObject *unicode) + + Do nothing and return ``0``. + This API is kept only for backward compatibility, but there are no plans + to remove it. + + .. versionadded:: 3.3 + + .. deprecated:: 3.10 + This API does nothing since Python 3.12. + Previously, this needed to be called for each string created using + the old API (:c:func:`!PyUnicode_FromUnicode` or similar). + + +.. c:function:: unsigned int PyUnicode_IS_READY(PyObject *unicode) + + Do nothing and return ``1``. + This API is kept only for backward compatibility, but there are no plans + to remove it. + + .. versionadded:: 3.3 + + .. deprecated:: 3.14 + This API does nothing since Python 3.12. + Previously, this could be called to check if + :c:func:`PyUnicode_READY` is necessary. diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 9f02bdb5896563..7eb9f0b54abd4e 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -13,8 +13,9 @@ the interpreter. Several of these functions accept a start symbol from the grammar as a parameter. The available start symbols are :c:data:`Py_eval_input`, -:c:data:`Py_file_input`, and :c:data:`Py_single_input`. These are described -following the functions which accept them as parameters. +:c:data:`Py_file_input`, :c:data:`Py_single_input`, and +:c:data:`Py_func_type_input`. These are described following the functions +which accept them as parameters. Note also that several of these functions take :c:expr:`FILE*` parameters. One particular issue which needs to be handled carefully is that the :c:type:`FILE` @@ -99,18 +100,12 @@ the same library that the Python runtime is using. Otherwise, Python may not handle script file with LF line ending correctly. -.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, - leaving *flags* set to ``NULL``. - - -.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) +.. c:function:: int PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) Read and execute a single statement from a file associated with an interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the - :term:`filesystem encoding and error handler`. + prompted using ``sys.ps1`` and ``sys.ps2``. *filename* must be a Python + :class:`str` object. Returns ``0`` when the input was executed successfully, ``-1`` if there was an exception, or an error code @@ -119,6 +114,19 @@ the same library that the Python runtime is using. :file:`Python.h`, so must be included specifically if needed.) +.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) + + This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, + leaving *flags* set to ``NULL``. + + +.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) + + Similar to :c:func:`PyRun_InteractiveOneObject`, but *filename* is a + :c:expr:`const char*`, which is decoded from the + :term:`filesystem encoding and error handler`. + + .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveLoopFlags` below, @@ -140,7 +148,7 @@ the same library that the Python runtime is using. interpreter prompt is about to become idle and wait for user input from the terminal. The return value is ignored. Overriding this hook can be used to integrate the interpreter's prompt with other - event loops, as done in the :file:`Modules/_tkinter.c` in the + event loops, as done in :file:`Modules/_tkinter.c` in the Python source code. .. versionchanged:: 3.12 @@ -183,7 +191,7 @@ the same library that the Python runtime is using. objects *globals* and *locals* with the compiler flags specified by *flags*. *globals* must be a dictionary; *locals* can be any object that implements the mapping protocol. The parameter *start* specifies - the start token that should be used to parse the source code. + the start symbol and must one of the :ref:`available start symbols `. Returns the result of executing the code as a Python object, or ``NULL`` if an exception was raised. @@ -231,9 +239,9 @@ the same library that the Python runtime is using. .. c:function:: PyObject* Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) Parse and compile the Python source code in *str*, returning the resulting code - object. The start token is given by *start*; this can be used to constrain the - code which can be compiled and should be :c:data:`Py_eval_input`, - :c:data:`Py_file_input`, or :c:data:`Py_single_input`. The filename specified by + object. The start symbol is given by *start*; this can be used to constrain the + code which can be compiled and should be :ref:`available start symbols + `. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or :exc:`SyntaxError` exception messages. This returns ``NULL`` if the code cannot be parsed or compiled. @@ -296,6 +304,57 @@ the same library that the Python runtime is using. true on success, false on failure. +.. c:struct:: PyCompilerFlags + + This is the structure used to hold compiler flags. In cases where code is only + being compiled, it is passed as ``int flags``, and in cases where code is being + executed, it is passed as ``PyCompilerFlags *flags``. In this case, ``from + __future__ import`` can modify *flags*. + + Whenever ``PyCompilerFlags *flags`` is ``NULL``, :c:member:`~PyCompilerFlags.cf_flags` is treated as + equal to ``0``, and any modification due to ``from __future__ import`` is + discarded. + + .. c:member:: int cf_flags + + Compiler flags. + + .. c:member:: int cf_feature_version + + *cf_feature_version* is the minor Python version. It should be + initialized to ``PY_MINOR_VERSION``. + + The field is ignored by default, it is used if and only if + ``PyCF_ONLY_AST`` flag is set in :c:member:`~PyCompilerFlags.cf_flags`. + + .. versionchanged:: 3.8 + Added *cf_feature_version* field. + + The available compiler flags are accessible as macros: + + .. c:namespace:: NULL + + .. c:macro:: PyCF_ALLOW_TOP_LEVEL_AWAIT + PyCF_ONLY_AST + PyCF_OPTIMIZED_AST + PyCF_TYPE_COMMENTS + + See :ref:`compiler flags ` in documentation of the + :py:mod:`!ast` Python module, which exports these constants under + the same names. + + The "``PyCF``" flags above can be combined with "``CO_FUTURE``" flags such + as :c:macro:`CO_FUTURE_ANNOTATIONS` to enable features normally + selectable using :ref:`future statements `. + See :ref:`c_codeobject_flags` for a complete list. + + +.. _start-symbols: + +Available start symbols +^^^^^^^^^^^^^^^^^^^^^^^ + + .. c:var:: int Py_eval_input .. index:: single: Py_CompileString (C function) @@ -322,34 +381,58 @@ the same library that the Python runtime is using. interpreter loop. -.. c:struct:: PyCompilerFlags +.. c:var:: int Py_func_type_input - This is the structure used to hold compiler flags. In cases where code is only - being compiled, it is passed as ``int flags``, and in cases where code is being - executed, it is passed as ``PyCompilerFlags *flags``. In this case, ``from - __future__ import`` can modify *flags*. + .. index:: single: Py_CompileString (C function) - Whenever ``PyCompilerFlags *flags`` is ``NULL``, :c:member:`~PyCompilerFlags.cf_flags` is treated as - equal to ``0``, and any modification due to ``from __future__ import`` is - discarded. + The start symbol from the Python grammar for a function type; for use with + :c:func:`Py_CompileString`. This is used to parse "signature type comments" + from :pep:`484`. - .. c:member:: int cf_flags + This requires the :c:macro:`PyCF_ONLY_AST` flag to be set. - Compiler flags. + .. seealso:: + * :py:class:`ast.FunctionType` + * :pep:`484` - .. c:member:: int cf_feature_version + .. versionadded:: 3.8 - *cf_feature_version* is the minor Python version. It should be - initialized to ``PY_MINOR_VERSION``. - The field is ignored by default, it is used if and only if - ``PyCF_ONLY_AST`` flag is set in :c:member:`~PyCompilerFlags.cf_flags`. +Stack Effects +^^^^^^^^^^^^^ - .. versionchanged:: 3.8 - Added *cf_feature_version* field. +.. seealso:: + :py:func:`dis.stack_effect` + + +.. c:macro:: PY_INVALID_STACK_EFFECT + + Sentinel value representing an invalid stack effect. + + This is currently equivalent to ``INT_MAX``. + + .. versionadded:: 3.8 + + +.. c:function:: int PyCompile_OpcodeStackEffect(int opcode, int oparg) + + Compute the stack effect of *opcode* with argument *oparg*. + + On success, this function returns the stack effect; on failure, this + returns :c:macro:`PY_INVALID_STACK_EFFECT`. + + .. versionadded:: 3.4 + + +.. c:function:: int PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) + + Similar to :c:func:`PyCompile_OpcodeStackEffect`, but don't include the + stack effect of jumping if *jump* is zero. + If *jump* is ``0``, this will not include the stack effect of jumping, but + if *jump* is ``1`` or ``-1``, this will include it. -.. c:var:: int CO_FUTURE_DIVISION + On success, this function returns the stack effect; on failure, this + returns :c:macro:`PY_INVALID_STACK_EFFECT`. - This bit can be set in *flags* to cause division operator ``/`` to be - interpreted as "true division" according to :pep:`238`. + .. versionadded:: 3.8 diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 8f233e16fb17cf..db6ae0a9d4ea3d 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -19,7 +19,14 @@ as much as it can. .. c:function:: int PyWeakref_CheckRef(PyObject *ob) - Return non-zero if *ob* is a reference object. This function always succeeds. + Return non-zero if *ob* is a reference object or a subclass of the reference + type. This function always succeeds. + + +.. c:function:: int PyWeakref_CheckRefExact(PyObject *ob) + + Return non-zero if *ob* is a reference object, but not a subclass of the + reference type. This function always succeeds. .. c:function:: int PyWeakref_CheckProxy(PyObject *ob) @@ -38,6 +45,10 @@ as much as it can. weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) @@ -50,6 +61,10 @@ as much as it can. is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) @@ -64,28 +79,13 @@ as much as it can. .. versionadded:: 3.13 -.. c:function:: PyObject* PyWeakref_GetObject(PyObject *ref) - - Return a :term:`borrowed reference` to the referenced object from a weak - reference, *ref*. If the referent is no longer live, returns ``Py_None``. - - .. note:: - - This function returns a :term:`borrowed reference` to the referenced object. - This means that you should always call :c:func:`Py_INCREF` on the object - except when it cannot be destroyed before the last usage of the borrowed - reference. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyWeakref_GetRef` instead. - - -.. c:function:: PyObject* PyWeakref_GET_OBJECT(PyObject *ref) +.. c:function:: int PyWeakref_IsDead(PyObject *ref) - Similar to :c:func:`PyWeakref_GetObject`, but does no error checking. + Test if the weak reference *ref* is dead. Returns 1 if the reference is + dead, 0 if it is alive, and -1 with an error set if *ref* is not a weak + reference object. - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyWeakref_GetRef` instead. + .. versionadded:: 3.14 .. c:function:: void PyObject_ClearWeakRefs(PyObject *object) diff --git a/Doc/conf.py b/Doc/conf.py index 7ee3c91581345d..f6efc5ff22a5e1 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -6,27 +6,34 @@ # The contents of this file are pickled, so don't put values in the namespace # that aren't pickleable (module imports are okay, they're removed automatically). -import importlib import os import sys -import time - -import sphinx +from importlib import import_module +from importlib.util import find_spec +# Make our custom extensions available to Sphinx sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) +# Python specific content from Doc/Tools/extensions/pyspecific.py from pyspecific import SOURCE_URI # General configuration # --------------------- +# Our custom Sphinx extensions are found in Doc/Tools/extensions/ extensions = [ 'audit_events', 'availability', 'c_annotations', + 'changes', 'glossary_search', + 'grammar_snippet', + 'implementation_detail', + 'issue_role', 'lexers', + 'misc_news', + 'pydoc_topics', 'pyspecific', 'sphinx.ext.coverage', 'sphinx.ext.doctest', @@ -34,19 +41,17 @@ ] # Skip if downstream redistributors haven't installed them -try: - import notfound.extension # noqa: F401 -except ImportError: - pass -else: - extensions.append('notfound.extension') -try: - import sphinxext.opengraph # noqa: F401 -except ImportError: - pass -else: - extensions.append('sphinxext.opengraph') - +_OPTIONAL_EXTENSIONS = ( + 'notfound.extension', + 'sphinxext.opengraph', +) +for optional_ext in _OPTIONAL_EXTENSIONS: + try: + if find_spec(optional_ext) is not None: + extensions.append(optional_ext) + except (ImportError, ValueError): + pass +del _OPTIONAL_EXTENSIONS doctest_global_setup = ''' try: @@ -54,7 +59,7 @@ except ImportError: _tkinter = None # Treat warnings as errors, done here to prevent warnings in Sphinx code from -# causing spurious test failures. +# causing spurious CPython test failures. import warnings warnings.simplefilter('error') del warnings @@ -64,41 +69,49 @@ # General substitutions. project = 'Python' -if sphinx.version_info[:2] >= (8, 1): - copyright = "2001-%Y, Python Software Foundation" -else: - copyright = f"2001-{time.strftime('%Y')}, Python Software Foundation" +copyright = "2001 Python Software Foundation" # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. # See Doc/tools/extensions/patchlevel.py -version, release = importlib.import_module('patchlevel').get_version_info() +version, release = import_module('patchlevel').get_version_info() rst_epilog = f""" .. |python_version_literal| replace:: ``Python {version}`` .. |python_x_dot_y_literal| replace:: ``python{version}`` +.. |python_x_dot_y_t_literal| replace:: ``python{version}t`` +.. |python_x_dot_y_t_literal_config| replace:: ``python{version}t-config`` +.. |x_dot_y_b2_literal| replace:: ``{version}.0b2`` +.. |applications_python_version_literal| replace:: ``/Applications/Python {version}/`` .. |usr_local_bin_python_x_dot_y_literal| replace:: ``/usr/local/bin/python{version}`` + +.. Apparently this how you hack together a formatted link: + (https://www.docutils.org/docs/ref/rst/directives.html#replacement-text) +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ """ -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: +# There are two options for replacing |today|. Either, you set today to some +# non-false value and use it. today = '' -# Else, today_fmt is used as the format for a strftime call. +# Or else, today_fmt is used as the format for a strftime call. today_fmt = '%B %d, %Y' # By default, highlight as Python 3. highlight_language = 'python3' # Minimum version of sphinx required -needs_sphinx = '7.2.6' +# Keep this version in sync with ``Doc/requirements.txt``. +needs_sphinx = '8.2.0' # Create table of contents entries for domain objects (e.g. functions, classes, # attributes, etc.). Default is True. -toc_object_entries = True -toc_object_entries_show_parents = 'hide' +toc_object_entries = False # Ignore any .rst files in the includes/ directory; -# they're embedded in pages but not rendered individually. +# they're embedded in pages but not rendered as individual pages. # Ignore any .rst files in the venv/ directory. exclude_patterns = ['includes/*.rst', 'venv/*', 'README.rst'] venvdir = os.getenv('VENVDIR') @@ -195,6 +208,7 @@ ('envvar', 'LC_TIME'), ('envvar', 'LINES'), ('envvar', 'LOGNAME'), + ('envvar', 'MANPAGER'), ('envvar', 'PAGER'), ('envvar', 'PATH'), ('envvar', 'PATHEXT'), @@ -207,99 +221,18 @@ ('envvar', 'USER'), ('envvar', 'USERNAME'), ('envvar', 'USERPROFILE'), - # Deprecated function that was never documented: - ('py:func', 'getargspec'), - ('py:func', 'inspect.getargspec'), - # Undocumented modules that users shouldn't have to worry about - # (implementation details of `os.path`): - ('py:mod', 'ntpath'), - ('py:mod', 'posixpath'), ] # Temporary undocumented names. # In future this list must be empty. nitpick_ignore += [ - # C API: Standard Python exception classes - ('c:data', 'PyExc_ArithmeticError'), - ('c:data', 'PyExc_AssertionError'), - ('c:data', 'PyExc_AttributeError'), - ('c:data', 'PyExc_BaseException'), - ('c:data', 'PyExc_BlockingIOError'), - ('c:data', 'PyExc_BrokenPipeError'), - ('c:data', 'PyExc_BufferError'), - ('c:data', 'PyExc_ChildProcessError'), - ('c:data', 'PyExc_ConnectionAbortedError'), - ('c:data', 'PyExc_ConnectionError'), - ('c:data', 'PyExc_ConnectionRefusedError'), - ('c:data', 'PyExc_ConnectionResetError'), - ('c:data', 'PyExc_EOFError'), - ('c:data', 'PyExc_Exception'), - ('c:data', 'PyExc_FileExistsError'), - ('c:data', 'PyExc_FileNotFoundError'), - ('c:data', 'PyExc_FloatingPointError'), - ('c:data', 'PyExc_GeneratorExit'), - ('c:data', 'PyExc_ImportError'), - ('c:data', 'PyExc_IndentationError'), - ('c:data', 'PyExc_IndexError'), - ('c:data', 'PyExc_InterruptedError'), - ('c:data', 'PyExc_IsADirectoryError'), - ('c:data', 'PyExc_KeyboardInterrupt'), - ('c:data', 'PyExc_KeyError'), - ('c:data', 'PyExc_LookupError'), - ('c:data', 'PyExc_MemoryError'), - ('c:data', 'PyExc_ModuleNotFoundError'), - ('c:data', 'PyExc_NameError'), - ('c:data', 'PyExc_NotADirectoryError'), - ('c:data', 'PyExc_NotImplementedError'), - ('c:data', 'PyExc_OSError'), - ('c:data', 'PyExc_OverflowError'), - ('c:data', 'PyExc_PermissionError'), - ('c:data', 'PyExc_ProcessLookupError'), - ('c:data', 'PyExc_PythonFinalizationError'), - ('c:data', 'PyExc_RecursionError'), - ('c:data', 'PyExc_ReferenceError'), - ('c:data', 'PyExc_RuntimeError'), - ('c:data', 'PyExc_StopAsyncIteration'), - ('c:data', 'PyExc_StopIteration'), - ('c:data', 'PyExc_SyntaxError'), - ('c:data', 'PyExc_SystemError'), - ('c:data', 'PyExc_SystemExit'), - ('c:data', 'PyExc_TabError'), - ('c:data', 'PyExc_TimeoutError'), - ('c:data', 'PyExc_TypeError'), - ('c:data', 'PyExc_UnboundLocalError'), - ('c:data', 'PyExc_UnicodeDecodeError'), - ('c:data', 'PyExc_UnicodeEncodeError'), - ('c:data', 'PyExc_UnicodeError'), - ('c:data', 'PyExc_UnicodeTranslateError'), - ('c:data', 'PyExc_ValueError'), - ('c:data', 'PyExc_ZeroDivisionError'), - # C API: Standard Python warning classes - ('c:data', 'PyExc_BytesWarning'), - ('c:data', 'PyExc_DeprecationWarning'), - ('c:data', 'PyExc_FutureWarning'), - ('c:data', 'PyExc_ImportWarning'), - ('c:data', 'PyExc_PendingDeprecationWarning'), - ('c:data', 'PyExc_ResourceWarning'), - ('c:data', 'PyExc_RuntimeWarning'), - ('c:data', 'PyExc_SyntaxWarning'), - ('c:data', 'PyExc_UnicodeWarning'), - ('c:data', 'PyExc_UserWarning'), - ('c:data', 'PyExc_Warning'), - # Undocumented public C macros - ('c:macro', 'Py_BUILD_ASSERT'), - ('c:macro', 'Py_BUILD_ASSERT_EXPR'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. ('py:meth', '_SubParsersAction.add_parser'), # Attributes/methods/etc. that definitely should be documented better, # but are deferred for now: - ('py:attr', '__annotations__'), - ('py:meth', '__missing__'), ('py:attr', '__wrapped__'), - ('py:attr', 'decimal.Context.clamp'), - ('py:meth', 'index'), # list.index, tuple.index, etc. ] # gh-106948: Copy standard C types declared in the "c:type" domain and C @@ -329,8 +262,9 @@ # Options for HTML output # ----------------------- -# Use our custom theme. +# Use our custom theme: https://github.com/python/python-docs-theme html_theme = 'python_docs_theme' +# Location of overrides for theme templates and static files html_theme_path = ['tools'] html_theme_options = { 'collapsiblesidebar': True, @@ -368,15 +302,9 @@ # This 'Last updated on:' timestamp is inserted at the bottom of every page. html_last_updated_fmt = '%b %d, %Y (%H:%M UTC)' -if sphinx.version_info[:2] >= (8, 1): - html_last_updated_use_utc = True -else: - html_time = int(os.environ.get('SOURCE_DATE_EPOCH', time.time())) - html_last_updated_fmt = time.strftime( - html_last_updated_fmt, time.gmtime(html_time) - ) +html_last_updated_use_utc = True -# Path to find HTML templates. +# Path to find HTML templates to override theme templates_path = ['tools/templates'] # Custom sidebar templates, filenames relative to this file. @@ -428,11 +356,12 @@ 'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). 'pointsize': '10pt', + 'maxlistdepth': '8', # See https://github.com/python/cpython/issues/139588 } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). -_stdauthor = 'Guido van Rossum and the Python development team' +_stdauthor = 'The Python development team' latex_documents = [ ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), ( @@ -507,11 +436,36 @@ epub_author = 'Python Documentation Authors' epub_publisher = 'Python Software Foundation' +epub_exclude_files = ( + 'index.xhtml', + 'download.xhtml', + '_static/tachyon-example-flamegraph.html', + '_static/tachyon-example-heatmap.html', +) # index pages are not valid xhtml # https://github.com/sphinx-doc/sphinx/issues/12359 epub_use_index = False +# translation tag +# --------------- + +language_code = None +for arg in sys.argv: + if arg.startswith('language='): + language_code = arg.split('=', 1)[1] + +if language_code: + tags.add('translation') # noqa: F821 + + rst_epilog += f"""\ +.. _TRANSLATION_REPO: https://github.com/python/python-docs-{language_code.replace("_", "-").lower()} +""" # noqa: F821 +else: + rst_epilog += """\ +.. _TRANSLATION_REPO: https://github.com/python +""" + # Options for the coverage checker # -------------------------------- @@ -558,8 +512,6 @@ r'https://github.com/python/cpython/tree/.*': 'https://github.com/python/cpython/blob/.*', # Intentional HTTP use at Misc/NEWS.d/3.5.0a1.rst r'http://www.python.org/$': 'https://www.python.org/$', - # Used in license page, keep as is - r'https://www.zope.org/': r'https://www.zope.dev/', # Microsoft's redirects to learn.microsoft.com r'https://msdn.microsoft.com/.*': 'https://learn.microsoft.com/.*', r'https://docs.microsoft.com/.*': 'https://learn.microsoft.com/.*', @@ -611,18 +563,8 @@ } extlinks_detect_hardcoded_links = True -if sphinx.version_info[:2] < (8, 1): - # Sphinx 8.1 has in-built CVE and CWE roles. - extlinks |= { - "cve": ( - "https://www.cve.org/CVERecord?id=CVE-%s", - "CVE-%s", - ), - "cwe": ("https://cwe.mitre.org/data/definitions/%s.html", "CWE-%s"), - } - -# Options for c_annotations -# ------------------------- +# Options for c_annotations extension +# ----------------------------------- # Relative filename of the data files refcount_file = 'data/refcounts.dat' @@ -631,11 +573,17 @@ # Options for sphinxext-opengraph # ------------------------------- -ogp_site_url = 'https://docs.python.org/3/' +ogp_canonical_url = 'https://docs.python.org/3/' ogp_site_name = 'Python documentation' -ogp_image = '_static/og-image.png' -ogp_custom_meta_tags = [ - '', - '', - '', -] +ogp_social_cards = { # Used when matplotlib is installed + 'image': '_static/og-image.png', + 'line_color': '#3776ab', +} +ogp_custom_meta_tags = ('',) +if 'create-social-cards' not in tags: # noqa: F821 + # Define a static preview image when not creating social cards + ogp_image = '_static/og-image.png' + ogp_custom_meta_tags += ( + '', + '', + ) diff --git a/Doc/constraints.txt b/Doc/constraints.txt index 26ac1862dbac0b..29cd4be1d3c8db 100644 --- a/Doc/constraints.txt +++ b/Doc/constraints.txt @@ -13,14 +13,12 @@ packaging<25 Pygments<3 requests<3 snowballstemmer<3 -# keep lower-bounds until Sphinx 8.1 is released -# https://github.com/sphinx-doc/sphinx/pull/12756 -sphinxcontrib-applehelp>=1.0.7,<3 -sphinxcontrib-devhelp>=1.0.6,<3 -sphinxcontrib-htmlhelp>=2.0.6,<3 -sphinxcontrib-jsmath>=1.0.1,<2 -sphinxcontrib-qthelp>=1.0.6,<3 -sphinxcontrib-serializinghtml>=1.1.9,<3 +sphinxcontrib-applehelp<3 +sphinxcontrib-devhelp<3 +sphinxcontrib-htmlhelp<3 +sphinxcontrib-jsmath<2 +sphinxcontrib-qthelp<3 +sphinxcontrib-serializinghtml<3 # Direct dependencies of Jinja2 (Jinja is a dependency of Sphinx, see above) MarkupSafe<3 diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 8629ed1fc38009..9210d5f50ed841 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2024 Python Software Foundation. All rights reserved. +Copyright © 2001 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 65d48f8bea7de8..64399f6ab1ff26 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -180,7 +180,7 @@ PyCapsule_IsValid:const char*:name:: PyCapsule_New:PyObject*::+1: PyCapsule_New:void*:pointer:: PyCapsule_New:const char *:name:: -PyCapsule_New::void (* destructor)(PyObject* ):: +PyCapsule_New:void (*)(PyObject *):destructor:: PyCapsule_SetContext:int::: PyCapsule_SetContext:PyObject*:self:0: @@ -349,11 +349,11 @@ PyComplex_CheckExact:int::: PyComplex_CheckExact:PyObject*:p:0: PyComplex_FromCComplex:PyObject*::+1: -PyComplex_FromCComplex::Py_complex v:: +PyComplex_FromCComplex:Py_complex:v:: PyComplex_FromDoubles:PyObject*::+1: -PyComplex_FromDoubles::double real:: -PyComplex_FromDoubles::double imag:: +PyComplex_FromDoubles:double:real:: +PyComplex_FromDoubles:double:imag:: PyComplex_ImagAsDouble:double::: PyComplex_ImagAsDouble:PyObject*:op:0: @@ -620,7 +620,9 @@ PyErr_GetExcInfo:PyObject**:pvalue:+1: PyErr_GetExcInfo:PyObject**:ptraceback:+1: PyErr_GetRaisedException:PyObject*::+1: -PyErr_SetRaisedException:::: + +PyErr_SetRaisedException:void::: +PyErr_SetRaisedException:PyObject *:exc:0:stolen PyErr_GivenExceptionMatches:int::: PyErr_GivenExceptionMatches:PyObject*:given:0: @@ -640,9 +642,9 @@ PyErr_NewExceptionWithDoc:PyObject*:dict:0: PyErr_NoMemory:PyObject*::null: PyErr_NormalizeException:void::: -PyErr_NormalizeException:PyObject**:exc::??? -PyErr_NormalizeException:PyObject**:val::??? -PyErr_NormalizeException:PyObject**:tb::??? +PyErr_NormalizeException:PyObject**:exc:+1:??? +PyErr_NormalizeException:PyObject**:val:+1:??? +PyErr_NormalizeException:PyObject**:tb:+1:??? PyErr_Occurred:PyObject*::0: @@ -961,21 +963,45 @@ PyFunction_Check:PyObject*:o:0: PyFunction_GetAnnotations:PyObject*::0: PyFunction_GetAnnotations:PyObject*:op:0: +PyFunction_GET_ANNOTATIONS:PyObject*::0: +PyFunction_GET_ANNOTATIONS:PyObject*:op:0: + PyFunction_GetClosure:PyObject*::0: PyFunction_GetClosure:PyObject*:op:0: +PyFunction_GET_CLOSURE:PyObject*::0: +PyFunction_GET_CLOSURE:PyObject*:op:0: + PyFunction_GetCode:PyObject*::0: PyFunction_GetCode:PyObject*:op:0: +PyFunction_GET_CODE:PyObject*::0: +PyFunction_GET_CODE:PyObject*:op:0: + PyFunction_GetDefaults:PyObject*::0: PyFunction_GetDefaults:PyObject*:op:0: +PyFunction_GET_DEFAULTS:PyObject*::0: +PyFunction_GET_DEFAULTS:PyObject*:op:0: + +PyFunction_GetKwDefaults:PyObject*::0: +PyFunction_GetKwDefaults:PyObject*:op:0: + +PyFunction_GET_KW_DEFAULTS:PyObject*::0: +PyFunction_GET_KW_DEFAULTS:PyObject*:op:0: + PyFunction_GetGlobals:PyObject*::0: PyFunction_GetGlobals:PyObject*:op:0: +PyFunction_GET_GLOBALS:PyObject*::0: +PyFunction_GET_GLOBALS:PyObject*:op:0: + PyFunction_GetModule:PyObject*::0: PyFunction_GetModule:PyObject*:op:0: +PyFunction_GET_MODULE:PyObject*::0: +PyFunction_GET_MODULE:PyObject*:op:0: + PyFunction_New:PyObject*::+1: PyFunction_New:PyObject*:code:+1: PyFunction_New:PyObject*:globals:+1: @@ -1091,9 +1117,6 @@ PyImport_ImportModuleLevelObject:PyObject*:locals:0:??? PyImport_ImportModuleLevelObject:PyObject*:fromlist:0:??? PyImport_ImportModuleLevelObject:int:level:: -PyImport_ImportModuleNoBlock:PyObject*::+1: -PyImport_ImportModuleNoBlock:const char*:name:: - PyImport_ReloadModule:PyObject*::+1: PyImport_ReloadModule:PyObject*:m:0: @@ -1118,6 +1141,9 @@ PyInterpreterState_Clear:PyInterpreterState*:interp:: PyInterpreterState_Delete:void::: PyInterpreterState_Delete:PyInterpreterState*:interp:: +PyInterpreterState_GetDict:PyObject*::0: +PyInterpreterState_GetDict:PyInterpreterState*:interp:: + PyInterpreterState_GetID:int64_t::: PyInterpreterState_GetID:PyInterpreterState*:interp:: @@ -1284,6 +1310,26 @@ PyLong_FromUnsignedLong:unsignedlong:v:: PyLong_FromVoidPtr:PyObject*::+1: PyLong_FromVoidPtr:void*:p:: +PyLong_IsPositive:int::: +PyLong_IsPositive:PyObject*:obj:0: + +PyLong_IsNegative:int::: +PyLong_IsNegative:PyObject*:obj:0: + +PyLong_IsZero:int::: +PyLong_IsZero:PyObject*:obj:0: + +PyLong_GetSign:int::: +PyLong_GetSign:PyObject*:v:0: +PyLong_GetSign:int*:sign:: + +PyLong_Export:int::: +PyLong_Export:PyObject*:obj:0: +PyLong_Export:PyLongExport*:export_long:: + +PyLongWriter_Finish:PyObject*::+1: +PyLongWriter_Finish:PyLongWriter*:writer:: + PyMapping_Check:int::: PyMapping_Check:PyObject*:o:0: @@ -1301,7 +1347,7 @@ PyMapping_GetItemString:const char*:key:: PyMapping_HasKey:int::: PyMapping_HasKey:PyObject*:o:0: -PyMapping_HasKey:PyObject*:key:: +PyMapping_HasKey:PyObject*:key:0: PyMapping_HasKeyString:int::: PyMapping_HasKeyString:PyObject*:o:0: @@ -1426,6 +1472,9 @@ PyModule_Create2:PyObject*::+1: PyModule_Create2:PyModuleDef*:def:: PyModule_Create2:int:module_api_version:: +PyModule_Exec:int::: +PyModule_ExecDef:PyObject*:module:0: + PyModule_ExecDef:int::: PyModule_ExecDef:PyObject*:module:0: PyModule_ExecDef:PyModuleDef*:def:: @@ -1439,6 +1488,10 @@ PyModule_FromDefAndSpec2:PyModuleDef*:def:: PyModule_FromDefAndSpec2:PyObject*:spec:0: PyModule_FromDefAndSpec2:int:module_api_version:: +PyModule_FromSlotsAndSpec:PyObject*::+1: +PyModule_FromSlotsAndSpec:const PyModuleDef_Slot *:slots:: +PyModule_FromSlotsAndSpec:PyObject*:spec:0: + PyModule_GetDef:PyModuleDef*::0: PyModule_GetDef:PyObject*:module:0: @@ -1460,8 +1513,16 @@ PyModule_GetNameObject:PyObject*:module:0: PyModule_GetState:void*::: PyModule_GetState:PyObject*:module:0: +PyModule_GetStateSize:int::: +PyModule_GetStateSize:PyObject*:module:0: +PyModule_GetToken:Py_ssize_t**:result:: + +PyModule_GetToken:int::: +PyModule_GetToken:PyObject*:module:0: +PyModule_GetToken:void**:result:: + PyModule_New:PyObject*::+1: -PyModule_New::char* name:: +PyModule_New:char*:name:: PyModule_NewObject:PyObject*::+1: PyModule_NewObject:PyObject*:name:+1: @@ -1470,9 +1531,6 @@ PyModule_SetDocString:int::: PyModule_SetDocString:PyObject*:module:0: PyModule_SetDocString:const char*:docstring:: -PyModuleDef_Init:PyObject*::0: -PyModuleDef_Init:PyModuleDef*:def:0: - PyNumber_Absolute:PyObject*::+1: PyNumber_Absolute:PyObject*:o:0: @@ -1834,6 +1892,9 @@ PyObject_RichCompareBool:PyObject*:o1:0: PyObject_RichCompareBool:PyObject*:o2:0: PyObject_RichCompareBool:int:opid:: +PyObject_SelfIter:PyObject*::+1: +PyObject_SelfIter:PyObject*:obj:0: + PyObject_SetAttr:int::: PyObject_SetAttr:PyObject*:o:0: PyObject_SetAttr:PyObject*:attr_name:0: @@ -1971,10 +2032,10 @@ PyRun_StringFlags:PyObject*:locals:0: PyRun_StringFlags:PyCompilerFlags*:flags:: PySeqIter_Check:int::: -PySeqIter_Check::op:: +PySeqIter_Check:PyObject *:op:0: PySeqIter_New:PyObject*::+1: -PySeqIter_New:PyObject*:seq:: +PySeqIter_New:PyObject*:seq:0: PySequence_Check:int::: PySequence_Check:PyObject*:o:0: @@ -2366,6 +2427,14 @@ PyType_GetFlags:PyTypeObject*:type:0: PyType_GetName:PyObject*::+1: PyType_GetName:PyTypeObject*:type:0: +PyType_GetModuleByToken:PyObject*::+1: +PyType_GetModuleByToken:PyTypeObject*:type:0: +PyType_GetModuleByToken:PyModuleDef*:def:: + +PyType_GetModuleByDef:PyObject*::0: +PyType_GetModuleByDef:PyTypeObject*:type:0: +PyType_GetModuleByDef:PyModuleDef*:def:: + PyType_GetQualName:PyObject*::+1: PyType_GetQualName:PyTypeObject*:type:0: @@ -2408,7 +2477,7 @@ PyUnicode_GET_LENGTH:PyObject*:o:0: PyUnicode_KIND:int::: PyUnicode_KIND:PyObject*:o:0: -PyUnicode_MAX_CHAR_VALUE:::: +PyUnicode_MAX_CHAR_VALUE:Py_UCS4::: PyUnicode_MAX_CHAR_VALUE:PyObject*:o:0: Py_UNICODE_ISALNUM:int::: @@ -2475,7 +2544,7 @@ PyUnicode_FromWideChar:const wchar_t*:w:: PyUnicode_FromWideChar:Py_ssize_t:size:: PyUnicode_AsWideChar:Py_ssize_t::: -PyUnicode_AsWideChar:PyObject*:*unicode:0: +PyUnicode_AsWideChar:PyObject*:unicode:0: PyUnicode_AsWideChar:wchar_t*:w:: PyUnicode_AsWideChar:Py_ssize_t:size:: @@ -2528,7 +2597,7 @@ PyUnicode_AsUTF8String:PyObject*:unicode:0: PyUnicode_AsUTF8AndSize:const char*::: PyUnicode_AsUTF8AndSize:PyObject*:unicode:0: -PyUnicode_AsUTF8AndSize:Py_ssize_t*:size:0: +PyUnicode_AsUTF8AndSize:Py_ssize_t*:size:: PyUnicode_AsUTF8:const char*::: PyUnicode_AsUTF8:PyObject*:unicode:0: @@ -2611,6 +2680,13 @@ PyUnicode_DecodeMBCSStateful:Py_ssize_t:size:: PyUnicode_DecodeMBCSStateful:const char*:errors:: PyUnicode_DecodeMBCSStateful:Py_ssize_t*:consumed:: +PyUnicode_DecodeCodePageStateful:PyObject*::+1: +PyUnicode_DecodeCodePageStateful:int:code_page:: +PyUnicode_DecodeCodePageStateful:const char*:s:: +PyUnicode_DecodeCodePageStateful:Py_ssize_t:size:: +PyUnicode_DecodeCodePageStateful:const char*:errors:: +PyUnicode_DecodeCodePageStateful:Py_ssize_t*:consumed:: + PyUnicode_EncodeCodePage:PyObject*::+1: PyUnicode_EncodeCodePage:int:code_page:: PyUnicode_EncodeCodePage:PyObject*:unicode:0: @@ -2623,13 +2699,26 @@ PyUnicode_Concat:PyObject*::+1: PyUnicode_Concat:PyObject*:left:0: PyUnicode_Concat:PyObject*:right:0: +PyUnicode_Partition:PyObject*::+1: +PyUnicode_Partition:PyObject*:unicode:0: +PyUnicode_Partition:PyObject*:sep:0: + +PyUnicode_RPartition:PyObject*::+1: +PyUnicode_RPartition:PyObject*:unicode:0: +PyUnicode_RPartition:PyObject*:sep:0: + +PyUnicode_RSplit:PyObject*::+1: +PyUnicode_RSplit:PyObject*:unicode:0: +PyUnicode_RSplit:PyObject*:sep:0: +PyUnicode_RSplit:Py_ssize_t:maxsplit:: + PyUnicode_Split:PyObject*::+1: -PyUnicode_Split:PyObject*:left:0: -PyUnicode_Split:PyObject*:right:0: +PyUnicode_Split:PyObject*:unicode:0: +PyUnicode_Split:PyObject*:sep:0: PyUnicode_Split:Py_ssize_t:maxsplit:: PyUnicode_Splitlines:PyObject*::+1: -PyUnicode_Splitlines:PyObject*:s:0: +PyUnicode_Splitlines:PyObject*:unicode:0: PyUnicode_Splitlines:int:keepend:: PyUnicode_Translate:PyObject*::+1: @@ -2725,6 +2814,23 @@ PyUnicode_FromFormatV:PyObject*::+1: PyUnicode_FromFormatV:const char*:format:: PyUnicode_FromFormatV:va_list:args:: +PyUnicode_FromOrdinal:PyObject*::+1: +PyUnicode_FromOrdinal:int:ordinal:: + +PyUnicode_Append:void::: +PyUnicode_Append:PyObject**:p_left:0: +PyUnicode_Append:PyObject*:right:: + +PyUnicode_AppendAndDel:void::: +PyUnicode_AppendAndDel:PyObject**:p_left:0: +PyUnicode_AppendAndDel:PyObject*:right:-1: + +PyUnicode_BuildEncodingMap:PyObject*::+1: +PyUnicode_BuildEncodingMap:PyObject*:string::: + +PyUnicode_GetDefaultEncoding:const char*::: +PyUnicode_GetDefaultEncoding::void:: + PyUnicode_GetLength:Py_ssize_t::: PyUnicode_GetLength:PyObject*:unicode:0: @@ -2735,6 +2841,10 @@ PyUnicode_CopyCharacters:PyObject*:from:0: PyUnicode_CopyCharacters:Py_ssize_t:from_start:: PyUnicode_CopyCharacters:Py_ssize_t:how_many:: +PyUnicode_Resize:int::: +PyUnicode_Resize:PyObject**:unicode:0: +PyUnicode_Resize:Py_ssize_t:length:: + PyUnicode_Fill:Py_ssize_t::: PyUnicode_Fill:PyObject*:unicode:0: PyUnicode_Fill:Py_ssize_t:start:: @@ -2851,19 +2961,13 @@ PyUnicodeDecodeError_SetStart:PyObject*:exc:0: PyUnicodeDecodeError_SetStart:Py_ssize_t:start:: PyWeakref_Check:int::: -PyWeakref_Check:PyObject*:ob:: +PyWeakref_Check:PyObject*:ob:0: PyWeakref_CheckProxy:int::: -PyWeakref_CheckProxy:PyObject*:ob:: +PyWeakref_CheckProxy:PyObject*:ob:0: PyWeakref_CheckRef:int::: -PyWeakref_CheckRef:PyObject*:ob:: - -PyWeakref_GET_OBJECT:PyObject*::0: -PyWeakref_GET_OBJECT:PyObject*:ref:0: - -PyWeakref_GetObject:PyObject*::0: -PyWeakref_GetObject:PyObject*:ref:0: +PyWeakref_CheckRef:PyObject*:ob:0: PyWeakref_GetRef:int::: PyWeakref_GetRef:PyObject*:ref:0: @@ -2944,18 +3048,8 @@ Py_GetCompiler:const char*::: Py_GetCopyright:const char*::: -Py_GetExecPrefix:wchar_t*::: - -Py_GetPath:wchar_t*::: - Py_GetPlatform:const char*::: -Py_GetPrefix:wchar_t*::: - -Py_GetProgramFullPath:wchar_t*::: - -Py_GetProgramName:wchar_t*::: - Py_GetVersion:const char*::: Py_INCREF:void::: @@ -3027,3 +3121,11 @@ _Py_c_quot:Py_complex:divisor:: _Py_c_sum:Py_complex::: _Py_c_sum:Py_complex:left:: _Py_c_sum:Py_complex:right:: + +PyImport_ImportModuleAttr:PyObject*::+1: +PyImport_ImportModuleAttr:PyObject*:mod_name:0: +PyImport_ImportModuleAttr:PyObject*:attr_name:0: + +PyImport_ImportModuleAttrString:PyObject*::+1: +PyImport_ImportModuleAttrString:const char *:mod_name:: +PyImport_ImportModuleAttrString:const char *:attr_name:: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 6f9d27297e8f65..510e683c87e8b9 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,5 +1,22 @@ role,name,added,ifdef_note,struct_abi_kind +macro,METH_CLASS,3.2,, +macro,METH_COEXIST,3.2,, +macro,METH_FASTCALL,3.7,, +macro,METH_METHOD,3.7,, +macro,METH_NOARGS,3.2,, +macro,METH_O,3.2,, +macro,METH_STATIC,3.2,, +macro,METH_VARARGS,3.2,, macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, +type,PyABIInfo,3.15,,full-abi +func,PyABIInfo_Check,3.15,, +macro,PyABIInfo_DEFAULT_ABI_VERSION,3.15,, +macro,PyABIInfo_DEFAULT_FLAGS,3.15,, +macro,PyABIInfo_FREETHREADED,3.15,, +macro,PyABIInfo_FREETHREADING_AGNOSTIC,3.15,, +macro,PyABIInfo_GIL,3.15,, +macro,PyABIInfo_STABLE,3.15,, +macro,PyABIInfo_VAR,3.15,, func,PyAIter_Check,3.10,, func,PyArg_Parse,3.2,, func,PyArg_ParseTuple,3.2,, @@ -8,6 +25,26 @@ func,PyArg_UnpackTuple,3.2,, func,PyArg_VaParse,3.2,, func,PyArg_VaParseTupleAndKeywords,3.2,, func,PyArg_ValidateKeywordArguments,3.2,, +macro,PyBUF_ANY_CONTIGUOUS,3.11,, +macro,PyBUF_CONTIG,3.11,, +macro,PyBUF_CONTIG_RO,3.11,, +macro,PyBUF_C_CONTIGUOUS,3.11,, +macro,PyBUF_FORMAT,3.11,, +macro,PyBUF_FULL,3.11,, +macro,PyBUF_FULL_RO,3.11,, +macro,PyBUF_F_CONTIGUOUS,3.11,, +macro,PyBUF_INDIRECT,3.11,, +macro,PyBUF_MAX_NDIM,3.11,, +macro,PyBUF_ND,3.11,, +macro,PyBUF_READ,3.11,, +macro,PyBUF_RECORDS,3.11,, +macro,PyBUF_RECORDS_RO,3.11,, +macro,PyBUF_SIMPLE,3.11,, +macro,PyBUF_STRIDED,3.11,, +macro,PyBUF_STRIDED_RO,3.11,, +macro,PyBUF_STRIDES,3.11,, +macro,PyBUF_WRITABLE,3.11,, +macro,PyBUF_WRITE,3.11,, data,PyBaseObject_Type,3.2,, func,PyBool_FromLong,3.2,, data,PyBool_Type,3.2,, @@ -123,6 +160,7 @@ func,PyDict_Merge,3.2,, func,PyDict_MergeFromSeq2,3.2,, func,PyDict_New,3.2,, func,PyDict_Next,3.2,, +func,PyDict_SetDefaultRef,3.15,, func,PyDict_SetItem,3.2,, func,PyDict_SetItemString,3.2,, func,PyDict_Size,3.2,, @@ -323,7 +361,6 @@ func,PyImport_ImportFrozenModuleObject,3.7,, func,PyImport_ImportModule,3.2,, func,PyImport_ImportModuleLevel,3.2,, func,PyImport_ImportModuleLevelObject,3.7,, -func,PyImport_ImportModuleNoBlock,3.2,, func,PyImport_ReloadModule,3.2,, func,PyIndex_Check,3.8,, type,PyInterpreterState,3.2,,opaque @@ -352,8 +389,14 @@ func,PyList_SetSlice,3.2,, func,PyList_Size,3.2,, func,PyList_Sort,3.2,, data,PyList_Type,3.2,, +type,PyLongExport,3.15,,full-abi +type,PyLongLayout,3.15,,full-abi type,PyLongObject,3.2,,opaque data,PyLongRangeIter_Type,3.2,, +type,PyLongWriter,3.15,,opaque +func,PyLongWriter_Create,3.15,, +func,PyLongWriter_Discard,3.15,, +func,PyLongWriter_Finish,3.15,, func,PyLong_AsDouble,3.2,, func,PyLong_AsInt,3.13,, func,PyLong_AsInt32,3.14,, @@ -362,6 +405,7 @@ func,PyLong_AsLong,3.2,, func,PyLong_AsLongAndOverflow,3.2,, func,PyLong_AsLongLong,3.2,, func,PyLong_AsLongLongAndOverflow,3.2,, +func,PyLong_AsNativeBytes,3.14,, func,PyLong_AsSize_t,3.2,, func,PyLong_AsSsize_t,3.2,, func,PyLong_AsUInt32,3.14,, @@ -371,11 +415,14 @@ func,PyLong_AsUnsignedLongLong,3.2,, func,PyLong_AsUnsignedLongLongMask,3.2,, func,PyLong_AsUnsignedLongMask,3.2,, func,PyLong_AsVoidPtr,3.2,, +func,PyLong_Export,3.15,, +func,PyLong_FreeExport,3.15,, func,PyLong_FromDouble,3.2,, func,PyLong_FromInt32,3.14,, func,PyLong_FromInt64,3.14,, func,PyLong_FromLong,3.2,, func,PyLong_FromLongLong,3.2,, +func,PyLong_FromNativeBytes,3.14,, func,PyLong_FromSize_t,3.2,, func,PyLong_FromSsize_t,3.2,, func,PyLong_FromString,3.2,, @@ -383,9 +430,12 @@ func,PyLong_FromUInt32,3.14,, func,PyLong_FromUInt64,3.14,, func,PyLong_FromUnsignedLong,3.2,, func,PyLong_FromUnsignedLongLong,3.2,, +func,PyLong_FromUnsignedNativeBytes,3.14,, func,PyLong_FromVoidPtr,3.2,, func,PyLong_GetInfo,3.2,, +func,PyLong_GetNativeLayout,3.15,, data,PyLong_Type,3.2,, +macro,PyMODEXPORT_FUNC,3.15,, data,PyMap_Type,3.2,, func,PyMapping_Check,3.2,, func,PyMapping_GetItemString,3.2,, @@ -423,6 +473,7 @@ data,PyMethodDescr_Type,3.2,, type,PyModuleDef,3.2,,full-abi type,PyModuleDef_Base,3.2,,full-abi func,PyModuleDef_Init,3.5,, +type,PyModuleDef_Slot,3.5,,full-abi data,PyModuleDef_Type,3.5,, func,PyModule_Add,3.13,, func,PyModule_AddFunctions,3.7,, @@ -432,8 +483,10 @@ func,PyModule_AddObjectRef,3.10,, func,PyModule_AddStringConstant,3.2,, func,PyModule_AddType,3.10,, func,PyModule_Create2,3.2,, +func,PyModule_Exec,3.15,, func,PyModule_ExecDef,3.7,, func,PyModule_FromDefAndSpec2,3.7,, +func,PyModule_FromSlotsAndSpec,3.15,, func,PyModule_GetDef,3.2,, func,PyModule_GetDict,3.2,, func,PyModule_GetFilename,3.2,, @@ -441,6 +494,8 @@ func,PyModule_GetFilenameObject,3.2,, func,PyModule_GetName,3.2,, func,PyModule_GetNameObject,3.7,, func,PyModule_GetState,3.2,, +func,PyModule_GetStateSize,3.15,, +func,PyModule_GetToken,3.15,, func,PyModule_New,3.2,, func,PyModule_NewObject,3.7,, func,PyModule_SetDocString,3.7,, @@ -626,9 +681,12 @@ func,PySys_Audit,3.13,, func,PySys_AuditTuple,3.13,, func,PySys_FormatStderr,3.2,, func,PySys_FormatStdout,3.2,, +func,PySys_GetAttr,3.15,, +func,PySys_GetAttrString,3.15,, func,PySys_GetObject,3.2,, +func,PySys_GetOptionalAttr,3.15,, +func,PySys_GetOptionalAttrString,3.15,, func,PySys_GetXOptions,3.7,, -func,PySys_ResetWarnOptions,3.2,, func,PySys_SetArgv,3.2,, func,PySys_SetArgvEx,3.2,, func,PySys_SetObject,3.2,, @@ -696,6 +754,7 @@ func,PyType_GetFlags,3.2,, func,PyType_GetFullyQualifiedName,3.13,, func,PyType_GetModule,3.10,, func,PyType_GetModuleByDef,3.13,, +func,PyType_GetModuleByToken,3.15,, func,PyType_GetModuleName,3.13,, func,PyType_GetModuleState,3.10,, func,PyType_GetName,3.11,, @@ -737,11 +796,7 @@ func,PyUnicode_Append,3.2,, func,PyUnicode_AppendAndDel,3.2,, func,PyUnicode_AsASCIIString,3.2,, func,PyUnicode_AsCharmapString,3.2,, -func,PyUnicode_AsDecodedObject,3.2,, -func,PyUnicode_AsDecodedUnicode,3.2,, -func,PyUnicode_AsEncodedObject,3.2,, func,PyUnicode_AsEncodedString,3.2,, -func,PyUnicode_AsEncodedUnicode,3.2,, func,PyUnicode_AsLatin1String,3.2,, func,PyUnicode_AsMBCSString,3.7,on Windows, func,PyUnicode_AsRawUnicodeEscapeString,3.2,, @@ -826,13 +881,20 @@ member,PyVarObject.ob_size,3.2,, func,PyVectorcall_Call,3.12,, func,PyVectorcall_NARGS,3.12,, type,PyWeakReference,3.2,,opaque -func,PyWeakref_GetObject,3.2,, func,PyWeakref_GetRef,3.13,, func,PyWeakref_NewProxy,3.2,, func,PyWeakref_NewRef,3.2,, data,PyWrapperDescr_Type,3.2,, func,PyWrapper_New,3.2,, data,PyZip_Type,3.2,, +macro,Py_ASNATIVEBYTES_ALLOW_INDEX,3.14,, +macro,Py_ASNATIVEBYTES_BIG_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_DEFAULTS,3.14,, +macro,Py_ASNATIVEBYTES_LITTLE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_NATIVE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_REJECT_NEGATIVE,3.14,, +macro,Py_ASNATIVEBYTES_UNSIGNED_BUFFER,3.14,, +macro,Py_AUDIT_READ,3.12,, func,Py_AddPendingCall,3.2,, func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, @@ -859,16 +921,11 @@ func,Py_GetCompiler,3.2,, func,Py_GetConstant,3.13,, func,Py_GetConstantBorrowed,3.13,, func,Py_GetCopyright,3.2,, -func,Py_GetExecPrefix,3.2,, -func,Py_GetPath,3.2,, func,Py_GetPlatform,3.2,, -func,Py_GetPrefix,3.2,, -func,Py_GetProgramFullPath,3.2,, -func,Py_GetProgramName,3.2,, -func,Py_GetPythonHome,3.2,, func,Py_GetRecursionLimit,3.2,, func,Py_GetVersion,3.2,, data,Py_HasFileSystemDefaultEncoding,3.2,, +func,Py_IS_TYPE,3.15,, func,Py_IncRef,3.2,, func,Py_Initialize,3.2,, func,Py_InitializeEx,3.2,, @@ -883,22 +940,149 @@ func,Py_Main,3.2,, func,Py_MakePendingCalls,3.2,, func,Py_NewInterpreter,3.2,, func,Py_NewRef,3.10,, +func,Py_PACK_FULL_VERSION,3.14,, +func,Py_PACK_VERSION,3.14,, +macro,Py_READONLY,3.12,, func,Py_REFCNT,3.14,, +macro,Py_RELATIVE_OFFSET,3.12,, func,Py_ReprEnter,3.2,, func,Py_ReprLeave,3.2,, +func,Py_SET_SIZE,3.15,, +func,Py_SIZE,3.15,, func,Py_SetProgramName,3.2,, func,Py_SetPythonHome,3.2,, func,Py_SetRecursionLimit,3.2,, +macro,Py_TPFLAGS_BASETYPE,3.2,, +macro,Py_TPFLAGS_DEFAULT,3.2,, +macro,Py_TPFLAGS_HAVE_GC,3.2,, +macro,Py_TPFLAGS_HAVE_VECTORCALL,3.12,, +macro,Py_TPFLAGS_ITEMS_AT_END,3.12,, +macro,Py_TPFLAGS_METHOD_DESCRIPTOR,3.8,, +macro,Py_TP_USE_SPEC,3.14,, func,Py_TYPE,3.14,, +macro,Py_T_BOOL,3.12,, +macro,Py_T_BYTE,3.12,, +macro,Py_T_CHAR,3.12,, +macro,Py_T_DOUBLE,3.12,, +macro,Py_T_FLOAT,3.12,, +macro,Py_T_INT,3.12,, +macro,Py_T_LONG,3.12,, +macro,Py_T_LONGLONG,3.12,, +macro,Py_T_OBJECT_EX,3.12,, +macro,Py_T_PYSSIZET,3.12,, +macro,Py_T_SHORT,3.12,, +macro,Py_T_STRING,3.12,, +macro,Py_T_STRING_INPLACE,3.12,, +macro,Py_T_UBYTE,3.12,, +macro,Py_T_UINT,3.12,, +macro,Py_T_ULONG,3.12,, +macro,Py_T_ULONGLONG,3.12,, +macro,Py_T_USHORT,3.12,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, data,Py_UTF8Mode,3.8,, func,Py_VaBuildValue,3.2,, data,Py_Version,3.11,, func,Py_XNewRef,3.10,, +macro,Py_am_aiter,3.5,, +macro,Py_am_anext,3.5,, +macro,Py_am_await,3.5,, +macro,Py_am_send,3.10,, +macro,Py_bf_getbuffer,3.11,, +macro,Py_bf_releasebuffer,3.11,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, +macro,Py_mod_abi,3.15,, +macro,Py_mod_create,3.5,, +macro,Py_mod_doc,3.15,, +macro,Py_mod_exec,3.5,, +macro,Py_mod_gil,3.13,, +macro,Py_mod_methods,3.15,, +macro,Py_mod_multiple_interpreters,3.12,, +macro,Py_mod_name,3.15,, +macro,Py_mod_state_clear,3.15,, +macro,Py_mod_state_free,3.15,, +macro,Py_mod_state_size,3.15,, +macro,Py_mod_state_traverse,3.15,, +macro,Py_mod_token,3.15,, +macro,Py_mp_ass_subscript,3.2,, +macro,Py_mp_length,3.2,, +macro,Py_mp_subscript,3.2,, +macro,Py_nb_absolute,3.2,, +macro,Py_nb_add,3.2,, +macro,Py_nb_and,3.2,, +macro,Py_nb_bool,3.2,, +macro,Py_nb_divmod,3.2,, +macro,Py_nb_float,3.2,, +macro,Py_nb_floor_divide,3.2,, +macro,Py_nb_index,3.2,, +macro,Py_nb_inplace_add,3.2,, +macro,Py_nb_inplace_and,3.2,, +macro,Py_nb_inplace_floor_divide,3.2,, +macro,Py_nb_inplace_lshift,3.2,, +macro,Py_nb_inplace_matrix_multiply,3.5,, +macro,Py_nb_inplace_multiply,3.2,, +macro,Py_nb_inplace_or,3.2,, +macro,Py_nb_inplace_power,3.2,, +macro,Py_nb_inplace_remainder,3.2,, +macro,Py_nb_inplace_rshift,3.2,, +macro,Py_nb_inplace_subtract,3.2,, +macro,Py_nb_inplace_true_divide,3.2,, +macro,Py_nb_inplace_xor,3.2,, +macro,Py_nb_int,3.2,, +macro,Py_nb_invert,3.2,, +macro,Py_nb_lshift,3.2,, +macro,Py_nb_matrix_multiply,3.5,, +macro,Py_nb_multiply,3.2,, +macro,Py_nb_negative,3.2,, +macro,Py_nb_or,3.2,, +macro,Py_nb_positive,3.2,, +macro,Py_nb_power,3.2,, +macro,Py_nb_remainder,3.2,, +macro,Py_nb_rshift,3.2,, +macro,Py_nb_subtract,3.2,, +macro,Py_nb_true_divide,3.2,, +macro,Py_nb_xor,3.2,, +macro,Py_sq_ass_item,3.2,, +macro,Py_sq_concat,3.2,, +macro,Py_sq_contains,3.2,, +macro,Py_sq_inplace_concat,3.2,, +macro,Py_sq_inplace_repeat,3.2,, +macro,Py_sq_item,3.2,, +macro,Py_sq_length,3.2,, +macro,Py_sq_repeat,3.2,, type,Py_ssize_t,3.2,, +macro,Py_tp_alloc,3.2,, +macro,Py_tp_base,3.2,, +macro,Py_tp_bases,3.2,, +macro,Py_tp_call,3.2,, +macro,Py_tp_clear,3.2,, +macro,Py_tp_dealloc,3.2,, +macro,Py_tp_del,3.2,, +macro,Py_tp_descr_get,3.2,, +macro,Py_tp_descr_set,3.2,, +macro,Py_tp_doc,3.2,, +macro,Py_tp_finalize,3.5,, +macro,Py_tp_free,3.2,, +macro,Py_tp_getattr,3.2,, +macro,Py_tp_getattro,3.2,, +macro,Py_tp_getset,3.2,, +macro,Py_tp_hash,3.2,, +macro,Py_tp_init,3.2,, +macro,Py_tp_is_gc,3.2,, +macro,Py_tp_iter,3.2,, +macro,Py_tp_iternext,3.2,, +macro,Py_tp_members,3.2,, +macro,Py_tp_methods,3.2,, +macro,Py_tp_new,3.2,, +macro,Py_tp_repr,3.2,, +macro,Py_tp_richcompare,3.2,, +macro,Py_tp_setattr,3.2,, +macro,Py_tp_setattro,3.2,, +macro,Py_tp_str,3.2,, +macro,Py_tp_token,3.14,, +macro,Py_tp_traverse,3.2,, +macro,Py_tp_vectorcall,3.14,, type,Py_uintptr_t,3.2,, type,allocfunc,3.2,, type,binaryfunc,3.2,, diff --git a/Doc/deprecations/c-api-pending-removal-in-3.14.rst b/Doc/deprecations/c-api-pending-removal-in-3.14.rst index 9e10bf2691e5c8..c805074669811a 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.14.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.14.rst @@ -6,67 +6,3 @@ Pending removal in Python 3.14 * Creating :c:data:`immutable types ` with mutable bases (:gh:`95388`). - -* Functions to configure Python's initialization, deprecated in Python 3.11: - - * :c:func:`!PySys_SetArgvEx()`: - Set :c:member:`PyConfig.argv` instead. - * :c:func:`!PySys_SetArgv()`: - Set :c:member:`PyConfig.argv` instead. - * :c:func:`!Py_SetProgramName()`: - Set :c:member:`PyConfig.program_name` instead. - * :c:func:`!Py_SetPythonHome()`: - Set :c:member:`PyConfig.home` instead. - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` instead. - -* Global configuration variables: - - * :c:var:`Py_DebugFlag`: - Use :c:member:`PyConfig.parser_debug` instead. - * :c:var:`Py_VerboseFlag`: - Use :c:member:`PyConfig.verbose` instead. - * :c:var:`Py_QuietFlag`: - Use :c:member:`PyConfig.quiet` instead. - * :c:var:`Py_InteractiveFlag`: - Use :c:member:`PyConfig.interactive` instead. - * :c:var:`Py_InspectFlag`: - Use :c:member:`PyConfig.inspect` instead. - * :c:var:`Py_OptimizeFlag`: - Use :c:member:`PyConfig.optimization_level` instead. - * :c:var:`Py_NoSiteFlag`: - Use :c:member:`PyConfig.site_import` instead. - * :c:var:`Py_BytesWarningFlag`: - Use :c:member:`PyConfig.bytes_warning` instead. - * :c:var:`Py_FrozenFlag`: - Use :c:member:`PyConfig.pathconfig_warnings` instead. - * :c:var:`Py_IgnoreEnvironmentFlag`: - Use :c:member:`PyConfig.use_environment` instead. - * :c:var:`Py_DontWriteBytecodeFlag`: - Use :c:member:`PyConfig.write_bytecode` instead. - * :c:var:`Py_NoUserSiteDirectory`: - Use :c:member:`PyConfig.user_site_directory` instead. - * :c:var:`Py_UnbufferedStdioFlag`: - Use :c:member:`PyConfig.buffered_stdio` instead. - * :c:var:`Py_HashRandomizationFlag`: - Use :c:member:`PyConfig.use_hash_seed` - and :c:member:`PyConfig.hash_seed` instead. - * :c:var:`Py_IsolatedFlag`: - Use :c:member:`PyConfig.isolated` instead. - * :c:var:`Py_LegacyWindowsFSEncodingFlag`: - Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` instead. - * :c:var:`Py_LegacyWindowsStdioFlag`: - Use :c:member:`PyConfig.legacy_windows_stdio` instead. - * :c:var:`!Py_FileSystemDefaultEncoding`: - Use :c:member:`PyConfig.filesystem_encoding` instead. - * :c:var:`!Py_HasFileSystemDefaultEncoding`: - Use :c:member:`PyConfig.filesystem_encoding` instead. - * :c:var:`!Py_FileSystemDefaultEncodeErrors`: - Use :c:member:`PyConfig.filesystem_errors` instead. - * :c:var:`!Py_UTF8Mode`: - Use :c:member:`PyPreConfig.utf8_mode` instead. - (see :c:func:`Py_PreInitialize`) - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` instead. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst index 0ce0f9c118c094..9927b876760d34 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.15.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -1,27 +1,135 @@ Pending removal in Python 3.15 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* The bundled copy of ``libmpdecimal``. -* The :c:func:`PyImport_ImportModuleNoBlock`: +* The :c:func:`!PyImport_ImportModuleNoBlock`: Use :c:func:`PyImport_ImportModule` instead. -* :c:func:`PyWeakref_GetObject` and :c:func:`PyWeakref_GET_OBJECT`: - Use :c:func:`PyWeakref_GetRef` instead. +* :c:func:`!PyWeakref_GetObject` and :c:func:`!PyWeakref_GET_OBJECT`: + Use :c:func:`PyWeakref_GetRef` instead. The `pythoncapi-compat project + `__ can be used to get + :c:func:`PyWeakref_GetRef` on Python 3.12 and older. * :c:type:`Py_UNICODE` type and the :c:macro:`!Py_UNICODE_WIDE` macro: Use :c:type:`wchar_t` instead. -* Python initialization functions: +* :c:func:`!PyUnicode_AsDecodedObject`: + Use :c:func:`PyCodec_Decode` instead. +* :c:func:`!PyUnicode_AsDecodedUnicode`: + Use :c:func:`PyCodec_Decode` instead; Note that some codecs (for example, "base64") + may return a type other than :class:`str`, such as :class:`bytes`. +* :c:func:`!PyUnicode_AsEncodedObject`: + Use :c:func:`PyCodec_Encode` instead. +* :c:func:`!PyUnicode_AsEncodedUnicode`: + Use :c:func:`PyCodec_Encode` instead; Note that some codecs (for example, "base64") + may return a type other than :class:`bytes`, such as :class:`str`. +* Python initialization functions, deprecated in Python 3.13: - * :c:func:`PySys_ResetWarnOptions`: + * :c:func:`!Py_GetPath`: + Use :c:func:`PyConfig_Get("module_search_paths") ` + (:data:`sys.path`) instead. + * :c:func:`!Py_GetPrefix`: + Use :c:func:`PyConfig_Get("base_prefix") ` + (:data:`sys.base_prefix`) instead. Use :c:func:`PyConfig_Get("prefix") + ` (:data:`sys.prefix`) if :ref:`virtual environments + ` need to be handled. + * :c:func:`!Py_GetExecPrefix`: + Use :c:func:`PyConfig_Get("base_exec_prefix") ` + (:data:`sys.base_exec_prefix`) instead. Use + :c:func:`PyConfig_Get("exec_prefix") ` + (:data:`sys.exec_prefix`) if :ref:`virtual environments ` need to + be handled. + * :c:func:`!Py_GetProgramFullPath`: + Use :c:func:`PyConfig_Get("executable") ` + (:data:`sys.executable`) instead. + * :c:func:`!Py_GetProgramName`: + Use :c:func:`PyConfig_Get("executable") ` + (:data:`sys.executable`) instead. + * :c:func:`!Py_GetPythonHome`: + Use :c:func:`PyConfig_Get("home") ` or the + :envvar:`PYTHONHOME` environment variable instead. + + The `pythoncapi-compat project + `__ can be used to get + :c:func:`PyConfig_Get` on Python 3.13 and older. + +* Functions to configure Python's initialization, deprecated in Python 3.11: + + * :c:func:`!PySys_SetArgvEx()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!PySys_SetArgv()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!Py_SetProgramName()`: + Set :c:member:`PyConfig.program_name` instead. + * :c:func:`!Py_SetPythonHome()`: + Set :c:member:`PyConfig.home` instead. + * :c:func:`!PySys_ResetWarnOptions`: Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - * :c:func:`Py_GetExecPrefix`: - Get :data:`sys.base_exec_prefix` and :data:`sys.exec_prefix` instead. - * :c:func:`Py_GetPath`: - Get :data:`sys.path` instead. - * :c:func:`Py_GetPrefix`: - Get :data:`sys.base_prefix` and :data:`sys.prefix` instead. - * :c:func:`Py_GetProgramFullPath`: - Get :data:`sys.executable` instead. - * :c:func:`Py_GetProgramName`: - Get :data:`sys.executable` instead. - * :c:func:`Py_GetPythonHome`: - Get :c:member:`PyConfig.home` - or the :envvar:`PYTHONHOME` environment variable instead. + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` instead. + +* Global configuration variables: + + * :c:var:`Py_DebugFlag`: + Use :c:member:`PyConfig.parser_debug` or + :c:func:`PyConfig_Get("parser_debug") ` instead. + * :c:var:`Py_VerboseFlag`: + Use :c:member:`PyConfig.verbose` or + :c:func:`PyConfig_Get("verbose") ` instead. + * :c:var:`Py_QuietFlag`: + Use :c:member:`PyConfig.quiet` or + :c:func:`PyConfig_Get("quiet") ` instead. + * :c:var:`Py_InteractiveFlag`: + Use :c:member:`PyConfig.interactive` or + :c:func:`PyConfig_Get("interactive") ` instead. + * :c:var:`Py_InspectFlag`: + Use :c:member:`PyConfig.inspect` or + :c:func:`PyConfig_Get("inspect") ` instead. + * :c:var:`Py_OptimizeFlag`: + Use :c:member:`PyConfig.optimization_level` or + :c:func:`PyConfig_Get("optimization_level") ` instead. + * :c:var:`Py_NoSiteFlag`: + Use :c:member:`PyConfig.site_import` or + :c:func:`PyConfig_Get("site_import") ` instead. + * :c:var:`Py_BytesWarningFlag`: + Use :c:member:`PyConfig.bytes_warning` or + :c:func:`PyConfig_Get("bytes_warning") ` instead. + * :c:var:`Py_FrozenFlag`: + Use :c:member:`PyConfig.pathconfig_warnings` or + :c:func:`PyConfig_Get("pathconfig_warnings") ` instead. + * :c:var:`Py_IgnoreEnvironmentFlag`: + Use :c:member:`PyConfig.use_environment` or + :c:func:`PyConfig_Get("use_environment") ` instead. + * :c:var:`Py_DontWriteBytecodeFlag`: + Use :c:member:`PyConfig.write_bytecode` or + :c:func:`PyConfig_Get("write_bytecode") ` instead. + * :c:var:`Py_NoUserSiteDirectory`: + Use :c:member:`PyConfig.user_site_directory` or + :c:func:`PyConfig_Get("user_site_directory") ` instead. + * :c:var:`Py_UnbufferedStdioFlag`: + Use :c:member:`PyConfig.buffered_stdio` or + :c:func:`PyConfig_Get("buffered_stdio") ` instead. + * :c:var:`Py_HashRandomizationFlag`: + Use :c:member:`PyConfig.use_hash_seed` + and :c:member:`PyConfig.hash_seed` or + :c:func:`PyConfig_Get("hash_seed") ` instead. + * :c:var:`Py_IsolatedFlag`: + Use :c:member:`PyConfig.isolated` or + :c:func:`PyConfig_Get("isolated") ` instead. + * :c:var:`Py_LegacyWindowsFSEncodingFlag`: + Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` or + :c:func:`PyConfig_Get("legacy_windows_fs_encoding") ` instead. + * :c:var:`Py_LegacyWindowsStdioFlag`: + Use :c:member:`PyConfig.legacy_windows_stdio` or + :c:func:`PyConfig_Get("legacy_windows_stdio") ` instead. + * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: + Use :c:member:`PyConfig.filesystem_encoding` or + :c:func:`PyConfig_Get("filesystem_encoding") ` instead. + * :c:var:`!Py_FileSystemDefaultEncodeErrors`: + Use :c:member:`PyConfig.filesystem_errors` or + :c:func:`PyConfig_Get("filesystem_errors") ` instead. + * :c:var:`!Py_UTF8Mode`: + Use :c:member:`PyPreConfig.utf8_mode` or + :c:func:`PyConfig_Get("utf8_mode") ` instead. + (see :c:func:`Py_PreInitialize`) + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` to set these options. Or :c:func:`PyConfig_Get` can be + used to get these options at runtime. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.16.rst b/Doc/deprecations/c-api-pending-removal-in-3.16.rst new file mode 100644 index 00000000000000..9453f83799c43d --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.16.rst @@ -0,0 +1,4 @@ +Pending removal in Python 3.16 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The bundled copy of ``libmpdec``. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.18.rst b/Doc/deprecations/c-api-pending-removal-in-3.18.rst new file mode 100644 index 00000000000000..022aee93aa70c4 --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.18.rst @@ -0,0 +1,47 @@ +Pending removal in Python 3.18 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The following private functions are deprecated + and planned for removal in Python 3.18: + + * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. + * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. + * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. + * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. + * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: + use :c:func:`PyLongWriter_Create`. + * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. + * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. + * :c:func:`!_PyUnicodeWriter_Init`: + replace ``_PyUnicodeWriter_Init(&writer)`` with + :c:func:`writer = PyUnicodeWriter_Create(0) `. + * :c:func:`!_PyUnicodeWriter_Finish`: + replace ``_PyUnicodeWriter_Finish(&writer)`` with + :c:func:`PyUnicodeWriter_Finish(writer) `. + * :c:func:`!_PyUnicodeWriter_Dealloc`: + replace ``_PyUnicodeWriter_Dealloc(&writer)`` with + :c:func:`PyUnicodeWriter_Discard(writer) `. + * :c:func:`!_PyUnicodeWriter_WriteChar`: + replace ``_PyUnicodeWriter_WriteChar(&writer, ch)`` with + :c:func:`PyUnicodeWriter_WriteChar(writer, ch) `. + * :c:func:`!_PyUnicodeWriter_WriteStr`: + replace ``_PyUnicodeWriter_WriteStr(&writer, str)`` with + :c:func:`PyUnicodeWriter_WriteStr(writer, str) `. + * :c:func:`!_PyUnicodeWriter_WriteSubstring`: + replace ``_PyUnicodeWriter_WriteSubstring(&writer, str, start, end)`` with + :c:func:`PyUnicodeWriter_WriteSubstring(writer, str, start, end) `. + * :c:func:`!_PyUnicodeWriter_WriteASCIIString`: + replace ``_PyUnicodeWriter_WriteASCIIString(&writer, str)`` with + :c:func:`PyUnicodeWriter_WriteASCII(writer, str) `. + * :c:func:`!_PyUnicodeWriter_WriteLatin1String`: + replace ``_PyUnicodeWriter_WriteLatin1String(&writer, str)`` with + :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. + * :c:func:`!_PyUnicodeWriter_Prepare`: (no replacement). + * :c:func:`!_PyUnicodeWriter_PrepareKind`: (no replacement). + * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. + * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. + + The `pythoncapi-compat project + `__ can be used to get + these new public functions on Python 3.13 and older. + (Contributed by Victor Stinner in :gh:`128863`.) diff --git a/Doc/deprecations/c-api-pending-removal-in-3.20.rst b/Doc/deprecations/c-api-pending-removal-in-3.20.rst new file mode 100644 index 00000000000000..8de55bbe7e695c --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.20.rst @@ -0,0 +1,16 @@ +Pending removal in Python 3.20 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* :c:func:`!_PyObject_CallMethodId`, :c:func:`!_PyObject_GetAttrId` and + :c:func:`!_PyUnicode_FromId` are deprecated since 3.15 and will be removed in + 3.20. Instead, use :c:func:`PyUnicode_InternFromString()` and cache the result in + the module state, then call :c:func:`PyObject_CallMethod` or + :c:func:`PyObject_GetAttr`. + (Contributed by Victor Stinner in :gh:`141049`.) + +* The ``cval`` field in :c:type:`PyComplexObject` (:gh:`128813`). + Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` + to convert a Python complex number to/from the C :c:type:`Py_complex` + representation. + +* Macros :c:macro:`!Py_MATH_PIl` and :c:macro:`!Py_MATH_El`. diff --git a/Doc/deprecations/c-api-pending-removal-in-future.rst b/Doc/deprecations/c-api-pending-removal-in-future.rst index 8fc1c80c35d092..841d1b455b6bec 100644 --- a/Doc/deprecations/c-api-pending-removal-in-future.rst +++ b/Doc/deprecations/c-api-pending-removal-in-future.rst @@ -18,14 +18,6 @@ although there is currently no date scheduled for their removal. Use :c:func:`PyOS_AfterFork_Child` instead. * :c:func:`PySlice_GetIndicesEx`: Use :c:func:`PySlice_Unpack` and :c:func:`PySlice_AdjustIndices` instead. -* :c:func:`!PyUnicode_AsDecodedObject`: - Use :c:func:`PyCodec_Decode` instead. -* :c:func:`!PyUnicode_AsDecodedUnicode`: - Use :c:func:`PyCodec_Decode` instead. -* :c:func:`!PyUnicode_AsEncodedObject`: - Use :c:func:`PyCodec_Encode` instead. -* :c:func:`!PyUnicode_AsEncodedUnicode`: - Use :c:func:`PyCodec_Encode` instead. * :c:func:`PyUnicode_READY`: Unneeded since Python 3.12 * :c:func:`!PyErr_Display`: @@ -34,7 +26,6 @@ although there is currently no date scheduled for their removal. Use :c:func:`!_PyErr_ChainExceptions1` instead. * :c:member:`!PyBytesObject.ob_shash` member: call :c:func:`PyObject_Hash` instead. -* :c:member:`!PyDictObject.ma_version_tag` member. * Thread Local Storage (TLS) API: * :c:func:`PyThread_create_key`: diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst index bac6e3f18d4594..c91c64a1092457 100644 --- a/Doc/deprecations/index.rst +++ b/Doc/deprecations/index.rst @@ -5,6 +5,14 @@ Deprecations .. include:: pending-removal-in-3.16.rst +.. include:: pending-removal-in-3.17.rst + +.. include:: pending-removal-in-3.18.rst + +.. include:: pending-removal-in-3.19.rst + +.. include:: pending-removal-in-3.20.rst + .. include:: pending-removal-in-future.rst C API deprecations @@ -12,4 +20,8 @@ C API deprecations .. include:: c-api-pending-removal-in-3.15.rst +.. include:: c-api-pending-removal-in-3.18.rst + +.. include:: c-api-pending-removal-in-3.20.rst + .. include:: c-api-pending-removal-in-future.rst diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst index 2fd2f12cc6a2c4..d5b8c80e8f9aa0 100644 --- a/Doc/deprecations/pending-removal-in-3.13.rst +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -38,15 +38,3 @@ APIs: * :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) * :class:`!webbrowser.MacOSX` (:gh:`86421`) * :class:`classmethod` descriptor chaining (:gh:`89519`) -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files` instead. Refer to `importlib-resources: Migrating from Legacy - `_ (:gh:`106531`) diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst index b8791b8d6c387e..171758156ab117 100644 --- a/Doc/deprecations/pending-removal-in-3.14.rst +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -1,13 +1,6 @@ Pending removal in Python 3.14 ------------------------------ -* The import system: - - * Setting :attr:`~module.__loader__` on a module while - failing to set :attr:`__spec__.loader ` - is deprecated. In Python 3.14, :attr:`!__loader__` will cease to be set or - taken into consideration by the import system or the standard library. - * :mod:`argparse`: The *type*, *choices*, and *metavar* parameters of :class:`!argparse.BooleanOptionalAction` are deprecated and will be removed in 3.14. @@ -45,12 +38,6 @@ Pending removal in Python 3.14 is no current event loop set and it decides to create one. (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) -* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. - Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, - or :class:`collections.abc.Buffer`. - (Contributed by Shantanu Jain in :gh:`91896`.) - * :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. (Contributed by Alan Williams in :gh:`72346`.) @@ -85,7 +72,7 @@ Pending removal in Python 3.14 :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is deprecated. -* :mod:`pkgutil`: :func:`~pkgutil.find_loader` and :func:`~pkgutil.get_loader` +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` now raise :exc:`DeprecationWarning`; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) @@ -103,19 +90,6 @@ Pending removal in Python 3.14 if :ref:`named placeholders ` are used and *parameters* is a sequence instead of a :class:`dict`. - * date and datetime adapter, date and timestamp converter: - see the :mod:`sqlite3` documentation for suggested replacement recipes. - -* :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was - deprecated in :pep:`626` - since 3.10 and was planned to be removed in 3.12, - but it only got a proper :exc:`DeprecationWarning` in 3.12. - May be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`101866`.) - -* :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, - now causes a :exc:`DeprecationWarning` to be emitted when it is used. - * :mod:`urllib`: :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a public API. diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index 17029b8d4773bd..00266b1725c8a1 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -3,9 +3,9 @@ Pending removal in Python 3.15 * The import system: - * Setting :attr:`~module.__cached__` on a module while + * Setting ``__cached__`` on a module while failing to set :attr:`__spec__.cached ` - is deprecated. In Python 3.15, :attr:`!__cached__` will cease to be set or + is deprecated. In Python 3.15, ``__cached__`` will cease to be set or take into consideration by the import system or standard library. (:gh:`97879`) * Setting :attr:`~module.__package__` on a module while @@ -20,7 +20,7 @@ Pending removal in Python 3.15 * :mod:`http.server`: - * The obsolete and rarely used :class:`~http.server.CGIHTTPRequestHandler` + * The obsolete and rarely used :class:`!CGIHTTPRequestHandler` has been deprecated since Python 3.13. No direct replacement exists. *Anything* is better than CGI to interface @@ -29,28 +29,27 @@ Pending removal in Python 3.15 * The :option:`!--cgi` flag to the :program:`python -m http.server` command-line interface has been deprecated since Python 3.13. -* :class:`locale`: +* :mod:`importlib`: - * The :func:`~locale.getdefaultlocale` function - has been deprecated since Python 3.11. - Its removal was originally planned for Python 3.13 (:gh:`90817`), - but has been postponed to Python 3.15. - Use :func:`~locale.getlocale`, :func:`~locale.setlocale`, - and :func:`~locale.getencoding` instead. - (Contributed by Hugo van Kemenade in :gh:`111187`.) + * ``load_module()`` method: use ``exec_module()`` instead. * :mod:`pathlib`: - * :meth:`.PurePath.is_reserved` + * :meth:`!.PurePath.is_reserved` has been deprecated since Python 3.13. Use :func:`os.path.isreserved` to detect reserved paths on Windows. * :mod:`platform`: - * :func:`~platform.java_ver` has been deprecated since Python 3.13. + * :func:`!platform.java_ver` has been deprecated since Python 3.13. This function is only useful for Jython support, has a confusing API, and is largely untested. +* :mod:`sysconfig`: + + * The *check_home* argument of :func:`sysconfig.is_python_build` has been + deprecated since Python 3.12. + * :mod:`threading`: * :func:`~threading.RLock` will take no arguments in Python 3.15. @@ -59,6 +58,15 @@ Pending removal in Python 3.15 but the C version allows any number of positional or keyword arguments, ignoring every argument. +* :mod:`types`: + + * :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + deprecated in :pep:`626` + since 3.10 and was planned to be removed in 3.12, + but it only got a proper :exc:`DeprecationWarning` in 3.12. + May be removed in 3.15. + (Contributed by Nikita Sobolev in :gh:`101866`.) + * :mod:`typing`: * The undocumented keyword argument syntax for creating @@ -67,14 +75,28 @@ Pending removal in Python 3.15 has been deprecated since Python 3.13. Use the class-based syntax or the functional syntax instead. - * The :func:`typing.no_type_check_decorator` decorator function + * When using the functional syntax of :class:`~typing.TypedDict`\s, failing + to pass a value to the *fields* parameter (``TD = TypedDict("TD")``) or + passing ``None`` (``TD = TypedDict("TD", None)``) has been deprecated + since Python 3.13. + Use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})`` + to create a TypedDict with zero field. + + * The :func:`!typing.no_type_check_decorator` decorator function has been deprecated since Python 3.13. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. +* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + * :mod:`wave`: - * The :meth:`~wave.Wave_read.getmark`, :meth:`!setmark`, - and :meth:`~wave.Wave_read.getmarkers` methods of + * The ``getmark()``, ``setmark()`` and ``getmarkers()`` methods of the :class:`~wave.Wave_read` and :class:`~wave.Wave_write` classes have been deprecated since Python 3.13. + +* :mod:`zipimport`: + + * :meth:`!zipimport.zipimporter.load_module` has been deprecated since + Python 3.10. Use :meth:`~zipimport.zipimporter.exec_module` instead. + (:gh:`125746`.) diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index fac500d34742ca..b00c7002b03772 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -1,6 +1,13 @@ Pending removal in Python 3.16 ------------------------------ +* The import system: + + * Setting :attr:`~module.__loader__` on a module while + failing to set :attr:`__spec__.loader ` + is deprecated. In Python 3.16, :attr:`!__loader__` will cease to be set or + taken into consideration by the import system or the standard library. + * :mod:`array`: * The ``'u'`` format code (:c:type:`wchar_t`) @@ -12,10 +19,34 @@ Pending removal in Python 3.16 * :mod:`asyncio`: * :func:`!asyncio.iscoroutinefunction` is deprecated - and will be removed in Python 3.16, + and will be removed in Python 3.16; use :func:`inspect.iscoroutinefunction` instead. (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. + In particular, the following classes and functions are deprecated: + + * :class:`asyncio.AbstractEventLoopPolicy` + * :class:`asyncio.DefaultEventLoopPolicy` + * :class:`asyncio.WindowsSelectorEventLoopPolicy` + * :class:`asyncio.WindowsProactorEventLoopPolicy` + * :func:`asyncio.get_event_loop_policy` + * :func:`asyncio.set_event_loop_policy` + + Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with + *loop_factory* to use the desired event loop implementation. + + For example, to use :class:`asyncio.SelectorEventLoop` on Windows:: + + import asyncio + + async def main(): + ... + + asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) + + (Contributed by Kumar Aditya in :gh:`127949`.) + * :mod:`builtins`: * Bitwise inversion on boolean types, ``~True`` or ``~False`` @@ -25,6 +56,25 @@ Pending removal in Python 3.16 In the rare case that you need the bitwise inversion of the underlying integer, convert to ``int`` explicitly (``~int(x)``). +* :mod:`functools`: + + * Calling the Python implementation of :func:`functools.reduce` with *function* + or *sequence* as keyword arguments has been deprecated since Python 3.14. + +* :mod:`logging`: + + * Support for custom logging handlers with the *strm* argument is deprecated + and scheduled for removal in Python 3.16. Define handlers with the *stream* + argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) + +* :mod:`mimetypes`: + + * Valid extensions start with a '.' or are empty for + :meth:`mimetypes.MimeTypes.add_type`. + Undotted extensions are deprecated and will + raise a :exc:`ValueError` in Python 3.16. + (Contributed by Hugo van Kemenade in :gh:`75223`.) + * :mod:`shutil`: * The :class:`!ExecError` exception @@ -43,6 +93,12 @@ Pending removal in Python 3.16 has been deprecated since Python 3.13. Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. +* :mod:`sysconfig`: + + * The :func:`!sysconfig.expand_makefile_vars` function + has been deprecated since Python 3.14. + Use the ``vars`` argument of :func:`sysconfig.get_paths` instead. + * :mod:`tarfile`: * The undocumented and unused :attr:`!TarFile.tarfile` attribute diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst new file mode 100644 index 00000000000000..e769c9d371e133 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -0,0 +1,57 @@ +Pending removal in Python 3.17 +------------------------------ + +* :mod:`collections.abc`: + + - :class:`collections.abc.ByteString` is scheduled for removal in Python 3.17. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` + were also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + (Contributed by Shantanu Jain in :gh:`91896`.) + + +* :mod:`encodings`: + + - Passing non-ascii *encoding* names to :func:`encodings.normalize_encoding` + is deprecated and scheduled for removal in Python 3.17. + (Contributed by Stan Ulbrych in :gh:`136702`) + +* :mod:`typing`: + + - Before Python 3.14, old-style unions were implemented using the private class + ``typing._UnionGenericAlias``. This class is no longer needed for the implementation, + but it has been retained for backward compatibility, with removal scheduled for Python + 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` + and :func:`typing.get_args` instead of relying on private implementation details. + - :class:`typing.ByteString`, deprecated since Python 3.9, is scheduled for removal in + Python 3.17. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` + were also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + (Contributed by Shantanu Jain in :gh:`91896`.) diff --git a/Doc/deprecations/pending-removal-in-3.18.rst b/Doc/deprecations/pending-removal-in-3.18.rst new file mode 100644 index 00000000000000..3e799219478424 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.18.rst @@ -0,0 +1,9 @@ +Pending removal in Python 3.18 +------------------------------ + +* :mod:`decimal`: + + * The non-standard and undocumented :class:`~decimal.Decimal` format + specifier ``'N'``, which is only supported in the :mod:`!decimal` module's + C implementation, has been deprecated since Python 3.13. + (Contributed by Serhiy Storchaka in :gh:`89902`.) diff --git a/Doc/deprecations/pending-removal-in-3.19.rst b/Doc/deprecations/pending-removal-in-3.19.rst new file mode 100644 index 00000000000000..25f9cba390de68 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.19.rst @@ -0,0 +1,24 @@ +Pending removal in Python 3.19 +------------------------------ + +* :mod:`ctypes`: + + * Implicitly switching to the MSVC-compatible struct layout by setting + :attr:`~ctypes.Structure._pack_` but not :attr:`~ctypes.Structure._layout_` + on non-Windows platforms. + +* :mod:`hashlib`: + + - In hash function constructors such as :func:`~hashlib.new` or the + direct hash-named constructors such as :func:`~hashlib.md5` and + :func:`~hashlib.sha256`, their optional initial data parameter could + also be passed a keyword argument named ``data=`` or ``string=`` in + various :mod:`!hashlib` implementations. + + Support for the ``string`` keyword argument name is now deprecated + and slated for removal in Python 3.19. + + Before Python 3.13, the ``string`` keyword parameter was not correctly + supported depending on the backend implementation of hash functions. + Prefer passing the initial data as a positional argument for maximum + backwards compatibility. diff --git a/Doc/deprecations/pending-removal-in-3.20.rst b/Doc/deprecations/pending-removal-in-3.20.rst new file mode 100644 index 00000000000000..4e4b2e1d5f8fff --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.20.rst @@ -0,0 +1,32 @@ +Pending removal in Python 3.20 +------------------------------ + +* The ``__version__``, ``version`` and ``VERSION`` attributes have been + deprecated in these standard library modules and will be removed in + Python 3.20. Use :py:data:`sys.version_info` instead. + + - :mod:`argparse` + - :mod:`csv` + - :mod:`ctypes` + - :mod:`!ctypes.macholib` + - :mod:`decimal` (use :data:`decimal.SPEC_VERSION` instead) + - :mod:`http.server` + - :mod:`imaplib` + - :mod:`ipaddress` + - :mod:`json` + - :mod:`logging` (``__date__`` also deprecated) + - :mod:`optparse` + - :mod:`pickle` + - :mod:`platform` + - :mod:`re` + - :mod:`socketserver` + - :mod:`tabnanny` + - :mod:`tkinter.font` + - :mod:`tkinter.ttk` + - :mod:`wsgiref.simple_server` + - :mod:`xml.etree.ElementTree` + - :mod:`!xml.sax.expatreader` + - :mod:`xml.sax.handler` + - :mod:`zlib` + + (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index 5a4502ac08a5f0..301867416701ea 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -4,9 +4,17 @@ Pending removal in future versions The following APIs will be removed in the future, although there is currently no date scheduled for their removal. +* :mod:`argparse`: + + * Nesting argument groups and nesting mutually exclusive + groups are deprecated. + * Passing the undocumented keyword argument *prefix_chars* to + :meth:`~argparse.ArgumentParser.add_argument_group` is now + deprecated. + * The :class:`argparse.FileType` type converter is deprecated. + * :mod:`builtins`: - * ``bool(NotImplemented)``. * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, the single argument signature. @@ -33,22 +41,13 @@ although there is currently no date scheduled for their removal. as a single positional argument. (Contributed by Serhiy Storchaka in :gh:`109218`.) -* :mod:`argparse`: - - * Nesting argument groups and nesting mutually exclusive - groups are deprecated. - * Passing the undocumented keyword argument *prefix_chars* to - :meth:`~argparse.ArgumentParser.add_argument_group` is now - deprecated. - * The :class:`argparse.FileType` type converter is deprecated. - -* :mod:`array`'s ``'u'`` format code (:gh:`57281`) - * :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are deprecated and replaced by :data:`calendar.JANUARY` and :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) +* :mod:`codecs`: use :func:`open` instead of :func:`codecs.open`. (:gh:`133038`) + * :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method instead. @@ -63,7 +62,6 @@ although there is currently no date scheduled for their removal. * :mod:`importlib`: - * ``load_module()`` method: use ``exec_module()`` instead. * :func:`~importlib.util.cache_from_source` *debug_override* parameter is deprecated: use the *optimization* parameter instead. @@ -78,7 +76,7 @@ although there is currently no date scheduled for their removal. * :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use BytesIO and binary mode instead. -* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. +* :mod:`os`: Calling :func:`os.register_at_fork` in a multi-threaded process. * :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is deprecated, use an exception instance. @@ -90,8 +88,6 @@ although there is currently no date scheduled for their removal. underscore. (Contributed by Serhiy Storchaka in :gh:`91760`.) -* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. - * :mod:`shutil`: :func:`~shutil.rmtree`'s *onerror* parameter is deprecated in Python 3.12; use the *onexc* parameter instead. @@ -112,9 +108,6 @@ although there is currently no date scheduled for their removal. * ``ssl.TLSVersion.TLSv1`` * ``ssl.TLSVersion.TLSv1_1`` -* :func:`sysconfig.is_python_build` *check_home* parameter is deprecated and - ignored. - * :mod:`threading` methods: * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. @@ -128,6 +121,11 @@ although there is currently no date scheduled for their removal. * :class:`typing.Text` (:gh:`92332`). +* The internal class ``typing._UnionGenericAlias`` is no longer used to implement + :class:`typing.Union`. To preserve compatibility with users using this private + class, a compatibility shim will be provided until at least Python 3.17. (Contributed by + Jelle Zijlstra in :gh:`105499`.) + * :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value that is not ``None`` from a test case. @@ -145,10 +143,6 @@ although there is currently no date scheduled for their removal. * ``splitvalue()`` * ``to_bytes()`` -* :mod:`urllib.request`: :class:`~urllib.request.URLopener` and - :class:`~urllib.request.FancyURLopener` style of invoking requests is - deprecated. Use newer :func:`~urllib.request.urlopen` functions and methods. - * :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial writes. @@ -157,5 +151,5 @@ although there is currently no date scheduled for their removal. will always return ``True``. Prefer explicit ``len(elem)`` or ``elem is not None`` tests instead. -* :meth:`zipimport.zipimporter.load_module` is deprecated: - use :meth:`~zipimport.zipimporter.exec_module` instead. +* :func:`sys._clear_type_cache` is deprecated: + use :func:`sys._clear_internal_caches` instead. diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index ddde567f6f3efa..098dde39ea597e 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -6,41 +6,10 @@ Building C and C++ Extensions ***************************** -A C extension for CPython is a shared library (e.g. a ``.so`` file on Linux, -``.pyd`` on Windows), which exports an *initialization function*. +A C extension for CPython is a shared library (for example, a ``.so`` file on +Linux, ``.pyd`` on Windows), which exports an *initialization function*. -To be importable, the shared library must be available on :envvar:`PYTHONPATH`, -and must be named after the module name, with an appropriate extension. -When using setuptools, the correct filename is generated automatically. - -The initialization function has the signature: - -.. c:function:: PyObject* PyInit_modulename(void) - -It returns either a fully initialized module, or a :c:type:`PyModuleDef` -instance. See :ref:`initializing-modules` for details. - -.. highlight:: python - -For modules with ASCII-only names, the function must be named -``PyInit_``, with ```` replaced by the name of the -module. When using :ref:`multi-phase-initialization`, non-ASCII module names -are allowed. In this case, the initialization function name is -``PyInitU_``, with ```` encoded using Python's -*punycode* encoding with hyphens replaced by underscores. In Python:: - - def initfunc_name(name): - try: - suffix = b'_' + name.encode('ascii') - except UnicodeEncodeError: - suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') - return b'PyInit' + suffix - -It is possible to export multiple modules from a single shared library by -defining multiple initialization functions. However, importing them requires -using symbolic links or a custom importer, because by default only the -function corresponding to the filename is found. -See the *"Multiple modules in one library"* section in :pep:`489` for details. +See :ref:`extension-modules` for details. .. highlight:: c @@ -51,7 +20,11 @@ See the *"Multiple modules in one library"* section in :pep:`489` for details. Building C and C++ Extensions with setuptools ============================================= -Python 3.12 and newer no longer come with distutils. Please refer to the -``setuptools`` documentation at -https://setuptools.readthedocs.io/en/latest/setuptools.html -to learn more about how build and distribute C/C++ extensions with setuptools. + +Building, packaging and distributing extension modules is best done with +third-party tools, and is out of scope of this document. +One suitable tool is Setuptools, whose documentation can be found at +https://setuptools.pypa.io/en/latest/setuptools.html. + +The :mod:`distutils` module, which was included in the standard library +until Python 3.12, is now maintained as part of Setuptools. diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 20397dc5add5db..cb41889437c8b0 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -196,8 +196,8 @@ interesting part with respect to embedding Python starts with :: After initializing the interpreter, the script is loaded using :c:func:`PyImport_Import`. This routine needs a Python string as its argument, -which is constructed using the :c:func:`PyUnicode_FromString` data conversion -routine. :: +which is constructed using the :c:func:`PyUnicode_DecodeFSDefault` data +conversion routine. :: pFunc = PyObject_GetAttrString(pModule, argv[2]); /* pFunc is a new reference */ @@ -245,21 +245,23 @@ Python extension. For example:: return PyLong_FromLong(numargs); } - static PyMethodDef EmbMethods[] = { + static PyMethodDef emb_module_methods[] = { {"numargs", emb_numargs, METH_VARARGS, "Return the number of arguments received by the process."}, {NULL, NULL, 0, NULL} }; - static PyModuleDef EmbModule = { - PyModuleDef_HEAD_INIT, "emb", NULL, -1, EmbMethods, - NULL, NULL, NULL, NULL + static struct PyModuleDef emb_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "emb", + .m_size = 0, + .m_methods = emb_module_methods, }; static PyObject* PyInit_emb(void) { - return PyModule_Create(&EmbModule); + return PyModuleDef_Init(&emb_module); } Insert the above code just above the :c:func:`main` function. Also, insert the diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index b0493bed75b151..c0066d315d092b 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -3,129 +3,20 @@ .. _extending-intro: -****************************** -Extending Python with C or C++ -****************************** +******************************** +Using the C API: Assorted topics +******************************** -It is quite easy to add new built-in modules to Python, if you know how to -program in C. Such :dfn:`extension modules` can do two things that can't be -done directly in Python: they can implement new built-in object types, and they -can call C library functions and system calls. - -To support extensions, the Python API (Application Programmers Interface) -defines a set of functions, macros and variables that provide access to most -aspects of the Python run-time system. The Python API is incorporated in a C -source file by including the header ``"Python.h"``. - -The compilation of an extension module depends on its intended use as well as on -your system setup; details are given in later chapters. - -.. note:: - - The C extension interface is specific to CPython, and extension modules do - not work on other Python implementations. In many cases, it is possible to - avoid writing C extensions and preserve portability to other implementations. - For example, if your use case is calling C library functions or system calls, - you should consider using the :mod:`ctypes` module or the `cffi - `_ library rather than writing - custom C code. - These modules let you write Python code to interface with C code and are more - portable between implementations of Python than writing and compiling a C - extension module. - - -.. _extending-simpleexample: - -A Simple Example -================ - -Let's create an extension module called ``spam`` (the favorite food of Monty -Python fans...) and let's say we want to create a Python interface to the C -library function :c:func:`system` [#]_. This function takes a null-terminated -character string as argument and returns an integer. We want this function to -be callable from Python as follows: - -.. code-block:: pycon - - >>> import spam - >>> status = spam.system("ls -l") - -Begin by creating a file :file:`spammodule.c`. (Historically, if a module is -called ``spam``, the C file containing its implementation is called -:file:`spammodule.c`; if the module name is very long, like ``spammify``, the -module name can be just :file:`spammify.c`.) - -The first two lines of our file can be:: - - #define PY_SSIZE_T_CLEAN - #include - -which pulls in the Python API (you can add a comment describing the purpose of -the module and a copyright notice if you like). - -.. note:: - - Since Python may define some pre-processor definitions which affect the standard - headers on some systems, you *must* include :file:`Python.h` before any standard - headers are included. - - ``#define PY_SSIZE_T_CLEAN`` was used to indicate that ``Py_ssize_t`` should be - used in some APIs instead of ``int``. - It is not necessary since Python 3.13, but we keep it here for backward compatibility. - See :ref:`arg-parsing-string-and-buffers` for a description of this macro. - -All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or -``PY``, except those defined in standard header files. For convenience, and -since they are used extensively by the Python interpreter, ``"Python.h"`` -includes a few standard header files: ````, ````, -````, and ````. If the latter header file does not exist on -your system, it declares the functions :c:func:`malloc`, :c:func:`free` and -:c:func:`realloc` directly. - -The next thing we add to our module file is the C function that will be called -when the Python expression ``spam.system(string)`` is evaluated (we'll see -shortly how it ends up being called):: - - static PyObject * - spam_system(PyObject *self, PyObject *args) - { - const char *command; - int sts; - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = system(command); - return PyLong_FromLong(sts); - } - -There is a straightforward translation from the argument list in Python (for -example, the single expression ``"ls -l"``) to the arguments passed to the C -function. The C function always has two arguments, conventionally named *self* -and *args*. - -The *self* argument points to the module object for module-level functions; -for a method it would point to the object instance. - -The *args* argument will be a pointer to a Python tuple object containing the -arguments. Each item of the tuple corresponds to an argument in the call's -argument list. The arguments are Python objects --- in order to do anything -with them in our C function we have to convert them to C values. The function -:c:func:`PyArg_ParseTuple` in the Python API checks the argument types and -converts them to C values. It uses a template string to determine the required -types of the arguments as well as the types of the C variables into which to -store the converted values. More about this later. - -:c:func:`PyArg_ParseTuple` returns true (nonzero) if all arguments have the right -type and its components have been stored in the variables whose addresses are -passed. It returns false (zero) if an invalid argument list was passed. In the -latter case it also raises an appropriate exception so the calling function can -return ``NULL`` immediately (as we saw in the example). +The :ref:`tutorial ` walked you through +creating a C API extension module, but left many areas unexplained. +This document looks at several concepts that you'll need to learn +in order to write more complex extensions. .. _extending-errors: -Intermezzo: Errors and Exceptions -================================= +Errors and Exceptions +===================== An important convention throughout the Python interpreter is the following: when a function fails, it should set an exception condition and return an error value @@ -203,31 +94,57 @@ function usually raises :c:data:`PyExc_TypeError`. If you have an argument whos value must be in a particular range or must satisfy other conditions, :c:data:`PyExc_ValueError` is appropriate. -You can also define a new exception that is unique to your module. For this, you -usually declare a static object variable at the beginning of your file:: +You can also define a new exception that is unique to your module. +The simplest way to do this is to declare a static global object variable at +the beginning of the file:: - static PyObject *SpamError; + static PyObject *SpamError = NULL; -and initialize it in your module's initialization function (:c:func:`!PyInit_spam`) -with an exception object:: +and initialize it by calling :c:func:`PyErr_NewException` in the module's +:c:data:`Py_mod_exec` function (:c:func:`!spam_module_exec`):: - PyMODINIT_FUNC - PyInit_spam(void) - { - PyObject *m; + SpamError = PyErr_NewException("spam.error", NULL, NULL); - m = PyModule_Create(&spammodule); - if (m == NULL) - return NULL; +Since :c:data:`!SpamError` is a global variable, it will be overwritten every time +the module is reinitialized, when the :c:data:`Py_mod_exec` function is called. +For now, let's avoid the issue: we will block repeated initialization by raising an +:py:exc:`ImportError`:: + + static PyObject *SpamError = NULL; + + static int + spam_module_exec(PyObject *m) + { + if (SpamError != NULL) { + PyErr_SetString(PyExc_ImportError, + "cannot initialize spam module more than once"); + return -1; + } SpamError = PyErr_NewException("spam.error", NULL, NULL); - if (PyModule_AddObjectRef(m, "error", SpamError) < 0) { - Py_CLEAR(SpamError); - Py_DECREF(m); - return NULL; + if (PyModule_AddObjectRef(m, "SpamError", SpamError) < 0) { + return -1; } - return m; + return 0; + } + + static PyModuleDef_Slot spam_module_slots[] = { + {Py_mod_exec, spam_module_exec}, + {0, NULL} + }; + + static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + .m_size = 0, // non-negative + .m_slots = spam_module_slots, + }; + + PyMODINIT_FUNC + PyInit_spam(void) + { + return PyModuleDef_Init(&spam_module); } Note that the Python name for the exception object is :exc:`!spam.error`. The @@ -242,6 +159,11 @@ needed to ensure that it will not be discarded, causing :c:data:`!SpamError` to become a dangling pointer. Should it become a dangling pointer, C code which raises the exception could cause a core dump or other unintended side effects. +For now, the :c:func:`Py_DECREF` call to remove this reference is missing. +Even when the Python interpreter shuts down, the global :c:data:`!SpamError` +variable will not be garbage-collected. It will "leak". +We did, however, ensure that this will happen at most once per process. + We discuss the use of :c:macro:`PyMODINIT_FUNC` as a function return type later in this sample. @@ -265,207 +187,14 @@ call to :c:func:`PyErr_SetString` as shown below:: } -.. _backtoexample: - -Back to the Example -=================== - -Going back to our example function, you should now be able to understand this -statement:: - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - -It returns ``NULL`` (the error indicator for functions returning object pointers) -if an error is detected in the argument list, relying on the exception set by -:c:func:`PyArg_ParseTuple`. Otherwise the string value of the argument has been -copied to the local variable :c:data:`!command`. This is a pointer assignment and -you are not supposed to modify the string to which it points (so in Standard C, -the variable :c:data:`!command` should properly be declared as ``const char -*command``). - -The next statement is a call to the Unix function :c:func:`system`, passing it -the string we just got from :c:func:`PyArg_ParseTuple`:: - - sts = system(command); - -Our :func:`!spam.system` function must return the value of :c:data:`!sts` as a -Python object. This is done using the function :c:func:`PyLong_FromLong`. :: - - return PyLong_FromLong(sts); - -In this case, it will return an integer object. (Yes, even integers are objects -on the heap in Python!) - -If you have a C function that returns no useful argument (a function returning -:c:expr:`void`), the corresponding Python function must return ``None``. You -need this idiom to do so (which is implemented by the :c:macro:`Py_RETURN_NONE` -macro):: - - Py_INCREF(Py_None); - return Py_None; - -:c:data:`Py_None` is the C name for the special Python object ``None``. It is a -genuine Python object rather than a ``NULL`` pointer, which means "error" in most -contexts, as we have seen. - - -.. _methodtable: - -The Module's Method Table and Initialization Function -===================================================== - -I promised to show how :c:func:`!spam_system` is called from Python programs. -First, we need to list its name and address in a "method table":: - - static PyMethodDef SpamMethods[] = { - ... - {"system", spam_system, METH_VARARGS, - "Execute a shell command."}, - ... - {NULL, NULL, 0, NULL} /* Sentinel */ - }; - -Note the third entry (``METH_VARARGS``). This is a flag telling the interpreter -the calling convention to be used for the C function. It should normally always -be ``METH_VARARGS`` or ``METH_VARARGS | METH_KEYWORDS``; a value of ``0`` means -that an obsolete variant of :c:func:`PyArg_ParseTuple` is used. - -When using only ``METH_VARARGS``, the function should expect the Python-level -parameters to be passed in as a tuple acceptable for parsing via -:c:func:`PyArg_ParseTuple`; more information on this function is provided below. - -The :c:macro:`METH_KEYWORDS` bit may be set in the third field if keyword -arguments should be passed to the function. In this case, the C function should -accept a third ``PyObject *`` parameter which will be a dictionary of keywords. -Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a -function. - -The method table must be referenced in the module definition structure:: - - static struct PyModuleDef spammodule = { - PyModuleDef_HEAD_INIT, - "spam", /* name of module */ - spam_doc, /* module documentation, may be NULL */ - -1, /* size of per-interpreter state of the module, - or -1 if the module keeps state in global variables. */ - SpamMethods - }; - -This structure, in turn, must be passed to the interpreter in the module's -initialization function. The initialization function must be named -:c:func:`!PyInit_name`, where *name* is the name of the module, and should be the -only non-\ ``static`` item defined in the module file:: - - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModule_Create(&spammodule); - } - -Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, -declares any special linkage declarations required by the platform, and for C++ -declares the function as ``extern "C"``. - -When the Python program imports module :mod:`!spam` for the first time, -:c:func:`!PyInit_spam` is called. (See below for comments about embedding Python.) -It calls :c:func:`PyModule_Create`, which returns a module object, and -inserts built-in function objects into the newly created module based upon the -table (an array of :c:type:`PyMethodDef` structures) found in the module definition. -:c:func:`PyModule_Create` returns a pointer to the module object -that it creates. It may abort with a fatal error for -certain errors, or return ``NULL`` if the module could not be initialized -satisfactorily. The init function must return the module object to its caller, -so that it then gets inserted into ``sys.modules``. - -When embedding Python, the :c:func:`!PyInit_spam` function is not called -automatically unless there's an entry in the :c:data:`PyImport_Inittab` table. -To add the module to the initialization table, use :c:func:`PyImport_AppendInittab`, -optionally followed by an import of the module:: - - #define PY_SSIZE_T_CLEAN - #include - - int - main(int argc, char *argv[]) - { - PyStatus status; - PyConfig config; - PyConfig_InitPythonConfig(&config); - - /* Add a built-in module, before Py_Initialize */ - if (PyImport_AppendInittab("spam", PyInit_spam) == -1) { - fprintf(stderr, "Error: could not extend in-built modules table\n"); - exit(1); - } - - /* Pass argv[0] to the Python interpreter */ - status = PyConfig_SetBytesString(&config, &config.program_name, argv[0]); - if (PyStatus_Exception(status)) { - goto exception; - } - - /* Initialize the Python interpreter. Required. - If this step fails, it will be a fatal error. */ - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - goto exception; - } - PyConfig_Clear(&config); - - /* Optionally import the module; alternatively, - import can be deferred until the embedded script - imports it. */ - PyObject *pmodule = PyImport_ImportModule("spam"); - if (!pmodule) { - PyErr_Print(); - fprintf(stderr, "Error: could not import module 'spam'\n"); - } - - // ... use Python C API here ... - - return 0; - - exception: - PyConfig_Clear(&config); - Py_ExitStatusException(status); - } - -.. note:: - - Removing entries from ``sys.modules`` or importing compiled modules into - multiple interpreters within a process (or following a :c:func:`fork` without an - intervening :c:func:`exec`) can create problems for some extension modules. - Extension module authors should exercise caution when initializing internal data - structures. - -A more substantial example module is included in the Python source distribution -as :file:`Modules/xxmodule.c`. This file may be used as a template or simply -read as an example. - -.. note:: - - Unlike our ``spam`` example, ``xxmodule`` uses *multi-phase initialization* - (new in Python 3.5), where a PyModuleDef structure is returned from - ``PyInit_spam``, and creation of the module is left to the import machinery. - For details on multi-phase initialization, see :PEP:`489`. - - .. _compilation: -Compilation and Linkage -======================= +Embedding an extension +====================== -There are two more things to do before you can use your new extension: compiling -and linking it with the Python system. If you use dynamic loading, the details -may depend on the style of dynamic loading your system uses; see the chapters -about building extension modules (chapter :ref:`building`) and additional -information that pertains only to building on Windows (chapter -:ref:`building-on-windows`) for more information about this. - -If you can't use dynamic loading, or if you want to make your module a permanent +If you want to make your module a permanent part of the Python interpreter, you will have to change the configuration setup -and rebuild the interpreter. Luckily, this is very simple on Unix: just place +and rebuild the interpreter. On Unix, place your file (:file:`spammodule.c` for example) in the :file:`Modules/` directory of an unpacked source distribution, add a line to the file :file:`Modules/Setup.local` describing your file: @@ -493,7 +222,7 @@ on the line in the configuration file as well, for instance: Calling Python Functions from C =============================== -So far we have concentrated on making C functions callable from Python. The +The tutorial concentrated on making C functions callable from Python. The reverse is also useful: calling Python functions from C. This is especially the case for libraries that support so-called "callback" functions. If a C interface makes use of callbacks, the equivalent Python often needs to provide a @@ -538,7 +267,7 @@ be part of a module definition:: } This function must be registered with the interpreter using the -:c:macro:`METH_VARARGS` flag; this is described in section :ref:`methodtable`. The +:c:macro:`METH_VARARGS` flag in :c:type:`PyMethodDef.ml_flags`. The :c:func:`PyArg_ParseTuple` function and its arguments are documented in section :ref:`parsetuple`. @@ -633,14 +362,21 @@ the above example, we use :c:func:`Py_BuildValue` to construct the dictionary. : Py_DECREF(result); +.. index:: single: PyArg_ParseTuple (C function) + .. _parsetuple: Extracting Parameters in Extension Functions ============================================ -.. index:: single: PyArg_ParseTuple (C function) +The :ref:`tutorial ` uses a ":c:data:`METH_O`" +function, which is limited to a single Python argument. +If you want more, you can use :c:data:`METH_VARARGS` instead. +With this flag, the C function will receive a :py:class:`tuple` of arguments +instead of a single object. -The :c:func:`PyArg_ParseTuple` function is declared as follows:: +For unpacking the tuple, CPython provides the :c:func:`PyArg_ParseTuple` +function, declared as follows:: int PyArg_ParseTuple(PyObject *arg, const char *format, ...); @@ -650,6 +386,19 @@ whose syntax is explained in :ref:`arg-parsing` in the Python/C API Reference Manual. The remaining arguments must be addresses of variables whose type is determined by the format string. +For example, to receive a single Python :py:class:`str` object and turn it +into a C buffer, you would use ``"s"`` as the format string:: + + const char *command; + if (!PyArg_ParseTuple(args, "s", &command)) { + return NULL; + } + +If an error is detected in the argument list, :c:func:`!PyArg_ParseTuple` +returns ``NULL`` (the error indicator for functions returning object pointers); +your function may return ``NULL``, relying on the exception set by +:c:func:`PyArg_ParseTuple`. + Note that while :c:func:`PyArg_ParseTuple` checks that the Python arguments have the required types, it cannot check the validity of the addresses of C variables passed to the call: if you make mistakes there, your code will probably crash or @@ -660,7 +409,6 @@ Note that any Python object references which are provided to the caller are Some example calls:: - #define PY_SSIZE_T_CLEAN #include :: @@ -730,6 +478,17 @@ Some example calls:: Keyword Parameters for Extension Functions ========================================== +If you also want your function to accept +:term:`keyword arguments `, use the :c:data:`METH_KEYWORDS` +flag in combination with :c:data:`METH_VARARGS`. +(:c:data:`!METH_KEYWORDS` can also be used with other flags; see its +documentation for the allowed combinations.) + +In this case, the C function should accept a third ``PyObject *`` parameter +which will be a dictionary of keywords. +Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a +function. + .. index:: single: PyArg_ParseTupleAndKeywords (C function) The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: @@ -790,20 +549,6 @@ Philbrick (philbrick@hks.com):: {NULL, NULL, 0, NULL} /* sentinel */ }; - static struct PyModuleDef keywdargmodule = { - PyModuleDef_HEAD_INIT, - "keywdarg", - NULL, - -1, - keywdarg_methods - }; - - PyMODINIT_FUNC - PyInit_keywdarg(void) - { - return PyModule_Create(&keywdargmodule); - } - .. _buildvalue: @@ -944,11 +689,11 @@ needed. Ownership of a reference can be transferred. There are three ways to dispose of an owned reference: pass it on, store it, or call :c:func:`Py_DECREF`. Forgetting to dispose of an owned reference creates a memory leak. -It is also possible to :dfn:`borrow` [#]_ a reference to an object. The +It is also possible to :dfn:`borrow` [#borrow]_ a reference to an object. The borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must not hold on to the object longer than the owner from which it was borrowed. Using a borrowed reference after the owner has disposed of it risks using freed -memory and should be avoided completely [#]_. +memory and should be avoided completely [#dont-check-refcount]_. The advantage of borrowing over owning a reference is that you don't need to take care of disposing of the reference on all possible paths through the code @@ -1042,7 +787,14 @@ references to all its items, so when item 1 is replaced, it has to dispose of the original item 1. Now let's suppose the original item 1 was an instance of a user-defined class, and let's further suppose that the class defined a :meth:`!__del__` method. If this class instance has a reference count of 1, -disposing of it will call its :meth:`!__del__` method. +disposing of it will call its :meth:`!__del__` method. Internally, +:c:func:`PyList_SetItem` calls :c:func:`Py_DECREF` on the replaced item, +which invokes replaced item's corresponding +:c:member:`~PyTypeObject.tp_dealloc` function. During +deallocation, :c:member:`~PyTypeObject.tp_dealloc` calls +:c:member:`~PyTypeObject.tp_finalize`, which is mapped to the +:meth:`!__del__` method for class instances (see :pep:`442`). This entire +sequence happens synchronously within the :c:func:`PyList_SetItem` call. Since it is written in Python, the :meth:`!__del__` method can execute arbitrary Python code. Could it perhaps do something to invalidate the reference to @@ -1072,8 +824,9 @@ why his :meth:`!__del__` methods would fail... The second case of problems with a borrowed reference is a variant involving threads. Normally, multiple threads in the Python interpreter can't get in each -other's way, because there is a global lock protecting Python's entire object -space. However, it is possible to temporarily release this lock using the macro +other's way, because there is a :term:`global lock ` +protecting Python's entire object space. +However, it is possible to temporarily release this lock using the macro :c:macro:`Py_BEGIN_ALLOW_THREADS`, and to re-acquire it using :c:macro:`Py_END_ALLOW_THREADS`. This is common around blocking I/O calls, to let other threads use the processor while waiting for the I/O to complete. @@ -1119,7 +872,7 @@ checking. The C function calling mechanism guarantees that the argument list passed to C functions (``args`` in the examples) is never ``NULL`` --- in fact it guarantees -that it is always a tuple [#]_. +that it is always a tuple [#old-calling-convention]_. It is a severe error to ever let a ``NULL`` pointer "escape" to the Python user. @@ -1176,8 +929,8 @@ the module whose functions one wishes to call might not have been loaded yet! Portability therefore requires not to make any assumptions about symbol visibility. This means that all symbols in extension modules should be declared ``static``, except for the module's initialization function, in order to -avoid name clashes with other extension modules (as discussed in section -:ref:`methodtable`). And it means that symbols that *should* be accessible from +avoid name clashes with other extension modules. And it means that symbols +that *should* be accessible from other extension modules must be exported in a different way. Python provides a special mechanism to pass C-level information (pointers) from @@ -1219,8 +972,9 @@ file corresponding to the module provides a macro that takes care of importing the module and retrieving its C API pointers; client modules only have to call this macro before accessing the C API. -The exporting module is a modification of the :mod:`!spam` module from section -:ref:`extending-simpleexample`. The function :func:`!spam.system` does not call +The exporting module is a modification of the :mod:`!spam` module from the +:ref:`tutorial `. +The function :func:`!spam.system` does not call the C library function :c:func:`system` directly, but a function :c:func:`!PySpam_System`, which would of course do something more complicated in reality (such as adding "spam" to every command). This function @@ -1259,20 +1013,15 @@ two more lines must be added:: #include "spammodule.h" The ``#define`` is used to tell the header file that it is being included in the -exporting module, not a client module. Finally, the module's initialization -function must take care of initializing the C API pointer array:: +exporting module, not a client module. Finally, the module's :c:data:`mod_exec +` function must take care of initializing the C API pointer array:: - PyMODINIT_FUNC - PyInit_spam(void) + static int + spam_module_exec(PyObject *m) { - PyObject *m; static void *PySpam_API[PySpam_API_pointers]; PyObject *c_api_object; - m = PyModule_Create(&spammodule); - if (m == NULL) - return NULL; - /* Initialize the C API pointer array */ PySpam_API[PySpam_System_NUM] = (void *)PySpam_System; @@ -1280,11 +1029,10 @@ function must take care of initializing the C API pointer array:: c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { - Py_DECREF(m); - return NULL; + return -1; } - return m; + return 0; } Note that ``PySpam_API`` is declared ``static``; otherwise the pointer @@ -1343,20 +1091,16 @@ like this:: All that a client module must do in order to have access to the function :c:func:`!PySpam_System` is to call the function (or rather macro) -:c:func:`!import_spam` in its initialization function:: +:c:func:`!import_spam` in its :c:data:`mod_exec ` function:: - PyMODINIT_FUNC - PyInit_client(void) + static int + client_module_exec(PyObject *m) { - PyObject *m; - - m = PyModule_Create(&clientmodule); - if (m == NULL) - return NULL; - if (import_spam() < 0) - return NULL; + if (import_spam() < 0) { + return -1; + } /* additional initialization can happen here */ - return m; + return 0; } The main disadvantage of this approach is that the file :file:`spammodule.h` is @@ -1372,15 +1116,14 @@ code distribution). .. rubric:: Footnotes -.. [#] An interface for this function already exists in the standard module :mod:`os` - --- it was chosen as a simple and straightforward example. - -.. [#] The metaphor of "borrowing" a reference is not completely correct: the owner - still has a copy of the reference. +.. [#borrow] The metaphor of "borrowing" a reference is not completely correct: + the owner still has a copy of the reference. -.. [#] Checking that the reference count is at least 1 **does not work** --- the +.. [#dont-check-refcount] Checking that the reference count is at least 1 + **does not work** --- the reference count itself could be in freed memory and may thus be reused for another object! -.. [#] These guarantees don't hold when you use the "old" style calling convention --- +.. [#old-calling-convention] These guarantees don't hold when you use the + "old" style calling convention --- this is still found in much existing code. diff --git a/Doc/extending/first-extension-module.rst b/Doc/extending/first-extension-module.rst new file mode 100644 index 00000000000000..5bde785c49e81e --- /dev/null +++ b/Doc/extending/first-extension-module.rst @@ -0,0 +1,667 @@ +.. highlight:: c + + +.. _extending-simpleexample: +.. _first-extension-module: + +********************************* +Your first C API extension module +********************************* + +This tutorial will take you through creating a simple +Python extension module written in C or C++. + +We will use the low-level Python C API directly. +For easier ways to create extension modules, see +the :ref:`recommended third party tools `. + +The tutorial assumes basic knowledge about Python: you should be able to +define functions in Python code before starting to write them in C. +See :ref:`tutorial-index` for an introduction to Python itself. + +The tutorial should be approachable for anyone who can write a basic C library. +While we will mention several concepts that a C beginner would not be expected +to know, like ``static`` functions or linkage declarations, understanding these +is not necessary for success. + +We will focus on giving you a "feel" of what Python's C API is like. +It will not teach you important concepts, like error handling +and reference counting, which are covered in later chapters. + +We will assume that you use a Unix-like system (including macOS and +Linux), or Windows. +On other systems, you might need to adjust some details -- for example, +a system command name. + +You need to have a suitable C compiler and Python development headers installed. +On Linux, headers are often in a package like ``python3-dev`` +or ``python3-devel``. + +You need to be able to install Python packages. +This tutorial uses `pip `__ (``pip install``), but you +can substitute any tool that can build and install ``pyproject.toml``-based +projects, like `uv `_ (``uv pip install``). +Preferably, have a :ref:`virtual environment ` activated. + + +.. note:: + + This tutorial uses APIs that were added in CPython 3.15. + To create an extension that's compatible with earlier versions of CPython, + please follow an earlier version of this documentation. + + This tutorial uses C syntax added in C11 and C++20. + If your extension needs to be compatible with earlier standards, + please follow tutorials in documentation for Python 3.14 or below. + + +What we'll do +============= + +Let's create an extension module called ``spam`` [#why-spam]_, +which will include a Python interface to the C +standard library function :c:func:`system`. +This function is defined in ``stdlib.h``. +It takes a C string as argument, runs the argument as a system +command, and returns a result value as an integer. +A manual page for :c:func:`system` might summarize it this way:: + + #include + int system(const char *command); + +Note that like many functions in the C standard library, +this function is already exposed in Python. +In production, use :py:func:`os.system` or :py:func:`subprocess.run` +rather than the module you'll write here. + +We want this function to be callable from Python as follows: + +.. code-block:: pycon + + >>> import spam + >>> status = spam.system("whoami") + User Name + >>> status + 0 + +.. note:: + + The system command ``whoami`` prints out your username. + It's useful in tutorials like this one because it has the same name on + both Unix and Windows. + + +Start with the headers +====================== + +Begin by creating a directory for this tutorial, and switching to it +on the command line. +Then, create a file named :file:`spammodule.c` in your directory. +[#why-spammodule]_ + +In this file, we'll include two headers: :file:`Python.h` to pull in +all declarations of the Python C API, and :file:`stdlib.h` for the +:c:func:`system` function. [#stdlib-h]_ + +Add the following lines to :file:`spammodule.c`: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-at: + :end-at: + +Be sure to put :file:`stdlib.h`, and any other standard library includes, +*after* :file:`Python.h`. +On some systems, Python may define some pre-processor definitions +that affect the standard headers. + + +Running your build tool +======================= + +With only the includes in place, your extension won't do anything. +Still, it's a good time to compile it and try to import it. +This will ensure that your build tool works, so that you can make +and test incremental changes as you follow the rest of the text. + +CPython itself does not come with a tool to build extension modules; +it is recommended to use a third-party project for this. +In this tutorial, we'll use `meson-python`_. +(If you want to use another one, see :ref:`first-extension-other-tools`.) + +.. at the time of writing, meson-python has the least overhead for a + simple extension using PyModExport. + Change this if another tool makes things easier. + +``meson-python`` requires defining a "project" using two extra files. + +First, add ``pyproject.toml`` with these contents: + +.. code-block:: toml + + [build-system] + build-backend = 'mesonpy' + requires = ['meson-python'] + + [project] + # Placeholder project information + # (change this before distributing the module) + name = 'sampleproject' + version = '0' + +Then, create ``meson.build`` containing the following: + +.. code-block:: meson + + project('sampleproject', 'c') + + py = import('python').find_installation(pure: false) + + py.extension_module( + 'spam', # name of the importable Python module + 'spammodule.c', # the C source file + install: true, + ) + +.. note:: + + See `meson-python documentation `_ for details on + configuration. + +Now, build install the *project in the current directory* (``.``) via ``pip``: + +.. code-block:: sh + + python -m pip install . + +.. tip:: + + If you don't have ``pip`` installed, run ``python -m ensurepip``, + preferably in a :ref:`virtual environment `. + (Or, if you prefer another tool that can build and install + ``pyproject.toml``-based projects, use that.) + +.. _meson-python: https://mesonbuild.com/meson-python/ +.. _virtual environment: https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments + +Note that you will need to run this command again every time you change your +extension. +Unlike Python, C has an explicit compilation step. + +When your extension is compiled and installed, start Python and try to +import it. +This should fail with the following exception: + +.. code-block:: pycon + + >>> import spam + Traceback (most recent call last): + ... + ImportError: dynamic module does not define module export function (PyModExport_spam or PyInit_spam) + + +Module export hook +================== + +The exception you got when you tried to import the module told you that Python +is looking for a "module export function", also known as a +:ref:`module export hook `. +Let's define one. + +First, add a prototype below the ``#include`` lines: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-after: /// Export hook prototype + :end-before: /// + +.. tip:: + The prototype is not strictly necessary, but some modern compilers emit + warnings without it. + It's generally better to add the prototype than to disable the warning. + +The :c:macro:`PyMODEXPORT_FUNC` macro declares the function's +return type, and adds any special linkage declarations needed +to make the function visible and usable when CPython loads it. + +After the prototype, add the function itself. +For now, make it return ``NULL``: + +.. code-block:: c + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return NULL; + } + +Compile and load the module again. +You should get a different error this time. + +.. code-block:: pycon + + >>> import spam + Traceback (most recent call last): + ... + SystemError: module export hook for module 'spam' failed without setting an exception + +Simply returning ``NULL`` is *not* correct behavior for an export hook, +and CPython complains about it. +That's good -- it means that CPython found the function! +Let's now make it do something useful. + + +The slot table +============== + +Rather than ``NULL``, the export hook should return the information needed to +create a module. +Let's start with the basics: the name and docstring. + +The information should be defined in a ``static`` array of +:c:type:`PyModuleDef_Slot` entries, which are essentially key-value pairs. +Define this array just before your export hook: + +.. code-block:: c + + static PyModuleDef_Slot spam_slots[] = { + {Py_mod_name, "spam"}, + {Py_mod_doc, "A wonderful module with an example function"}, + {0, NULL} + }; + +For both :c:data:`Py_mod_name` and :c:data:`Py_mod_doc`, the values are C +strings -- that is, NUL-terminated, UTF-8 encoded byte arrays. + +Note the zero-filled sentinel entry at the end. +If you forget it, you'll trigger undefined behavior. + +The array is defined as ``static`` -- that is, not visible outside this ``.c`` file. +This will be a common theme. +CPython only needs to access the export hook; all global variables +and all other functions should generally be ``static``, so that they don't +clash with other extensions. + +Return this array from your export hook instead of ``NULL``: + +.. code-block:: c + :emphasize-lines: 4 + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return spam_slots; + } + +Now, recompile and try it out: + +.. code-block:: pycon + + >>> import spam + >>> print(spam) + + +You have an extension module! +Try ``help(spam)`` to see the docstring. + +The next step will be adding a function. + + +.. _backtoexample: + +Exposing a function +=================== + +To expose the :c:func:`system` C function directly to Python, +we'll need to write a layer of glue code to convert arguments from Python +objects to C values, and the C return value back to Python. + +One of the simplest ways to write glue code is a ":c:data:`METH_O`" function, +which takes two Python objects and returns one. +All Python objects -- regardless of the Python type -- are represented in C +as pointers to the :c:type:`PyObject` structure. + +Add such a function above the slots array:: + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + Py_RETURN_NONE; + } + +For now, we ignore the arguments, and use the :c:macro:`Py_RETURN_NONE` +macro, which expands to a ``return`` statement that properly returns +a Python :py:data:`None` object. + +Recompile your extension to make sure you don't have syntax errors. +We haven't yet added ``spam_system`` to the module, so you might get a +warning that ``spam_system`` is unused. + +.. _methodtable: + +Method definitions +------------------ + +To expose the C function to Python, you will need to provide several pieces of +information in a structure called +:c:type:`PyMethodDef` [#why-pymethoddef]_: + +* ``ml_name``: the name of the Python function; +* ``ml_doc``: a docstring; +* ``ml_meth``: the C function to be called; and +* ``ml_flags``: a set of flags describing details like how Python arguments are + passed to the C function. + We'll use :c:data:`METH_O` here -- the flag that matches our + ``spam_system`` function's signature. + +Because modules typically create several functions, these definitions +need to be collected in an array, with a zero-filled sentinel at the end. +Add this array just below the ``spam_system`` function: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-after: /// Module method table + :end-before: /// + +As with module slots, a zero-filled sentinel marks the end of the array. + +Next, we'll add the method to the module. +Add a :c:data:`Py_mod_methods` slot to your :c:type:`PyMethodDef` array: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-after: /// Module slot table + :end-before: /// + :emphasize-lines: 5 + +Recompile your extension again, and test it. +Be sure to restart the Python interpreter, so that ``import spam`` picks +up the new version of the module. + +You should now be able to call the function: + +.. code-block:: pycon + + >>> import spam + >>> print(spam.system) + + >>> print(spam.system('whoami')) + None + +Note that our ``spam.system`` does not yet run the ``whoami`` command; +it only returns ``None``. + +Check that the function accepts exactly one argument, as specified by +the :c:data:`METH_O` flag: + +.. code-block:: pycon + + >>> print(spam.system('too', 'many', 'arguments')) + Traceback (most recent call last): + ... + TypeError: spam.system() takes exactly one argument (3 given) + + +Returning an integer +==================== + +Now, let's take a look at the return value. +Instead of ``None``, we'll want ``spam.system`` to return a number -- that is, +a Python :py:type:`int` object. +Eventually this will be the exit code of a system command, +but let's start with a fixed value, say, ``3``. + +The Python C API provides a function to create a Python :py:type:`int` object +from a C ``int`` value: :c:func:`PyLong_FromLong`. [#why-pylongfromlong]_ + +To call it, replace the ``Py_RETURN_NONE`` with the following 3 lines: + +.. this could be a one-liner, but we want to show the data types here + +.. code-block:: c + :emphasize-lines: 4-6 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + int status = 3; + PyObject *result = PyLong_FromLong(status); + return result; + } + + +Recompile, restart the Python interpreter again, +and check that the function now returns 3: + +.. code-block:: pycon + + >>> import spam + >>> spam.system('whoami') + 3 + + +Accepting a string +================== + +Finally, let's handle the function argument. + +Our C function, :c:func:`!spam_system`, takes two arguments. +The first one, ``PyObject *self``, will be set to the ``spam`` module +object. +This isn't useful in our case, so we'll ignore it. + +The other one, ``PyObject *arg``, will be set to the object that the user +passed from Python. +We expect that it should be a Python string. +In order to use the information in it, we will need +to convert it to a C value -- in this case, a C string (``const char *``). + +There's a slight type mismatch here: Python's :py:class:`str` objects store +Unicode text, but C strings are arrays of bytes. +So, we'll need to *encode* the data, and we'll use the UTF-8 encoding for it. +(UTF-8 might not always be correct for system commands, but it's what +:py:meth:`str.encode` uses by default, +and the C API has special support for it.) + +The function to encode a Python string into a UTF-8 buffer is named +:c:func:`PyUnicode_AsUTF8` [#why-pyunicodeasutf8]_. +Call it like this: + +.. code-block:: c + :emphasize-lines: 4 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + const char *command = PyUnicode_AsUTF8(arg); + int status = 3; + PyObject *result = PyLong_FromLong(status); + return result; + } + +If :c:func:`PyUnicode_AsUTF8` is successful, *command* will point to the +resulting array of bytes. +This buffer is managed by the *arg* object, which means we don't need to free +it, but we must follow some rules: + +* We should only use the buffer inside the ``spam_system`` function. + When ``spam_system`` returns, *arg* and the buffer it manages might be + garbage-collected. +* We must not modify it. This is why we use ``const``. + +If :c:func:`PyUnicode_AsUTF8` was *not* successful, it returns a ``NULL`` +pointer. +When calling *any* Python C API, we always need to handle such error cases. +The way to do this in general is left for later chapters of this documentation. +For now, be assured that we are already handling errors from +:c:func:`PyLong_FromLong` correctly. + +For the :c:func:`PyUnicode_AsUTF8` call, the correct way to handle errors is +returning ``NULL`` from ``spam_system``. +Add an ``if`` block for this: + + +.. code-block:: c + :emphasize-lines: 5-7 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + const char *command = PyUnicode_AsUTF8(arg); + if (command == NULL) { + return NULL; + } + int status = 3; + PyObject *result = PyLong_FromLong(status); + return result; + } + +That's it for the setup. +Now, all that is left is calling the C library function :c:func:`system` with +the ``char *`` buffer, and using its result instead of the ``3``: + +.. code-block:: c + :emphasize-lines: 8 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + const char *command = PyUnicode_AsUTF8(arg); + if (command == NULL) { + return NULL; + } + int status = system(command); + PyObject *result = PyLong_FromLong(status); + return result; + } + +Compile your module, restart Python, and test. +This time, you should see your username -- the output of the ``whoami`` +system command: + +.. code-block:: pycon + + >>> import spam + >>> result = spam.system('whoami') + User Name + >>> result + 0 + +You might also want to test error cases: + +.. code-block:: pycon + + >>> import spam + >>> result = spam.system('nonexistent-command') + sh: line 1: nonexistent-command: command not found + >>> result + 32512 + + >>> spam.system(3) + Traceback (most recent call last): + ... + TypeError: bad argument type for built-in operation + + +The result +========== + + +Congratulations! +You have written a complete Python C API extension module, +and completed this tutorial! + +Here is the entire source file, for your convenience: + +.. _extending-spammodule-source: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-at: /// + + +.. _first-extension-other-tools: + +Appendix: Other build tools +=========================== + +You should be able to follow this tutorial -- except the +*Running your build tool* section itself -- with a build tool other +than ``meson-python``. + +The Python Packaging User Guide has a `list of recommended tools `_; +be sure to choose one for the C language. + + +Workaround for missing PyInit function +-------------------------------------- + +If your build tool output complains about missing ``PyInit_spam``, +add the following function to your module for now: + +.. code-block:: c + + // A workaround + void *PyInit_spam(void) { return NULL; } + +This is a shim for an old-style :ref:`initialization function `, +which was required in extension modules for CPython 3.14 and below. +Current CPython does not need it, but some build tools may still assume that +all extension modules need to define it. + +If you use this workaround, you will get the exception +``SystemError: initialization of spam failed without raising an exception`` +instead of +``ImportError: dynamic module does not define module export function``. + + +Compiling directly +------------------ + +Using a third-party build tool is heavily recommended, +as it will take care of various details of your platform and Python +installation, of naming the resulting extension, and, later, of distributing +your work. + +If you are building an extension for as *specific* system, or for yourself +only, you might instead want to run your compiler directly. +The way to do this is system-specific; be prepared for issues you will need +to solve yourself. + +Linux +^^^^^ + +On Linux, the Python development package may include a ``python3-config`` +command that prints out the required compiler flags. +If you use it, check that it corresponds to the CPython interpreter you'll use +to load the module. +Then, start with the following command: + +.. code-block:: sh + + gcc --shared $(python3-config --cflags --ldflags) spammodule.c -o spam.so + +This should generate a ``spam.so`` file that you need to put in a directory +on :py:attr:`sys.path`. + + +.. rubric:: Footnotes + +.. [#why-spam] ``spam`` is the favorite food of Monty Python fans... +.. [#why-spammodule] The source file name is entirely up to you, + though some tools can be picky about the ``.c`` extension. + This tutorial uses the traditional ``*module.c`` suffix. + Some people would just use :file:`spam.c` to implement a module + named ``spam``, + projects where Python isn't the primary language might use ``py_spam.c``, + and so on. +.. [#stdlib-h] Including :file:`stdlib.h` is technically not necessary, + since :file:`Python.h` includes it and + :ref:`several other standard headers ` for its own use + or for backwards compatibility. + However, it is good practice to explicitly include what you need. +.. [#why-pymethoddef] The :c:type:`!PyMethodDef` structure is also used + to create methods of classes, so there's no separate + ":c:type:`!PyFunctionDef`". +.. [#why-pylongfromlong] The name :c:func:`PyLong_FromLong` + might not seem obvious. + ``PyLong`` refers to a the Python :py:class:`int`, which was originally + called ``long``; the ``FromLong`` refers to the C ``long`` (or ``long int``) + type. +.. [#why-pyunicodeasutf8] Here, ``PyUnicode`` refers to the original name of + the Python :py:class:`str` class: ``unicode``. diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst index 01b4df6d44acff..c0c494c3059d99 100644 --- a/Doc/extending/index.rst +++ b/Doc/extending/index.rst @@ -5,15 +5,17 @@ ################################################## This document describes how to write modules in C or C++ to extend the Python -interpreter with new modules. Those modules can not only define new functions -but also new object types and their methods. The document also describes how +interpreter with new modules. Those modules can do what Python code does -- +define functions, object types and methods -- but also interact with native +libraries or achieve better performance by avoiding the overhead of an +interpreter. The document also describes how to embed the Python interpreter in another application, for use as an extension language. Finally, it shows how to compile and link extension modules so that they can be loaded dynamically (at run time) into the interpreter, if the underlying operating system supports this feature. -This document assumes basic knowledge about Python. For an informal -introduction to the language, see :ref:`tutorial-index`. :ref:`reference-index` +This document assumes basic knowledge about C and Python. For an informal +introduction to Python, see :ref:`tutorial-index`. :ref:`reference-index` gives a more formal definition of the language. :ref:`library-index` documents the existing object types, functions and modules (both built-in and written in Python) that give the language its wide application range. @@ -21,43 +23,75 @@ Python) that give the language its wide application range. For a detailed description of the whole Python/C API, see the separate :ref:`c-api-index`. +To support extensions, Python's C API (Application Programmers Interface) +defines a set of functions, macros and variables that provide access to most +aspects of the Python run-time system. The Python API is incorporated in a C +source file by including the header ``"Python.h"``. + +.. note:: + + The C extension interface is specific to CPython, and extension modules do + not work on other Python implementations. In many cases, it is possible to + avoid writing C extensions and preserve portability to other implementations. + For example, if your use case is calling C library functions or system calls, + you should consider using the :mod:`ctypes` module or the `cffi + `_ library rather than writing + custom C code. + These modules let you write Python code to interface with C code and are more + portable between implementations of Python than writing and compiling a C + extension module. + + +.. toctree:: + :hidden: + + first-extension-module.rst + extending.rst + newtypes_tutorial.rst + newtypes.rst + building.rst + windows.rst + embedding.rst + Recommended third party tools ============================= -This guide only covers the basic tools for creating extensions provided -as part of this version of CPython. Third party tools like -`Cython `_, `cffi `_, -`SWIG `_ and `Numba `_ -offer both simpler and more sophisticated approaches to creating C and C++ -extensions for Python. +This document only covers the basic tools for creating extensions provided +as part of this version of CPython. Some :ref:`third party tools +` offer both simpler and more sophisticated approaches to creating +C and C++ extensions for Python. + +While this document is aimed at extension authors, it should also be helpful to +the authors of such tools. +For example, the tutorial module can serve as a simple test case for a build +tool or sample expected output of a code generator. -.. seealso:: - `Python Packaging User Guide: Binary Extensions `_ - The Python Packaging User Guide not only covers several available - tools that simplify the creation of binary extensions, but also - discusses the various reasons why creating an extension module may be - desirable in the first place. +C API Tutorial +============== +This tutorial describes how to write a simple module in C or C++, +using the Python C API -- that is, using the basic tools provided +as part of this version of CPython. -Creating extensions without third party tools -============================================= + +#. :ref:`first-extension-module` + + +Guides for intermediate topics +============================== This section of the guide covers creating C and C++ extensions without assistance from third party tools. It is intended primarily for creators of those tools, rather than being a recommended way to create your own C extensions. -.. toctree:: - :maxdepth: 2 - :numbered: - - extending.rst - newtypes_tutorial.rst - newtypes.rst - building.rst - windows.rst +* :ref:`extending-intro` +* :ref:`defining-new-types` +* :ref:`new-types-topics` +* :ref:`building` +* :ref:`building-on-windows` Embedding the CPython runtime in a larger application ===================================================== @@ -67,8 +101,4 @@ interpreter as the main application, it is desirable to instead embed the CPython runtime inside a larger application. This section covers some of the details involved in doing that successfully. -.. toctree:: - :maxdepth: 2 - :numbered: - - embedding.rst +* :ref:`embedding` diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 7f57a3a6aac0ed..26085b5cebd3ad 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -70,22 +70,24 @@ object itself needs to be freed here as well. Here is an example of this function:: static void - newdatatype_dealloc(newdatatypeobject *obj) + newdatatype_dealloc(PyObject *op) { - free(obj->obj_UnderlyingDatatypePtr); - Py_TYPE(obj)->tp_free((PyObject *)obj); + newdatatypeobject *self = (newdatatypeobject *) op; + free(self->obj_UnderlyingDatatypePtr); + Py_TYPE(self)->tp_free(self); } If your type supports garbage collection, the destructor should call :c:func:`PyObject_GC_UnTrack` before clearing any member fields:: static void - newdatatype_dealloc(newdatatypeobject *obj) + newdatatype_dealloc(PyObject *op) { - PyObject_GC_UnTrack(obj); - Py_CLEAR(obj->other_obj); + newdatatypeobject *self = (newdatatypeobject *) op; + PyObject_GC_UnTrack(op); + Py_CLEAR(self->other_obj); ... - Py_TYPE(obj)->tp_free((PyObject *)obj); + Py_TYPE(self)->tp_free(self); } .. index:: @@ -117,17 +119,19 @@ done. This can be done using the :c:func:`PyErr_Fetch` and PyErr_Fetch(&err_type, &err_value, &err_traceback); cbresult = PyObject_CallNoArgs(self->my_callback); - if (cbresult == NULL) - PyErr_WriteUnraisable(self->my_callback); - else + if (cbresult == NULL) { + PyErr_WriteUnraisable(self->my_callback); + } + else { Py_DECREF(cbresult); + } /* This restores the saved exception state */ PyErr_Restore(err_type, err_value, err_traceback); Py_DECREF(self->my_callback); } - Py_TYPE(obj)->tp_free((PyObject*)self); + Py_TYPE(self)->tp_free(self); } .. note:: @@ -168,10 +172,11 @@ representation of the instance for which it is called. Here is a simple example:: static PyObject * - newdatatype_repr(newdatatypeobject *obj) + newdatatype_repr(PyObject *op) { + newdatatypeobject *self = (newdatatypeobject *) op; return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:%d}}", - obj->obj_UnderlyingDatatypePtr->size); + self->obj_UnderlyingDatatypePtr->size); } If no :c:member:`~PyTypeObject.tp_repr` handler is specified, the interpreter will supply a @@ -188,10 +193,11 @@ used instead. Here is a simple example:: static PyObject * - newdatatype_str(newdatatypeobject *obj) + newdatatype_str(PyObject *op) { + newdatatypeobject *self = (newdatatypeobject *) op; return PyUnicode_FromFormat("Stringified_newdatatype{{size:%d}}", - obj->obj_UnderlyingDatatypePtr->size); + self->obj_UnderlyingDatatypePtr->size); } @@ -329,16 +335,16 @@ method of a class would be called. Here is an example:: static PyObject * - newdatatype_getattr(newdatatypeobject *obj, char *name) + newdatatype_getattr(PyObject *op, char *name) { - if (strcmp(name, "data") == 0) - { - return PyLong_FromLong(obj->data); + newdatatypeobject *self = (newdatatypeobject *) op; + if (strcmp(name, "data") == 0) { + return PyLong_FromLong(self->data); } PyErr_Format(PyExc_AttributeError, "'%.100s' object has no attribute '%.400s'", - Py_TYPE(obj)->tp_name, name); + Py_TYPE(self)->tp_name, name); return NULL; } @@ -349,7 +355,7 @@ example that simply raises an exception; if this were really all you wanted, the :c:member:`~PyTypeObject.tp_setattr` handler should be set to ``NULL``. :: static int - newdatatype_setattr(newdatatypeobject *obj, char *name, PyObject *v) + newdatatype_setattr(PyObject *op, char *name, PyObject *v) { PyErr_Format(PyExc_RuntimeError, "Read-only attribute: %s", name); return -1; @@ -379,8 +385,10 @@ Here is a sample implementation, for a datatype that is considered equal if the size of an internal pointer is equal:: static PyObject * - newdatatype_richcmp(newdatatypeobject *obj1, newdatatypeobject *obj2, int op) + newdatatype_richcmp(PyObject *lhs, PyObject *rhs, int op) { + newdatatypeobject *obj1 = (newdatatypeobject *) lhs; + newdatatypeobject *obj2 = (newdatatypeobject *) rhs; PyObject *result; int c, size1, size2; @@ -399,8 +407,7 @@ size of an internal pointer is equal:: case Py_GE: c = size1 >= size2; break; } result = c ? Py_True : Py_False; - Py_INCREF(result); - return result; + return Py_NewRef(result); } @@ -439,12 +446,14 @@ This function, if you choose to provide it, should return a hash number for an instance of your data type. Here is a simple example:: static Py_hash_t - newdatatype_hash(newdatatypeobject *obj) + newdatatype_hash(PyObject *op) { + newdatatypeobject *self = (newdatatypeobject *) op; Py_hash_t result; - result = obj->some_size + 32767 * obj->some_number; - if (result == -1) - result = -2; + result = self->some_size + 32767 * self->some_number; + if (result == -1) { + result = -2; + } return result; } @@ -478,8 +487,9 @@ This function takes three arguments: Here is a toy ``tp_call`` implementation:: static PyObject * - newdatatype_call(newdatatypeobject *obj, PyObject *args, PyObject *kwds) + newdatatype_call(PyObject *op, PyObject *args, PyObject *kwds) { + newdatatypeobject *self = (newdatatypeobject *) op; PyObject *result; const char *arg1; const char *arg2; @@ -490,7 +500,7 @@ Here is a toy ``tp_call`` implementation:: } result = PyUnicode_FromFormat( "Returning -- value: [%d] arg1: [%s] arg2: [%s] arg3: [%s]\n", - obj->obj_UnderlyingDatatypePtr->size, + self->obj_UnderlyingDatatypePtr->size, arg1, arg2, arg3); return result; } @@ -550,6 +560,8 @@ For an object to be weakly referenceable, the extension type must set the field. The legacy :c:member:`~PyTypeObject.tp_weaklistoffset` field should be left as zero. +If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. + Concretely, here is how the statically declared type object would look:: static PyTypeObject TrivialType = { @@ -563,12 +575,12 @@ The only further addition is that ``tp_dealloc`` needs to clear any weak references (by calling :c:func:`PyObject_ClearWeakRefs`):: static void - Trivial_dealloc(TrivialObject *self) + Trivial_dealloc(PyObject *op) { /* Clear weakrefs first before calling any destructors */ - PyObject_ClearWeakRefs((PyObject *) self); + PyObject_ClearWeakRefs(op); /* ... remainder of destruction code omitted for brevity ... */ - Py_TYPE(self)->tp_free((PyObject *) self); + Py_TYPE(op)->tp_free(op); } diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index bcf938f117d148..3bbee33bd50698 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -55,8 +55,10 @@ from the previous chapter. This file defines three things: #. How the :class:`!Custom` **type** behaves: this is the ``CustomType`` struct, which defines a set of flags and function pointers that the interpreter inspects when specific operations are requested. -#. How to initialize the :mod:`!custom` module: this is the ``PyInit_custom`` - function and the associated ``custommodule`` struct. +#. How to define and execute the :mod:`!custom` module: this is the + ``PyInit_custom`` function and the associated ``custom_module`` struct for + defining the module, and the ``custom_module_exec`` function to set up + a fresh module object. The first bit is:: @@ -171,18 +173,18 @@ implementation provided by the API function :c:func:`PyType_GenericNew`. :: .tp_new = PyType_GenericNew, Everything else in the file should be familiar, except for some code in -:c:func:`!PyInit_custom`:: +:c:func:`!custom_module_exec`:: - if (PyType_Ready(&CustomType) < 0) - return; + if (PyType_Ready(&CustomType) < 0) { + return -1; + } This initializes the :class:`!Custom` type, filling in a number of members to the appropriate default values, including :c:member:`~PyObject.ob_type` that we initially set to ``NULL``. :: if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; + return -1; } This adds the type to the module dictionary. This allows us to create @@ -250,16 +252,17 @@ Because we now have data to manage, we have to be more careful about object allocation and deallocation. At a minimum, we need a deallocation method:: static void - Custom_dealloc(CustomObject *self) + Custom_dealloc(PyObject *op) { + CustomObject *self = (CustomObject *) op; Py_XDECREF(self->first); Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject *) self); + Py_TYPE(self)->tp_free(self); } which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member:: - .tp_dealloc = (destructor) Custom_dealloc, + .tp_dealloc = Custom_dealloc, This method first clears the reference counts of the two Python attributes. :c:func:`Py_XDECREF` correctly handles the case where its argument is @@ -270,11 +273,31 @@ the object's type might not be :class:`!CustomType`, because the object may be an instance of a subclass. .. note:: - The explicit cast to ``destructor`` above is needed because we defined - ``Custom_dealloc`` to take a ``CustomObject *`` argument, but the ``tp_dealloc`` - function pointer expects to receive a ``PyObject *`` argument. Otherwise, - the compiler will emit a warning. This is object-oriented polymorphism, - in C! + + The explicit cast to ``CustomObject *`` above is needed because we defined + ``Custom_dealloc`` to take a ``PyObject *`` argument, as the ``tp_dealloc`` + function pointer expects to receive a ``PyObject *`` argument. + By assigning to the ``tp_dealloc`` slot of a type, we declare + that it can only be called with instances of our ``CustomObject`` + class, so the cast to ``(CustomObject *)`` is safe. + This is object-oriented polymorphism, in C! + + In existing code, or in previous versions of this tutorial, + you might see similar functions take a pointer to the subtype + object structure (``CustomObject*``) directly, like this:: + + Custom_dealloc(CustomObject *self) + { + Py_XDECREF(self->first); + Py_XDECREF(self->last); + Py_TYPE(self)->tp_free((PyObject *) self); + } + ... + .tp_dealloc = (destructor) Custom_dealloc, + + This does the same thing on all architectures that CPython + supports, but according to the C standard, it invokes + undefined behavior. We want to make sure that the first and last names are initialized to empty strings, so we provide a ``tp_new`` implementation:: @@ -352,8 +375,9 @@ We also define an initialization function which accepts arguments to provide initial values for our instance:: static int - Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) + Custom_init(PyObject *op, PyObject *args, PyObject *kwds) { + CustomObject *self = (CustomObject *) op; static char *kwlist[] = {"first", "last", "number", NULL}; PyObject *first = NULL, *last = NULL, *tmp; @@ -379,7 +403,7 @@ initial values for our instance:: by filling the :c:member:`~PyTypeObject.tp_init` slot. :: - .tp_init = (initproc) Custom_init, + .tp_init = Custom_init, The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the :meth:`~object.__init__` method. It is used to initialize an object after it's @@ -403,8 +427,8 @@ the new attribute values. We might be tempted, for example to assign the But this would be risky. Our type doesn't restrict the type of the ``first`` member, so it could be any kind of object. It could have a destructor that causes code to be executed that tries to access the -``first`` member; or that destructor could release the -:term:`Global interpreter Lock ` and let arbitrary code run in other +``first`` member; or that destructor could detach the +:term:`thread state ` and let arbitrary code run in other threads that accesses and modifies our object. To be paranoid and protect ourselves against this possibility, we almost @@ -413,8 +437,8 @@ don't we have to do this? * when we absolutely know that the reference count is greater than 1; -* when we know that deallocation of the object [#]_ will neither release - the :term:`GIL` nor cause any calls back into our type's code; +* when we know that deallocation of the object [#]_ will neither detach + the :term:`thread state ` nor cause any calls back into our type's code; * when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc` handler on a type which doesn't support cyclic garbage collection [#]_. @@ -451,8 +475,9 @@ We define a single method, :meth:`!Custom.name`, that outputs the objects name a concatenation of the first and last names. :: static PyObject * - Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) + Custom_name(PyObject *op, PyObject *Py_UNUSED(dummy)) { + CustomObject *self = (CustomObject *) op; if (self->first == NULL) { PyErr_SetString(PyExc_AttributeError, "first"); return NULL; @@ -486,7 +511,7 @@ Now that we've defined the method, we need to create an array of method definitions:: static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, + {"name", Custom_name, METH_NOARGS, "Return the name, combining the first and last name" }, {NULL} /* Sentinel */ @@ -543,15 +568,17 @@ we'll use custom getter and setter functions. Here are the functions for getting and setting the :attr:`!first` attribute:: static PyObject * - Custom_getfirst(CustomObject *self, void *closure) + Custom_getfirst(PyObject *op, void *closure) { + CustomObject *self = (CustomObject *) op; Py_INCREF(self->first); return self->first; } static int - Custom_setfirst(CustomObject *self, PyObject *value, void *closure) + Custom_setfirst(PyObject *op, PyObject *value, void *closure) { + CustomObject *self = (CustomObject *) op; PyObject *tmp; if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); @@ -583,9 +610,9 @@ new value is not a string. We create an array of :c:type:`PyGetSetDef` structures:: static PyGetSetDef Custom_getsetters[] = { - {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, + {"first", Custom_getfirst, Custom_setfirst, "first name", NULL}, - {"last", (getter) Custom_getlast, (setter) Custom_setlast, + {"last", Custom_getlast, Custom_setlast, "last name", NULL}, {NULL} /* Sentinel */ }; @@ -609,8 +636,9 @@ We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only allow strings [#]_ to be passed:: static int - Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) + Custom_init(PyObject *op, PyObject *args, PyObject *kwds) { + CustomObject *self = (CustomObject *) op; static char *kwlist[] = {"first", "last", "number", NULL}; PyObject *first = NULL, *last = NULL, *tmp; @@ -689,8 +717,9 @@ First, the traversal method lets the cyclic GC know about subobjects that could participate in cycles:: static int - Custom_traverse(CustomObject *self, visitproc visit, void *arg) + Custom_traverse(PyObject *op, visitproc visit, void *arg) { + CustomObject *self = (CustomObject *) op; int vret; if (self->first) { vret = visit(self->first, arg); @@ -716,8 +745,9 @@ functions. With :c:func:`Py_VISIT`, we can minimize the amount of boilerplate in ``Custom_traverse``:: static int - Custom_traverse(CustomObject *self, visitproc visit, void *arg) + Custom_traverse(PyObject *op, visitproc visit, void *arg) { + CustomObject *self = (CustomObject *) op; Py_VISIT(self->first); Py_VISIT(self->last); return 0; @@ -731,8 +761,9 @@ Second, we need to provide a method for clearing any subobjects that can participate in cycles:: static int - Custom_clear(CustomObject *self) + Custom_clear(PyObject *op) { + CustomObject *self = (CustomObject *) op; Py_CLEAR(self->first); Py_CLEAR(self->last); return 0; @@ -765,11 +796,11 @@ Here is our reimplemented deallocator using :c:func:`PyObject_GC_UnTrack` and ``Custom_clear``:: static void - Custom_dealloc(CustomObject *self) + Custom_dealloc(PyObject *op) { - PyObject_GC_UnTrack(self); - Custom_clear(self); - Py_TYPE(self)->tp_free((PyObject *) self); + PyObject_GC_UnTrack(op); + (void)Custom_clear(op); + Py_TYPE(op)->tp_free(op); } Finally, we add the :c:macro:`Py_TPFLAGS_HAVE_GC` flag to the class flags:: @@ -825,9 +856,10 @@ When a Python object is a :class:`!SubList` instance, its ``PyObject *`` pointer can be safely cast to both ``PyListObject *`` and ``SubListObject *``:: static int - SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) + SubList_init(PyObject *op, PyObject *args, PyObject *kwds) { - if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) + SubListObject *self = (SubListObject *) op; + if (PyList_Type.tp_init(op, args, kwds) < 0) return -1; self->state = 0; return 0; @@ -845,27 +877,22 @@ but let the base class handle it by calling its own :c:member:`~PyTypeObject.tp_ The :c:type:`PyTypeObject` struct supports a :c:member:`~PyTypeObject.tp_base` specifying the type's concrete base class. Due to cross-platform compiler issues, you can't fill that field directly with a reference to -:c:type:`PyList_Type`; it should be done later in the module initialization +:c:type:`PyList_Type`; it should be done in the :c:data:`Py_mod_exec` function:: - PyMODINIT_FUNC - PyInit_sublist(void) + static int + sublist_module_exec(PyObject *m) { - PyObject* m; SubListType.tp_base = &PyList_Type; - if (PyType_Ready(&SubListType) < 0) - return NULL; - - m = PyModule_Create(&sublistmodule); - if (m == NULL) - return NULL; + if (PyType_Ready(&SubListType) < 0) { + return -1; + } if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { - Py_DECREF(m); - return NULL; + return -1; } - return m; + return 0; } Before calling :c:func:`PyType_Ready`, the type structure must have the diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst index e366d6cb9f79e3..a97c6182553c30 100644 --- a/Doc/extending/windows.rst +++ b/Doc/extending/windows.rst @@ -96,6 +96,13 @@ gives you access to spam's names, but does not create a separate copy. On Unix, linking with a library is more like ``from spam import *``; it does create a separate copy. +.. c:macro:: Py_NO_LINK_LIB + + Turn off the implicit, ``#pragma``-based linkage with the Python + library, performed inside CPython header files. + + .. versionadded:: 3.14 + .. _win-dlls: @@ -108,21 +115,46 @@ Using DLLs in Practice Windows Python is built in Microsoft Visual C++; using other compilers may or may not work. The rest of this section is MSVC++ specific. -When creating DLLs in Windows, you must pass :file:`pythonXY.lib` to the linker. -To build two DLLs, spam and ni (which uses C functions found in spam), you could -use these commands:: +When creating DLLs in Windows, you can use the CPython library in two ways: + +1. By default, inclusion of :file:`PC/pyconfig.h` directly or via + :file:`Python.h` triggers an implicit, configure-aware link with the + library. The header file chooses :file:`pythonXY_d.lib` for Debug, + :file:`pythonXY.lib` for Release, and :file:`pythonX.lib` for Release with + the :ref:`Limited API ` enabled. + + To build two DLLs, spam and ni (which uses C functions found in spam), you + could use these commands:: + + cl /LD /I/python/include spam.c + cl /LD /I/python/include ni.c spam.lib + + The first command created three files: :file:`spam.obj`, :file:`spam.dll` + and :file:`spam.lib`. :file:`Spam.dll` does not contain any Python + functions (such as :c:func:`PyArg_ParseTuple`), but it does know how to find + the Python code thanks to the implicitly linked :file:`pythonXY.lib`. + + The second command created :file:`ni.dll` (and :file:`.obj` and + :file:`.lib`), which knows how to find the necessary functions from spam, + and also from the Python executable. + +2. Manually by defining :c:macro:`Py_NO_LINK_LIB` macro before including + :file:`Python.h`. You must pass :file:`pythonXY.lib` to the linker. + + To build two DLLs, spam and ni (which uses C functions found in spam), you + could use these commands:: - cl /LD /I/python/include spam.c ../libs/pythonXY.lib - cl /LD /I/python/include ni.c spam.lib ../libs/pythonXY.lib + cl /LD /DPy_NO_LINK_LIB /I/python/include spam.c ../libs/pythonXY.lib + cl /LD /DPy_NO_LINK_LIB /I/python/include ni.c spam.lib ../libs/pythonXY.lib -The first command created three files: :file:`spam.obj`, :file:`spam.dll` and -:file:`spam.lib`. :file:`Spam.dll` does not contain any Python functions (such -as :c:func:`PyArg_ParseTuple`), but it does know how to find the Python code -thanks to :file:`pythonXY.lib`. + The first command created three files: :file:`spam.obj`, :file:`spam.dll` + and :file:`spam.lib`. :file:`Spam.dll` does not contain any Python + functions (such as :c:func:`PyArg_ParseTuple`), but it does know how to find + the Python code thanks to :file:`pythonXY.lib`. -The second command created :file:`ni.dll` (and :file:`.obj` and :file:`.lib`), -which knows how to find the necessary functions from spam, and also from the -Python executable. + The second command created :file:`ni.dll` (and :file:`.obj` and + :file:`.lib`), which knows how to find the necessary functions from spam, + and also from the Python executable. Not every identifier is exported to the lookup table. If you want any other modules (including Python) to be able to see your identifiers, you have to say diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index e2710fab9cf800..ac0aa81e56bb07 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -420,10 +420,12 @@ strings representing the files in the current directory. Functions which operate on this output would generally not break if you added another file or two to the directory. -Tuples are immutable, meaning that once a tuple has been created, you can't -replace any of its elements with a new value. Lists are mutable, meaning that -you can always change a list's elements. Only immutable elements can be used as -dictionary keys, and hence only tuples and not lists can be used as keys. +Tuples are :term:`immutable`, meaning that once a tuple has been created, you can't +replace any of its elements with a new value. Lists are :term:`mutable`, meaning that +you can always change a list's elements. Only :term:`hashable` objects can +be used as dictionary keys. Most immutable types are hashable, which is why +tuples, but not lists, can be used as keys. Note, however, that a tuple is +only hashable if all of its elements are hashable. How are lists implemented in CPython? @@ -589,9 +591,9 @@ exhaustive test suites that exercise every line of code in a module. An appropriate testing discipline can help build large complex applications in Python as well as having interface specifications would. In fact, it can be better because an interface specification cannot test certain properties of a -program. For example, the :meth:`!list.append` method is expected to add new elements +program. For example, the :meth:`list.append` method is expected to add new elements to the end of some internal list; an interface specification cannot test that -your :meth:`!list.append` implementation will actually do this correctly, but it's +your :meth:`list.append` implementation will actually do this correctly, but it's trivial to check this property in a test suite. Writing test suites is very helpful, and you might want to design your code to diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 3147fda7c37124..1d5abed2317b0c 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -37,24 +37,9 @@ Writing C is hard; are there any alternatives? ---------------------------------------------- There are a number of alternatives to writing your own C extensions, depending -on what you're trying to do. - -.. XXX make sure these all work - -`Cython `_ and its relative `Pyrex -`_ are compilers -that accept a slightly modified form of Python and generate the corresponding -C code. Cython and Pyrex make it possible to write an extension without having -to learn Python's C API. - -If you need to interface to some C or C++ library for which no Python extension -currently exists, you can try wrapping the library's data types and functions -with a tool such as `SWIG `_. `SIP -`__, `CXX -`_ `Boost -`_, or `Weave -`_ are also -alternatives for wrapping C++ libraries. +on what you're trying to do. :ref:`Recommended third party tools ` +offer both simpler and more sophisticated approaches to creating C and C++ +extensions for Python. How can I execute arbitrary Python statements from C? diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 578777d7f23621..698f2117ff5c64 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -186,7 +186,7 @@ How do I get documentation on Python? ------------------------------------- The standard documentation for the current stable version of Python is available -at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are +at https://docs.python.org/3/. EPUB, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. The documentation is written in reStructuredText and processed by `the Sphinx diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index fa7b22bde1dc6f..138a5ca7a7516f 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -454,7 +454,7 @@ There are two factors that produce this result: (the list), and both ``x`` and ``y`` refer to it. 2) Lists are :term:`mutable`, which means that you can change their content. -After the call to :meth:`!append`, the content of the mutable object has +After the call to :meth:`~sequence.append`, the content of the mutable object has changed from ``[]`` to ``[10]``. Since both the variables refer to the same object, using either name accesses the modified value ``[10]``. @@ -986,8 +986,8 @@ There are various techniques. f() -Is there an equivalent to Perl's chomp() for removing trailing newlines from strings? -------------------------------------------------------------------------------------- +Is there an equivalent to Perl's ``chomp()`` for removing trailing newlines from strings? +----------------------------------------------------------------------------------------- You can use ``S.rstrip("\r\n")`` to remove all occurrences of any line terminator from the end of the string ``S`` without removing other trailing @@ -1005,8 +1005,8 @@ Since this is typically only desired when reading text one line at a time, using ``S.rstrip()`` this way works well. -Is there a scanf() or sscanf() equivalent? ------------------------------------------- +Is there a ``scanf()`` or ``sscanf()`` equivalent? +-------------------------------------------------- Not as such. @@ -1020,8 +1020,8 @@ For more complicated input parsing, regular expressions are more powerful than C's ``sscanf`` and better suited for the task. -What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean? -------------------------------------------------------------------- +What does ``UnicodeDecodeError`` or ``UnicodeEncodeError`` error mean? +---------------------------------------------------------------------- See the :ref:`unicode-howto`. @@ -1036,7 +1036,7 @@ A raw string ending with an odd number of backslashes will escape the string's q >>> r'C:\this\will\not\work\' File "", line 1 r'C:\this\will\not\work\' - ^ + ^ SyntaxError: unterminated string literal (detected at line 1) There are several workarounds for this. One is to use regular strings and double @@ -1226,13 +1226,13 @@ This converts the list into a set, thereby removing duplicates, and then back into a list. -How do you remove multiple items from a list --------------------------------------------- +How do you remove multiple items from a list? +--------------------------------------------- As with removing duplicates, explicitly iterating in reverse with a delete condition is one possibility. However, it is easier and faster to use slice replacement with an implicit or explicit forward iteration. -Here are three variations.:: +Here are three variations:: mylist[:] = filter(keep_function, mylist) mylist[:] = (x for x in mylist if keep_condition) @@ -1397,9 +1397,9 @@ To see why this happens, you need to know that (a) if an object implements an :meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented assignment is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`!extend` on the list -and returning the list. That's why we say that for lists, ``+=`` is a -"shorthand" for :meth:`!list.extend`:: +and (b) for lists, :meth:`!__iadd__` is equivalent to calling +:meth:`~sequence.extend` on the list and returning the list. +That's why we say that for lists, ``+=`` is a "shorthand" for :meth:`list.extend`:: >>> a_list = [] >>> a_list += [1] @@ -1868,15 +1868,15 @@ object identity is assured. Generally, there are three circumstances where identity is guaranteed: 1) Assignments create new names but do not change object identity. After the -assignment ``new = old``, it is guaranteed that ``new is old``. + assignment ``new = old``, it is guaranteed that ``new is old``. 2) Putting an object in a container that stores object references does not -change object identity. After the list assignment ``s[0] = x``, it is -guaranteed that ``s[0] is x``. + change object identity. After the list assignment ``s[0] = x``, it is + guaranteed that ``s[0] is x``. 3) If an object is a singleton, it means that only one instance of that object -can exist. After the assignments ``a = None`` and ``b = None``, it is -guaranteed that ``a is b`` because ``None`` is a singleton. + can exist. After the assignments ``a = None`` and ``b = None``, it is + guaranteed that ``a is b`` because ``None`` is a singleton. In most other circumstances, identity tests are inadvisable and equality tests are preferred. In particular, identity tests should not be used to check @@ -1906,28 +1906,30 @@ In the standard library code, you will see several common patterns for correctly using identity tests: 1) As recommended by :pep:`8`, an identity test is the preferred way to check -for ``None``. This reads like plain English in code and avoids confusion with -other objects that may have boolean values that evaluate to false. + for ``None``. This reads like plain English in code and avoids confusion + with other objects that may have boolean values that evaluate to false. 2) Detecting optional arguments can be tricky when ``None`` is a valid input -value. In those situations, you can create a singleton sentinel object -guaranteed to be distinct from other objects. For example, here is how -to implement a method that behaves like :meth:`dict.pop`:: + value. In those situations, you can create a singleton sentinel object + guaranteed to be distinct from other objects. For example, here is how + to implement a method that behaves like :meth:`dict.pop`: + + .. code-block:: python - _sentinel = object() + _sentinel = object() - def pop(self, key, default=_sentinel): - if key in self: - value = self[key] - del self[key] - return value - if default is _sentinel: - raise KeyError(key) - return default + def pop(self, key, default=_sentinel): + if key in self: + value = self[key] + del self[key] + return value + if default is _sentinel: + raise KeyError(key) + return default 3) Container implementations sometimes need to augment equality tests with -identity tests. This prevents the code from being confused by objects such as -``float('NaN')`` that are not equal to themselves. + identity tests. This prevents the code from being confused by objects + such as ``float('NaN')`` that are not equal to themselves. For example, here is the implementation of :meth:`!collections.abc.Sequence.__contains__`:: diff --git a/Doc/faq/python-video-icon.png b/Doc/faq/python-video-icon.png deleted file mode 100644 index 265da50c7b38fc..00000000000000 Binary files a/Doc/faq/python-video-icon.png and /dev/null differ diff --git a/Doc/glossary.rst b/Doc/glossary.rst index f67f3ecad0bc40..68035c2dfb57d4 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -21,7 +21,9 @@ Glossary right delimiters (parentheses, square brackets, curly braces or triple quotes), or after specifying a decorator. - * The :const:`Ellipsis` built-in constant. + .. index:: single: ...; ellipsis literal + + * The three dots form of the :ref:`Ellipsis ` object. abstract base class Abstract base classes complement :term:`duck-typing` by @@ -107,7 +109,7 @@ Glossary statements. asynchronous generator iterator - An object created by a :term:`asynchronous generator` function. + An object created by an :term:`asynchronous generator` function. This is an :term:`asynchronous iterator` which when called using the :meth:`~object.__anext__` method returns an awaitable object which will execute @@ -115,7 +117,7 @@ Glossary :keyword:`yield` expression. Each :keyword:`yield` temporarily suspends processing, remembering the - location execution state (including local variables and pending + execution state (including local variables and pending try-statements). When the *asynchronous generator iterator* effectively resumes with another awaitable returned by :meth:`~object.__anext__`, it picks up where it left off. See :pep:`492` and :pep:`525`. @@ -132,6 +134,36 @@ Glossary iterator's :meth:`~object.__anext__` method until it raises a :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. + atomic operation + An operation that appears to execute as a single, indivisible step: no + other thread can observe it half-done, and its effects become visible all + at once. Python does not guarantee that high-level statements are atomic + (for example, ``x += 1`` performs multiple bytecode operations and is not + atomic). Atomicity is only guaranteed where explicitly documented. See + also :term:`race condition` and :term:`data race`. + + attached thread state + + A :term:`thread state` that is active for the current OS thread. + + When a :term:`thread state` is attached, the OS thread has + access to the full Python C API and can safely invoke the + bytecode interpreter. + + Unless a function explicitly notes otherwise, attempting to call + the C API without an attached thread state will result in a fatal + error or undefined behavior. A thread state can be attached and detached + explicitly by the user through the C API, or implicitly by the runtime, + including during blocking C calls and by the bytecode interpreter in between + calls. + + On most builds of Python, having an attached thread state implies that the + caller holds the :term:`GIL` for the current interpreter, so only + one OS thread can have an attached thread state at a given moment. In + :term:`free-threaded ` builds of Python, threads can concurrently + hold an attached thread state, allowing for true parallelism of the bytecode + interpreter. + attribute A value associated with an object which is usually referenced by name using dotted expressions. @@ -265,6 +297,22 @@ Glossary advanced mathematical feature. If you're not aware of a need for them, it's almost certain you can safely ignore them. + concurrency + The ability of a computer program to perform multiple tasks at the same + time. Python provides libraries for writing programs that make use of + different forms of concurrency. :mod:`asyncio` is a library for dealing + with asynchronous tasks and coroutines. :mod:`threading` provides + access to operating system threads and :mod:`multiprocessing` to + operating system processes. Multi-core processors can execute threads and + processes on different CPU cores at the same time (see + :term:`parallelism`). + + concurrent modification + When multiple threads modify shared data at the same time. Concurrent + modification without proper synchronization can cause + :term:`race conditions `, and might also trigger a + :term:`data race `, data corruption, or both. + context This term has different meanings depending on where and how it is used. Some common meanings: @@ -333,6 +381,34 @@ Glossary tasks (see :mod:`asyncio`) associate each task with a context which becomes the current context whenever the task starts or resumes execution. + cyclic isolate + A subgroup of one or more objects that reference each other in a reference + cycle, but are not referenced by objects outside the group. The goal of + the :term:`cyclic garbage collector ` is to identify these groups and break the reference + cycles so that the memory can be reclaimed. + + data race + A situation where multiple threads access the same memory location + concurrently, at least one of the accesses is a write, and the threads + do not use any synchronization to control their access. Data races + lead to :term:`non-deterministic` behavior and can cause data corruption. + Proper use of :term:`locks ` and other :term:`synchronization primitives + ` prevents data races. Note that data races + can only happen in native code, but that :term:`native code` might be + exposed in a Python API. See also :term:`race condition` and + :term:`thread-safe`. + + deadlock + A situation in which two or more tasks (threads, processes, or coroutines) + wait indefinitely for each other to release resources or complete actions, + preventing any from making progress. For example, if thread A holds lock + 1 and waits for lock 2, while thread B holds lock 2 and waits for lock 1, + both threads will wait indefinitely. In Python this often arises from + acquiring multiple locks in conflicting orders or from circular + join/await dependencies. Deadlocks can be avoided by always acquiring + multiple :term:`locks ` in a consistent order. See also + :term:`lock` and :term:`reentrant`. + decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for @@ -407,6 +483,11 @@ Glossary with :term:`abstract base classes `.) Instead, it typically employs :func:`hasattr` tests or :term:`EAFP` programming. + dunder + An informal short-hand for "double underscore", used when talking about a + :term:`special method`. For example, ``__init__`` is often pronounced + "dunder init". + EAFP Easier to ask for forgiveness than permission. This common Python coding style assumes the existence of valid keys or attributes and catches @@ -434,7 +515,8 @@ Glossary core and with user code. f-string - String literals prefixed with ``'f'`` or ``'F'`` are commonly called + f-strings + String literals prefixed with ``f`` or ``F`` are commonly called "f-strings" which is short for :ref:`formatted string literals `. See also :pep:`498`. @@ -564,7 +646,7 @@ Glossary An object created by a :term:`generator` function. Each :keyword:`yield` temporarily suspends processing, remembering the - location execution state (including local variables and pending + execution state (including local variables and pending try-statements). When the *generator iterator* resumes, it picks up where it left off (in contrast to functions which start fresh on every invocation). @@ -622,6 +704,18 @@ Glossary multi-threaded applications and makes it easier to use multi-core CPUs efficiently. For more details, see :pep:`703`. + In prior versions of Python's C API, a function might declare that it + requires the GIL to be held in order to use it. This refers to having an + :term:`attached thread state`. + + global state + Data that is accessible throughout a program, such as module-level + variables, class variables, or C static variables in :term:`extension modules + `. In multi-threaded programs, global state shared + between threads typically requires synchronization to avoid + :term:`race conditions ` and + :term:`data races `. + hash-based pyc A bytecode cache file that uses the hash rather than the last-modified time of the corresponding source file to determine its validity. See @@ -658,12 +752,17 @@ Glossary and therefore it is never deallocated while the interpreter is running. For example, :const:`True` and :const:`None` are immortal in CPython. + Immortal objects can be identified via :func:`sys._is_immortal`, or + via :c:func:`PyUnstable_IsImmortal` in the C API. + immutable An object with a fixed value. Immutable objects include numbers, strings and tuples. Such an object cannot be altered. A new object has to be created if a different value has to be stored. They play an important role in places where a constant hash value is needed, for example as a key - in a dictionary. + in a dictionary. Immutable objects are inherently :term:`thread-safe` + because their state cannot be modified after creation, eliminating concerns + about improperly synchronized :term:`concurrent modification`. import path A list of locations (or :term:`path entries `) that are @@ -716,7 +815,7 @@ Glossary iterables include all sequence types (such as :class:`list`, :class:`str`, and :class:`tuple`) and some non-sequence types like :class:`dict`, :term:`file objects `, and objects of any classes you define - with an :meth:`~iterator.__iter__` method or with a + with an :meth:`~object.__iter__` method or with a :meth:`~object.__getitem__` method that implements :term:`sequence` semantics. @@ -753,8 +852,9 @@ Glossary CPython does not consistently apply the requirement that an iterator define :meth:`~iterator.__iter__`. - And also please note that the free-threading CPython does not guarantee - the thread-safety of iterator operations. + And also please note that :term:`free-threaded ` + CPython does not guarantee :term:`thread-safe` behavior of iterator + operations. key function @@ -770,7 +870,7 @@ Glossary :func:`itertools.groupby`. There are several ways to create a key function. For example. the - :meth:`str.lower` method can serve as a key function for case insensitive + :meth:`str.casefold` method can serve as a key function for case insensitive sorts. Alternatively, a key function can be built from a :keyword:`lambda` expression such as ``lambda r: (r[0], r[2])``. Also, :func:`operator.attrgetter`, :func:`operator.itemgetter`, and @@ -792,10 +892,15 @@ Glossary :keyword:`if` statements. In a multi-threaded environment, the LBYL approach can risk introducing a - race condition between "the looking" and "the leaping". For example, the - code, ``if key in mapping: return mapping[key]`` can fail if another + :term:`race condition` between "the looking" and "the leaping". For example, + the code, ``if key in mapping: return mapping[key]`` can fail if another thread removes *key* from *mapping* after the test, but before the lookup. - This issue can be solved with locks or by using the EAFP approach. + This issue can be solved with :term:`locks ` or by using the + :term:`EAFP` approach. See also :term:`thread-safe`. + + lexical analyzer + + Formal name for the *tokenizer*; see :term:`token`. list A built-in Python :term:`sequence`. Despite its name it is more akin @@ -810,10 +915,25 @@ Glossary clause is optional. If omitted, all elements in ``range(256)`` are processed. + lock + A :term:`synchronization primitive` that allows only one thread at a + time to access a shared resource. A thread must acquire a lock before + accessing the protected resource and release it afterward. If a thread + attempts to acquire a lock that is already held by another thread, it + will block until the lock becomes available. Python's :mod:`threading` + module provides :class:`~threading.Lock` (a basic lock) and + :class:`~threading.RLock` (a :term:`reentrant` lock). Locks are used + to prevent :term:`race conditions ` and ensure + :term:`thread-safe` access to shared data. Alternative design patterns + to locks exist such as queues, producer/consumer patterns, and + thread-local state. See also :term:`deadlock`, and :term:`reentrant`. + loader - An object that loads a module. It must define a method named - :meth:`load_module`. A loader is typically returned by a - :term:`finder`. See also: + An object that loads a module. + It must define the :meth:`!exec_module` and :meth:`!create_module` methods + to implement the :class:`~importlib.abc.Loader` interface. + A loader is typically returned by a :term:`finder`. + See also: * :ref:`finders-and-loaders` * :class:`importlib.abc.Loader` @@ -893,8 +1013,11 @@ Glossary See :term:`method resolution order`. mutable - Mutable objects can change their value but keep their :func:`id`. See - also :term:`immutable`. + An :term:`object` with state that is allowed to change during the course + of the program. In multi-threaded programs, mutable objects that are + shared between threads require careful synchronization to avoid + :term:`race conditions `. See also :term:`immutable`, + :term:`thread-safe`, and :term:`concurrent modification`. named tuple The term "named tuple" applies to any type or class that inherits from @@ -934,13 +1057,25 @@ Glossary modules, respectively. namespace package - A :pep:`420` :term:`package` which serves only as a container for - subpackages. Namespace packages may have no physical representation, + A :term:`package` which serves only as a container for subpackages. + Namespace packages may have no physical representation, and specifically are not like a :term:`regular package` because they have no ``__init__.py`` file. + Namespace packages allow several individually installable packages to have a common parent package. + Otherwise, it is recommended to use a :term:`regular package`. + + For more information, see :pep:`420` and :ref:`reference-namespace-package`. + See also :term:`module`. + native code + Code that is compiled to machine instructions and runs directly on the + processor, as opposed to code that is interpreted or runs in a virtual + machine. In the context of Python, native code typically refers to + C, C++, Rust or Fortran code in :term:`extension modules ` + that can be called from Python. See also :term:`extension module`. + nested scope The ability to refer to a variable in an enclosing definition. For instance, a function defined inside another function can refer to @@ -957,6 +1092,15 @@ Glossary properties, :meth:`~object.__getattribute__`, class methods, and static methods. + non-deterministic + Behavior where the outcome of a program can vary between executions with + the same inputs. In multi-threaded programs, non-deterministic behavior + often results from :term:`race conditions ` where the + relative timing or interleaving of threads affects the result. + Proper synchronization using :term:`locks ` and other + :term:`synchronization primitives ` helps + ensure deterministic behavior. + object Any data with state (attributes or value) and defined behavior (methods). Also the ultimate base class of any :term:`new-style @@ -971,6 +1115,15 @@ Glossary applied to all scopes, only those relying on a known set of local and nonlocal variable names are restricted to optimized scopes. + optional module + An :term:`extension module` that is part of the :term:`standard library`, + but may be absent in some builds of :term:`CPython`, + usually due to missing third-party libraries or because the module + is not available for a given platform. + + See :ref:`optional-module-requirements` for a list of optional modules + that require third-party libraries. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a @@ -978,6 +1131,16 @@ Glossary See also :term:`regular package` and :term:`namespace package`. + parallelism + Executing multiple operations at the same time (e.g. on multiple CPU + cores). In Python builds with the + :term:`global interpreter lock (GIL) `, only one + thread runs Python bytecode at a time, so taking advantage of multiple + CPU cores typically involves multiple processes + (e.g. :mod:`multiprocessing`) or native extensions that release the GIL. + In :term:`free-threaded ` Python, multiple Python threads + can run Python code simultaneously on different cores. + parameter A named entity in a :term:`function` (or method) definition that specifies an :term:`argument` (or in some cases, arguments) that the @@ -1152,6 +1315,18 @@ Glossary >>> email.mime.text.__name__ 'email.mime.text' + race condition + A condition of a program where the its behavior + depends on the relative timing or ordering of events, particularly in + multi-threaded programs. Race conditions can lead to + :term:`non-deterministic` behavior and bugs that are difficult to + reproduce. A :term:`data race` is a specific type of race condition + involving unsynchronized access to shared memory. The :term:`LBYL` + coding style is particularly susceptible to race conditions in + multi-threaded code. Using :term:`locks ` and other + :term:`synchronization primitives ` + helps prevent race conditions. + reference count The number of references to an object. When the reference count of an object drops to zero, it is deallocated. Some objects are @@ -1162,12 +1337,36 @@ Glossary :func:`sys.getrefcount` function to return the reference count for a particular object. + In :term:`CPython`, reference counts are not considered to be stable + or well-defined values; the number of references to an object, and how + that number is affected by Python code, may be different between + versions. + regular package A traditional :term:`package`, such as a directory containing an ``__init__.py`` file. See also :term:`namespace package`. + reentrant + A property of a function or :term:`lock` that allows it to be called or + acquired multiple times by the same thread without causing errors or a + :term:`deadlock`. + + For functions, reentrancy means the function can be safely called again + before a previous invocation has completed, which is important when + functions may be called recursively or from signal handlers. Thread-unsafe + functions may be :term:`non-deterministic` if they're called reentrantly in a + multithreaded program. + + For locks, Python's :class:`threading.RLock` (reentrant lock) is + reentrant, meaning a thread that already holds the lock can acquire it + again without blocking. In contrast, :class:`threading.Lock` is not + reentrant - attempting to acquire it twice from the same thread will cause + a deadlock. + + See also :term:`lock` and :term:`deadlock`. + REPL An acronym for the "read–eval–print loop", another name for the :term:`interactive` interpreter shell. @@ -1192,8 +1391,9 @@ Glossary The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just :meth:`~object.__getitem__` and :meth:`~object.__len__`, adding - :meth:`!count`, :meth:`!index`, :meth:`~object.__contains__`, and - :meth:`~object.__reversed__`. Types that implement this expanded + :meth:`~sequence.count`, :meth:`~sequence.index`, + :meth:`~object.__contains__`, and :meth:`~object.__reversed__`. + Types that implement this expanded interface can be registered explicitly using :func:`~abc.ABCMeta.register`. For more documentation on sequence methods generally, see @@ -1234,6 +1434,16 @@ Glossary and ending with double underscores. Special methods are documented in :ref:`specialnames`. + standard library + The collection of :term:`packages `, :term:`modules ` + and :term:`extension modules ` distributed as a part + of the official Python interpreter package. The exact membership of the + collection may vary based on platform, available system libraries, or + other criteria. Documentation can be found at :ref:`library-index`. + + See also :data:`sys.stdlib_module_names` for a list of all possible + standard library module names. + statement A statement is part of a suite (a "block" of code). A statement is either an :term:`expression` or one of several constructs with a keyword, such @@ -1244,6 +1454,9 @@ Glossary issues such as incorrect types. See also :term:`type hints ` and the :mod:`typing` module. + stdlib + An abbreviation of :term:`standard library`. + strong reference In Python's C API, a strong reference is a reference to an object which is owned by the code holding the reference. The strong @@ -1258,6 +1471,24 @@ Glossary See also :term:`borrowed reference`. + synchronization primitive + A basic building block for coordinating (synchronizing) the execution of + multiple threads to ensure :term:`thread-safe` access to shared resources. + Python's :mod:`threading` module provides several synchronization primitives + including :class:`~threading.Lock`, :class:`~threading.RLock`, + :class:`~threading.Semaphore`, :class:`~threading.Condition`, + :class:`~threading.Event`, and :class:`~threading.Barrier`. Additionally, + the :mod:`queue` module provides multi-producer, multi-consumer queues + that are especially useful in multithreaded programs. These + primitives help prevent :term:`race conditions ` and + coordinate thread execution. See also :term:`lock`. + + t-string + t-strings + String literals prefixed with ``t`` or ``T`` are commonly called + "t-strings" which is short for + :ref:`template string literals `. + text encoding A string in Python is a sequence of Unicode code points (in range ``U+0000``--``U+10FFFF``). To store or transfer a string, it needs to be @@ -1281,6 +1512,53 @@ Glossary See also :term:`binary file` for a file object able to read and write :term:`bytes-like objects `. + thread state + + The information used by the :term:`CPython` runtime to run in an OS thread. + For example, this includes the current exception, if any, and the + state of the bytecode interpreter. + + Each thread state is bound to a single OS thread, but threads may have + many thread states available. At most, one of them may be + :term:`attached ` at once. + + An :term:`attached thread state` is required to call most + of Python's C API, unless a function explicitly documents otherwise. + The bytecode interpreter only runs under an attached thread state. + + Each thread state belongs to a single interpreter, but each interpreter + may have many thread states, including multiple for the same OS thread. + Thread states from multiple interpreters may be bound to the same + thread, but only one can be :term:`attached ` in + that thread at any given moment. + + See :ref:`Thread State and the Global Interpreter Lock ` for more + information. + + thread-safe + A module, function, or class that behaves correctly when used by multiple + threads concurrently. Thread-safe code uses appropriate + :term:`synchronization primitives ` like + :term:`locks ` to protect shared mutable state, or is designed + to avoid shared mutable state entirely. In the + :term:`free-threaded ` build, built-in types like + :class:`dict`, :class:`list`, and :class:`set` use internal locking + to make many operations thread-safe, although thread safety is not + necessarily guaranteed. Code that is not thread-safe may experience + :term:`race conditions ` and :term:`data races ` + when used in multi-threaded programs. + + token + + A small unit of source code, generated by the + :ref:`lexical analyzer ` (also called the *tokenizer*). + Names, numbers, strings, operators, + newlines and similar are represented by tokens. + + The :mod:`tokenize` module exposes Python's lexical analyzer. + The :mod:`token` module contains information on the various types + of tokens. + triple-quoted string A string which is bound by three instances of either a quotation mark (") or an apostrophe ('). While they don't provide any functionality @@ -1369,6 +1647,11 @@ Glossary A computer defined entirely in software. Python's virtual machine executes the :term:`bytecode` emitted by the bytecode compiler. + walrus operator + A light-hearted way to refer to the :ref:`assignment expression + ` operator ``:=`` because it looks a bit like a + walrus if you turn your head. + Zen of Python Listing of Python design principles and philosophies that are helpful in understanding and using the language. The listing can be found by typing diff --git a/Doc/howto/a-conceptual-overview-of-asyncio.rst b/Doc/howto/a-conceptual-overview-of-asyncio.rst new file mode 100644 index 00000000000000..3adfedbf410ecc --- /dev/null +++ b/Doc/howto/a-conceptual-overview-of-asyncio.rst @@ -0,0 +1,619 @@ +.. _a-conceptual-overview-of-asyncio: + +**************************************** +A conceptual overview of :mod:`!asyncio` +**************************************** + +This :ref:`HOWTO ` article seeks to help you build a sturdy mental +model of how :mod:`asyncio` fundamentally works, helping you understand the +how and why behind the recommended patterns. + +You might be curious about some key :mod:`!asyncio` concepts. +By the end of this article, you'll be able to comfortably answer these questions: + +- What's happening behind the scenes when an object is awaited? +- How does :mod:`!asyncio` differentiate between a task which doesn't need + CPU time (such as a network request or file read) as opposed to a task that + does (such as computing n-factorial)? +- How to write an asynchronous variant of an operation, such as + an async sleep or database request. + +.. seealso:: + + * The `guide `_ that inspired this HOWTO article, by Alexander Nordin. + * This in-depth `YouTube tutorial series `_ on + ``asyncio`` created by Python core team member, Łukasz Langa. + * `500 Lines or Less: A Web Crawler With asyncio Coroutines `_ by A. + Jesse Jiryu Davis and Guido van Rossum. + +-------------------------------------------- +A conceptual overview part 1: the high-level +-------------------------------------------- + +In part 1, we'll cover the main, high-level building blocks of :mod:`!asyncio`: +the event loop, coroutine functions, coroutine objects, tasks, and ``await``. + +========== +Event loop +========== + +Everything in :mod:`!asyncio` happens relative to the event loop. +It's the star of the show, but prefers to work behind the scenes, managing +and coordinating resources. +It's like an orchestra conductor. +Some power is explicitly granted to it, but a lot of its ability to get things +done comes from the respect and cooperation of its band members. + +In more technical terms, the event loop contains a collection of jobs to be run. +Some jobs are added directly by you, and some indirectly by :mod:`!asyncio`. +The event loop takes a job from its backlog of work and invokes it (or "gives +it control"), similar to calling a function, and then that job runs. +Once it pauses or completes, it returns control to the event loop. +The event loop will then select another job from its pool and invoke it. +You can *roughly* think of the collection of jobs as a queue: jobs are added and +then processed one at a time, generally (but not always) in order. +This process repeats indefinitely, with the event loop cycling endlessly +onwards. +If there are no more jobs pending execution, the event loop is smart enough to +rest and avoid needlessly wasting CPU cycles, and will come back when there's +more work to be done, such as when I/O operations complete or timers expire. + +Effective execution relies on jobs sharing well and cooperating; a greedy job +could hog control and leave the other jobs to starve, rendering the overall +event loop approach rather useless. + +:: + + import asyncio + + # This creates an event loop and indefinitely cycles through + # its collection of jobs. + event_loop = asyncio.new_event_loop() + event_loop.run_forever() + +===================================== +Asynchronous functions and coroutines +===================================== + +This is a basic, boring Python function:: + + def hello_printer(): + print( + "Hi, I am a lowly, simple printer, though I have all I " + "need in life -- \nfresh paper and my dearly beloved octopus " + "partner in crime." + ) + +Calling a regular function invokes its logic or body:: + + >>> hello_printer() + Hi, I am a lowly, simple printer, though I have all I need in life -- + fresh paper and my dearly beloved octopus partner in crime. + +The :ref:`async def `, as opposed to just a plain ``def``, makes +this an asynchronous function (or "coroutine function"). +Calling it creates and returns a :ref:`coroutine ` object. + +:: + + async def loudmouth_penguin(magic_number: int): + print( + "I am a super special talking penguin. Far cooler than that printer. " + f"By the way, my lucky number is: {magic_number}." + ) + +Calling the async function, ``loudmouth_penguin``, does not execute the print statement; +instead, it creates a coroutine object:: + + >>> loudmouth_penguin(magic_number=3) + + +The terms "coroutine function" and "coroutine object" are often conflated +as coroutine. +That can be confusing! +In this article, coroutine specifically refers to a coroutine object, or more +precisely, an instance of :data:`types.CoroutineType` (native coroutine). +Note that coroutines can also exist as instances of +:class:`collections.abc.Coroutine` -- a distinction that matters for type +checking. + +A coroutine represents the function's body or logic. +A coroutine has to be explicitly started; again, merely creating the coroutine +does not start it. +Notably, the coroutine can be paused and resumed at various points within the +function's body. +That pausing and resuming ability is what allows for asynchronous behavior! + +Coroutines and coroutine functions were built by leveraging the functionality +of :term:`generators ` and +:term:`generator functions `. +Recall, a generator function is a function that :keyword:`yield`\s, like this +one:: + + def get_random_number(): + # This would be a bad random number generator! + print("Hi") + yield 1 + print("Hello") + yield 7 + print("Howdy") + yield 4 + ... + +Similar to a coroutine function, calling a generator function does not run it. +Instead, it creates a generator object:: + + >>> get_random_number() + + +You can proceed to the next ``yield`` of a generator by using the +built-in function :func:`next`. +In other words, the generator runs, then pauses. +For example:: + + >>> generator = get_random_number() + >>> next(generator) + Hi + 1 + >>> next(generator) + Hello + 7 + +===== +Tasks +===== + +Roughly speaking, :ref:`tasks ` are coroutines (not coroutine +functions) tied to an event loop. +A task also maintains a list of callback functions whose importance will become +clear in a moment when we discuss :keyword:`await`. + +Creating a task automatically schedules it for execution (by adding a +callback to run it in the event loop's to-do list, that is, collection of jobs). +The recommended way to create tasks is via :func:`asyncio.create_task`. + +:mod:`!asyncio` automatically associates tasks with the event loop for you. +This automatic association was purposely designed into :mod:`!asyncio` for +the sake of simplicity. +Without it, you'd have to keep track of the event loop object and pass it to +any coroutine function that wants to create tasks, adding redundant clutter +to your code. + +:: + + coroutine = loudmouth_penguin(magic_number=5) + # This creates a Task object and schedules its execution via the event loop. + task = asyncio.create_task(coroutine) + +Earlier, we manually created the event loop and set it to run forever. +In practice, it's recommended to use (and common to see) :func:`asyncio.run`, +which takes care of managing the event loop and ensuring the provided +coroutine finishes before advancing. +For example, many async programs follow this setup:: + + import asyncio + + async def main(): + # Perform all sorts of wacky, wild asynchronous things... + ... + + if __name__ == "__main__": + asyncio.run(main()) + # The program will not reach the following print statement until the + # coroutine main() finishes. + print("coroutine main() is done!") + +It's important to be aware that the task itself is not added to the event loop, +only a callback to the task is. +This matters if the task object you created is garbage collected before it's +called by the event loop. +For example, consider this program: + +.. code-block:: + :linenos: + + async def hello(): + print("hello!") + + async def main(): + asyncio.create_task(hello()) + # Other asynchronous instructions which run for a while + # and cede control to the event loop... + ... + + asyncio.run(main()) + +Because there's no reference to the task object created on line 5, it *might* +be garbage collected before the event loop invokes it. +Later instructions in the coroutine ``main()`` hand control back to the event +loop so it can invoke other jobs. +When the event loop eventually tries to run the task, it might fail and +discover the task object does not exist! +This can also happen even if a coroutine keeps a reference to a task but +completes before that task finishes. +When the coroutine exits, local variables go out of scope and may be subject +to garbage collection. +In practice, ``asyncio`` and Python's garbage collector work pretty hard to +ensure this sort of thing doesn't happen. +But that's no reason to be reckless! + +===== +await +===== + +:keyword:`await` is a Python keyword that's commonly used in one of two +different ways:: + + await task + await coroutine + +In a crucial way, the behavior of ``await`` depends on the type of object +being awaited. + +^^^^^^^^^^^^^^ +Awaiting tasks +^^^^^^^^^^^^^^ + +Awaiting a task will cede control from the current task or coroutine to +the event loop. +In the process of relinquishing control, a few important things happen. +We'll use the following code example to illustrate:: + + async def plant_a_tree(): + dig_the_hole_task = asyncio.create_task(dig_the_hole()) + await dig_the_hole_task + + # Other instructions associated with planting a tree. + ... + +In this example, imagine the event loop has passed control to the start of the +coroutine ``plant_a_tree()``. +As seen above, the coroutine creates a task and then awaits it. +The ``await dig_the_hole_task`` instruction adds a callback (which will resume +``plant_a_tree()``) to the ``dig_the_hole_task`` object's list of callbacks. +And then, the instruction cedes control to the event loop. +Some time later, the event loop will pass control to ``dig_the_hole_task`` +and the task will finish whatever it needs to do. +Once the task finishes, it will add its various callbacks to the event loop, +in this case, a call to resume ``plant_a_tree()``. + +Generally speaking, when the awaited task finishes (``dig_the_hole_task``), +the original task or coroutine (``plant_a_tree()``) is added back to the event +loop's to-do list to be resumed. + +This is a basic, yet reliable mental model. +In practice, the control handoffs are slightly more complex, but not by much. +In part 2, we'll walk through the details that make this possible. + +^^^^^^^^^^^^^^^^^^^ +Awaiting coroutines +^^^^^^^^^^^^^^^^^^^ + +**Unlike tasks, awaiting a coroutine does not hand control back to the event +loop!** +Wrapping a coroutine in a task first, then awaiting that would cede +control. +The behavior of ``await coroutine`` is effectively the same as invoking a +regular, synchronous Python function. +Consider this program:: + + import asyncio + + async def coro_a(): + print("I am coro_a(). Hi!") + + async def coro_b(): + print("I am coro_b(). I sure hope no one hogs the event loop...") + + async def main(): + task_b = asyncio.create_task(coro_b()) + num_repeats = 3 + for _ in range(num_repeats): + await coro_a() + await task_b + + asyncio.run(main()) + +The first statement in the coroutine ``main()`` creates ``task_b`` and schedules +it for execution via the event loop. +Then, ``coro_a()`` is repeatedly awaited. Control never cedes to the +event loop, which is why we see the output of all three ``coro_a()`` +invocations before ``coro_b()``'s output: + +.. code-block:: none + + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_b(). I sure hope no one hogs the event loop... + +If we change ``await coro_a()`` to ``await asyncio.create_task(coro_a())``, the +behavior changes. +The coroutine ``main()`` cedes control to the event loop with that statement. +The event loop then proceeds through its backlog of work, calling ``task_b`` +and then the task which wraps ``coro_a()`` before resuming the coroutine +``main()``. + +.. code-block:: none + + I am coro_b(). I sure hope no one hogs the event loop... + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_a(). Hi! + +This behavior of ``await coroutine`` can trip a lot of people up! +That example highlights how using only ``await coroutine`` could +unintentionally hog control from other tasks and effectively stall the event +loop. +:func:`asyncio.run` can help you detect such occurrences via the +``debug=True`` flag, which enables +:ref:`debug mode `. +Among other things, it will log any coroutines that monopolize execution for +100ms or longer. + +The design intentionally trades off some conceptual clarity around usage of +``await`` for improved performance. +Each time a task is awaited, control needs to be passed all the way up the +call stack to the event loop. +Then, the event loop needs to manage its internal state and work through +its processing logic to resume the next job. +That might sound minor, but in a large program with many ``await``\ s, that +overhead can add up to a non-negligible performance drag. + +------------------------------------------------ +A conceptual overview part 2: the nuts and bolts +------------------------------------------------ + +Part 2 goes into detail on the mechanisms :mod:`!asyncio` uses to manage +control flow. +This is where the magic happens. +You'll come away from this section knowing what ``await`` does behind the scenes +and how to make your own asynchronous operators. + +================================ +The inner workings of coroutines +================================ + +:mod:`!asyncio` leverages four components of Python to pass +around control. + +:meth:`coroutine.send(arg) ` is the method used to start or +resume a coroutine. +If the coroutine was paused and is now being resumed, the argument ``arg`` +will be sent in as the return value of the ``yield`` statement which originally +paused it. +If the coroutine is being used for the first time (as opposed to being resumed), +``arg`` must be ``None``. + +.. code-block:: + :linenos: + + class Rock: + def __await__(self): + value_sent_in = yield 7 + print(f"Rock.__await__ resuming with value: {value_sent_in}.") + return value_sent_in + + async def main(): + print("Beginning coroutine main().") + rock = Rock() + print("Awaiting rock...") + value_from_rock = await rock + print(f"Coroutine received value: {value_from_rock} from rock.") + return 23 + + coroutine = main() + intermediate_result = coroutine.send(None) + print(f"Coroutine paused and returned intermediate value: {intermediate_result}.") + + print(f"Resuming coroutine and sending in value: 42.") + try: + coroutine.send(42) + except StopIteration as e: + returned_value = e.value + print(f"Coroutine main() finished and provided value: {returned_value}.") + +:ref:`yield `, as usual, pauses execution and returns control +to the caller. +In the example above, the ``yield``, on line 3, is called by +``... = await rock`` on line 11. +More broadly speaking, ``await`` calls the :meth:`~object.__await__` method of +the given object. +``await`` also does one more very special thing: it propagates (or "passes +along") any ``yield``\ s it receives up the call chain. +In this case, that's back to ``... = coroutine.send(None)`` on line 16. + +The coroutine is resumed via the ``coroutine.send(42)`` call on line 21. +The coroutine picks back up from where it ``yield``\ ed (or paused) on line 3 +and executes the remaining statements in its body. +When a coroutine finishes, it raises a :exc:`StopIteration` exception with the +return value attached in the :attr:`~StopIteration.value` attribute. + +That snippet produces this output: + +.. code-block:: none + + Beginning coroutine main(). + Awaiting rock... + Coroutine paused and returned intermediate value: 7. + Resuming coroutine and sending in value: 42. + Rock.__await__ resuming with value: 42. + Coroutine received value: 42 from rock. + Coroutine main() finished and provided value: 23. + +It's worth pausing for a moment here and making sure you followed the various +ways that control flow and values were passed. A lot of important ideas were +covered and it's worth ensuring your understanding is firm. + +The only way to yield (or effectively cede control) from a coroutine is to +``await`` an object that ``yield``\ s in its ``__await__`` method. +That might sound odd to you. You might be thinking: + + 1. What about a ``yield`` directly within the coroutine function? The + coroutine function becomes an + :ref:`async generator function `, a + different beast entirely. + + 2. What about a :ref:`yield from ` within the coroutine function to a (plain) + generator? + That causes the error: ``SyntaxError: yield from not allowed in a coroutine.`` + This was intentionally designed for the sake of simplicity -- mandating only + one way of using coroutines. + Despite that, ``yield from`` and ``await`` effectively do the same thing. + Initially ``yield`` was barred as well, but was re-accepted to allow for + async generators. + +======= +Futures +======= + +A :ref:`future ` is an object meant to represent a +computation's status and result. +The term is a nod to the idea of something still to come or not yet happened, +and the object is a way to keep an eye on that something. + +A future has a few important attributes. One is its state, which can be either +"pending", "cancelled", or "done". +Another is its result, which is set when the state transitions to done. +Unlike a coroutine, a future does not represent the actual computation to be +done; instead, it represents the status and result of that computation, kind of +like a status light (red, yellow, or green) or indicator. + +:class:`asyncio.Task` subclasses :class:`asyncio.Future` in order to gain +these various capabilities. +The prior section said tasks store a list of callbacks, which wasn't entirely +accurate. +It's actually the ``Future`` class that implements this logic, which ``Task`` +inherits. + +Futures may also be used directly (not via tasks). +Tasks mark themselves as done when their coroutine is complete. +Futures are much more versatile and will be marked as done when you say so. +In this way, they're the flexible interface for you to make your own conditions +for waiting and resuming. + +======================== +A homemade asyncio.sleep +======================== + +We'll go through an example of how you could leverage a future to create your +own variant of asynchronous sleep (``async_sleep``) which mimics +:func:`asyncio.sleep`. + +This snippet registers a few tasks with the event loop and then awaits the task +created by ``asyncio.create_task``, which wraps the ``async_sleep(3)`` coroutine. +We want that task to finish only after three seconds have elapsed, but without +preventing other tasks from running. + +:: + + async def other_work(): + print("I like work. Work work.") + + async def main(): + # Add a few other tasks to the event loop, so there's something + # to do while asynchronously sleeping. + work_tasks = [ + asyncio.create_task(other_work()), + asyncio.create_task(other_work()), + asyncio.create_task(other_work()) + ] + print( + "Beginning asynchronous sleep at time: " + f"{datetime.datetime.now().strftime("%H:%M:%S")}." + ) + await asyncio.create_task(async_sleep(3)) + print( + "Done asynchronous sleep at time: " + f"{datetime.datetime.now().strftime("%H:%M:%S")}." + ) + # asyncio.gather effectively awaits each task in the collection. + await asyncio.gather(*work_tasks) + + +Below, we use a future to enable custom control over when that task will be +marked as done. +If :meth:`future.set_result() ` (the method +responsible for marking that future as done) is never called, then this task +will never finish. +We've also enlisted the help of another task, which we'll see in a moment, that +will monitor how much time has elapsed and, accordingly, call +``future.set_result()``. + +:: + + async def async_sleep(seconds: float): + future = asyncio.Future() + time_to_wake = time.time() + seconds + # Add the watcher-task to the event loop. + watcher_task = asyncio.create_task(_sleep_watcher(future, time_to_wake)) + # Block until the future is marked as done. + await future + +Below, we use a rather bare ``YieldToEventLoop()`` object to ``yield`` +from its ``__await__`` method, ceding control to the event loop. +This is effectively the same as calling ``asyncio.sleep(0)``, but this approach +offers more clarity, not to mention it's somewhat cheating to use +``asyncio.sleep`` when showcasing how to implement it! + +As usual, the event loop cycles through its tasks, giving them control +and receiving control back when they pause or finish. +The ``watcher_task``, which runs the coroutine ``_sleep_watcher(...)``, will +be invoked once per full cycle of the event loop. +On each resumption, it'll check the time and if not enough has elapsed, then +it'll pause once again and hand control back to the event loop. +Once enough time has elapsed, ``_sleep_watcher(...)`` +marks the future as done and completes by exiting its +infinite ``while`` loop. +Given this helper task is only invoked once per cycle of the event loop, +you'd be correct to note that this asynchronous sleep will sleep *at least* +three seconds, rather than exactly three seconds. +Note this is also true of ``asyncio.sleep``. + +:: + + class YieldToEventLoop: + def __await__(self): + yield + + async def _sleep_watcher(future, time_to_wake): + while True: + if time.time() >= time_to_wake: + # This marks the future as done. + future.set_result(None) + break + else: + await YieldToEventLoop() + +Here is the full program's output: + +.. code-block:: none + + $ python custom-async-sleep.py + Beginning asynchronous sleep at time: 14:52:22. + I like work. Work work. + I like work. Work work. + I like work. Work work. + Done asynchronous sleep at time: 14:52:25. + +You might feel this implementation of asynchronous sleep was unnecessarily +convoluted. +And, well, it was. +The example was meant to showcase the versatility of futures with a simple +example that could be mimicked for more complex needs. +For reference, you could implement it without futures, like so:: + + async def simpler_async_sleep(seconds): + time_to_wake = time.time() + seconds + while True: + if time.time() >= time_to_wake: + return + else: + await YieldToEventLoop() + +But that's all for now. Hopefully you're ready to more confidently dive into +some async programming or check out advanced topics in the +:mod:`rest of the documentation `. diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 78f3704ba5d000..d7deb6c6bc1768 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -248,4 +248,9 @@ quirks by using :func:`annotationlib.get_annotations` on Python 3.14+ or :func:`inspect.get_annotations` on Python 3.10+. On earlier versions of Python, you can avoid these bugs by accessing the annotations from the class's :attr:`~type.__dict__` -(e.g., ``cls.__dict__.get('__annotations__', None)``). +(for example, ``cls.__dict__.get('__annotations__', None)``). + +In some versions of Python, instances of classes may have an ``__annotations__`` +attribute. However, this is not supported functionality. If you need the +annotations of an instance, you can use :func:`type` to access its class +(for example, ``annotationlib.get_annotations(type(myinstance))`` on Python 3.14+). diff --git a/Doc/howto/argparse-optparse.rst b/Doc/howto/argparse-optparse.rst index cef2d893b28a62..b684619885b4c7 100644 --- a/Doc/howto/argparse-optparse.rst +++ b/Doc/howto/argparse-optparse.rst @@ -1,20 +1,14 @@ .. currentmodule:: argparse .. _upgrading-optparse-code: +.. _migrating-optparse-code: -========================== -Upgrading optparse code -========================== +============================================ +Migrating ``optparse`` code to ``argparse`` +============================================ -Originally, the :mod:`argparse` module had attempted to maintain compatibility -with :mod:`optparse`. However, :mod:`optparse` was difficult to extend -transparently, particularly with the changes required to support -``nargs=`` specifiers and better usage messages. When most everything in -:mod:`optparse` had either been copy-pasted over or monkey-patched, it no -longer seemed practical to try to maintain the backwards compatibility. - -The :mod:`argparse` module improves on the :mod:`optparse` -module in a number of ways including: +The :mod:`argparse` module offers several higher level features not natively +provided by the :mod:`optparse` module, including: * Handling positional arguments. * Supporting subcommands. @@ -23,7 +17,23 @@ module in a number of ways including: * Producing more informative usage messages. * Providing a much simpler interface for custom ``type`` and ``action``. -A partial upgrade path from :mod:`optparse` to :mod:`argparse`: +Originally, the :mod:`argparse` module attempted to maintain compatibility +with :mod:`optparse`. However, the fundamental design differences between +supporting declarative command line option processing (while leaving positional +argument processing to application code), and supporting both named options +and positional arguments in the declarative interface mean that the +API has diverged from that of ``optparse`` over time. + +As described in :ref:`choosing-an-argument-parser`, applications that are +currently using :mod:`optparse` and are happy with the way it works can +just continue to use ``optparse``. + +Application developers that are considering migrating should also review +the list of intrinsic behavioural differences described in that section +before deciding whether or not migration is desirable. + +For applications that do choose to migrate from :mod:`optparse` to :mod:`argparse`, +the following suggestions should be helpful: * Replace all :meth:`optparse.OptionParser.add_option` calls with :meth:`ArgumentParser.add_argument` calls. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 1efbee64d60bb3..902c50de00803c 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -13,11 +13,16 @@ recommended command-line parsing module in the Python standard library. .. note:: - There are two other modules that fulfill the same task, namely - :mod:`getopt` (an equivalent for ``getopt()`` from the C - language) and the deprecated :mod:`optparse`. - Note also that :mod:`argparse` is based on :mod:`optparse`, - and therefore very similar in terms of usage. + The standard library includes two other libraries directly related + to command-line parameter processing: the lower level :mod:`optparse` + module (which may require more code to configure for a given application, + but also allows an application to request behaviors that ``argparse`` + doesn't support), and the very low level :mod:`getopt` (which specifically + serves as an equivalent to the :c:func:`!getopt` family of functions + available to C programmers). + While neither of those modules is covered directly in this guide, many of + the core concepts in ``argparse`` first originated in ``optparse``, so + some aspects of this tutorial will also be relevant to ``optparse`` users. Concepts diff --git a/Doc/howto/cporting.rst b/Doc/howto/cporting.rst index 7773620b40b973..cf857aed0425ec 100644 --- a/Doc/howto/cporting.rst +++ b/Doc/howto/cporting.rst @@ -14,13 +14,11 @@ We recommend the following resources for porting extension modules to Python 3: module. * The `Porting guide`_ from the *py3c* project provides opinionated suggestions with supporting code. -* The `Cython`_ and `CFFI`_ libraries offer abstractions over - Python's C API. +* :ref:`Recommended third party tools ` offer abstractions over + the Python's C API. Extensions generally need to be re-written to use one of them, but the library then handles differences between various Python versions and implementations. .. _Migrating C extensions: http://python3porting.com/cextensions.html .. _Porting guide: https://py3c.readthedocs.io/en/latest/guide.html -.. _Cython: https://cython.org/ -.. _CFFI: https://cffi.readthedocs.io/en/latest/ diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index f9ad81e38f8dc3..816639552d7cd6 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -145,8 +145,8 @@ importing the :func:`curses.wrapper` function and using it like this:: v = i-10 stdscr.addstr(i, 0, '10 divided by {} is {}'.format(v, 10/v)) - stdscr.refresh() - stdscr.getkey() + stdscr.refresh() + stdscr.getkey() wrapper(main) @@ -161,6 +161,8 @@ your terminal won't be left in a funny state on exception and you'll be able to read the exception's message and traceback. +.. _windows-and-pads: + Windows and Pads ================ diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 01264bfe823746..9d5a9ac8b718cb 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -42,7 +42,7 @@ add new capabilities one by one. Simple example: A descriptor that returns a constant ---------------------------------------------------- -The :class:`Ten` class is a descriptor whose :meth:`__get__` method always +The :class:`!Ten` class is a descriptor whose :meth:`~object.__get__` method always returns the constant ``10``: .. testcode:: @@ -120,10 +120,10 @@ different, updated answers each time:: 2 Besides showing how descriptors can run computations, this example also -reveals the purpose of the parameters to :meth:`__get__`. The *self* +reveals the purpose of the parameters to :meth:`~object.__get__`. The *self* parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that -lets the :meth:`__get__` method learn the target directory. The *objtype* +lets the :meth:`~object.__get__` method learn the target directory. The *objtype* parameter is the class *Directory*. @@ -133,7 +133,7 @@ Managed attributes A popular use for descriptors is managing access to instance data. The descriptor is assigned to a public attribute in the class dictionary while the actual data is stored as a private attribute in the instance dictionary. The -descriptor's :meth:`__get__` and :meth:`__set__` methods are triggered when +descriptor's :meth:`~object.__get__` and :meth:`~object.__set__` methods are triggered when the public attribute is accessed. In the following example, *age* is the public attribute and *_age* is the @@ -215,9 +215,9 @@ Customized names When a class uses descriptors, it can inform each descriptor about which variable name was used. -In this example, the :class:`Person` class has two descriptor instances, -*name* and *age*. When the :class:`Person` class is defined, it makes a -callback to :meth:`__set_name__` in *LoggedAccess* so that the field names can +In this example, the :class:`!Person` class has two descriptor instances, +*name* and *age*. When the :class:`!Person` class is defined, it makes a +callback to :meth:`~object.__set_name__` in *LoggedAccess* so that the field names can be recorded, giving each descriptor its own *public_name* and *private_name*: .. testcode:: @@ -253,8 +253,8 @@ be recorded, giving each descriptor its own *public_name* and *private_name*: def birthday(self): self.age += 1 -An interactive session shows that the :class:`Person` class has called -:meth:`__set_name__` so that the field names would be recorded. Here +An interactive session shows that the :class:`!Person` class has called +:meth:`~object.__set_name__` so that the field names would be recorded. Here we call :func:`vars` to look up the descriptor without triggering it: .. doctest:: @@ -294,10 +294,10 @@ The two *Person* instances contain only the private names: Closing thoughts ---------------- -A :term:`descriptor` is what we call any object that defines :meth:`__get__`, -:meth:`__set__`, or :meth:`__delete__`. +A :term:`descriptor` is what we call any object that defines :meth:`~object.__get__`, +:meth:`~object.__set__`, or :meth:`~object.__delete__`. -Optionally, descriptors can have a :meth:`__set_name__` method. This is only +Optionally, descriptors can have a :meth:`~object.__set_name__` method. This is only used in cases where a descriptor needs to know either the class where it was created or the name of class variable it was assigned to. (This method, if present, is called even if the class is not a descriptor.) @@ -337,7 +337,7 @@ any data, it verifies that the new value meets various type and range restrictions. If those restrictions aren't met, it raises an exception to prevent data corruption at its source. -This :class:`Validator` class is both an :term:`abstract base class` and a +This :class:`!Validator` class is both an :term:`abstract base class` and a managed attribute descriptor: .. testcode:: @@ -360,8 +360,8 @@ managed attribute descriptor: def validate(self, value): pass -Custom validators need to inherit from :class:`Validator` and must supply a -:meth:`validate` method to test various restrictions as needed. +Custom validators need to inherit from :class:`!Validator` and must supply a +:meth:`!validate` method to test various restrictions as needed. Custom validators @@ -369,13 +369,13 @@ Custom validators Here are three practical data validation utilities: -1) :class:`OneOf` verifies that a value is one of a restricted set of options. +1) :class:`!OneOf` verifies that a value is one of a restricted set of options. -2) :class:`Number` verifies that a value is either an :class:`int` or +2) :class:`!Number` verifies that a value is either an :class:`int` or :class:`float`. Optionally, it verifies that a value is between a given minimum or maximum. -3) :class:`String` verifies that a value is a :class:`str`. Optionally, it +3) :class:`!String` verifies that a value is a :class:`str`. Optionally, it validates a given minimum or maximum length. It can validate a user-defined `predicate `_ as well. @@ -420,7 +420,7 @@ Here are three practical data validation utilities: def validate(self, value): if not isinstance(value, str): - raise TypeError(f'Expected {value!r} to be an str') + raise TypeError(f'Expected {value!r} to be a str') if self.minsize is not None and len(value) < self.minsize: raise ValueError( f'Expected {value!r} to be no smaller than {self.minsize!r}' @@ -501,8 +501,8 @@ Definition and introduction --------------------------- In general, a descriptor is an attribute value that has one of the methods in -the descriptor protocol. Those methods are :meth:`__get__`, :meth:`__set__`, -and :meth:`__delete__`. If any of those methods are defined for an +the descriptor protocol. Those methods are :meth:`~object.__get__`, :meth:`~object.__set__`, +and :meth:`~object.__delete__`. If any of those methods are defined for an attribute, it is said to be a :term:`descriptor`. The default behavior for attribute access is to get, set, or delete the @@ -534,8 +534,8 @@ That is all there is to it. Define any of these methods and an object is considered a descriptor and can override default behavior upon being looked up as an attribute. -If an object defines :meth:`__set__` or :meth:`__delete__`, it is considered -a data descriptor. Descriptors that only define :meth:`__get__` are called +If an object defines :meth:`~object.__set__` or :meth:`~object.__delete__`, it is considered +a data descriptor. Descriptors that only define :meth:`~object.__get__` are called non-data descriptors (they are often used for methods but other uses are possible). @@ -545,9 +545,9 @@ has an entry with the same name as a data descriptor, the data descriptor takes precedence. If an instance's dictionary has an entry with the same name as a non-data descriptor, the dictionary entry takes precedence. -To make a read-only data descriptor, define both :meth:`__get__` and -:meth:`__set__` with the :meth:`__set__` raising an :exc:`AttributeError` when -called. Defining the :meth:`__set__` method with an exception raising +To make a read-only data descriptor, define both :meth:`~object.__get__` and +:meth:`~object.__set__` with the :meth:`~object.__set__` raising an :exc:`AttributeError` when +called. Defining the :meth:`~object.__set__` method with an exception raising placeholder is enough to make it a data descriptor. @@ -574,7 +574,7 @@ Invocation from an instance Instance lookup scans through a chain of namespaces giving data descriptors the highest priority, followed by instance variables, then non-data -descriptors, then class variables, and lastly :meth:`__getattr__` if it is +descriptors, then class variables, and lastly :meth:`~object.__getattr__` if it is provided. If a descriptor is found for ``a.x``, then it is invoked with: @@ -719,12 +719,12 @@ a pure Python equivalent: >>> object_getattribute(u2, 'x') == u2.x == (D1, u2, U2) True -Note, there is no :meth:`__getattr__` hook in the :meth:`__getattribute__` -code. That is why calling :meth:`__getattribute__` directly or with -``super().__getattribute__`` will bypass :meth:`__getattr__` entirely. +Note, there is no :meth:`~object.__getattr__` hook in the :meth:`~object.__getattribute__` +code. That is why calling :meth:`~object.__getattribute__` directly or with +``super().__getattribute__`` will bypass :meth:`~object.__getattr__` entirely. Instead, it is the dot operator and the :func:`getattr` function that are -responsible for invoking :meth:`__getattr__` whenever :meth:`__getattribute__` +responsible for invoking :meth:`~object.__getattr__` whenever :meth:`~object.__getattribute__` raises an :exc:`AttributeError`. Their logic is encapsulated in a helper function: @@ -776,8 +776,8 @@ Invocation from a class ----------------------- The logic for a dotted lookup such as ``A.x`` is in -:meth:`type.__getattribute__`. The steps are similar to those for -:meth:`object.__getattribute__` but the instance dictionary lookup is replaced +:meth:`!type.__getattribute__`. The steps are similar to those for +:meth:`!object.__getattribute__` but the instance dictionary lookup is replaced by a search through the class's :term:`method resolution order`. If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. @@ -789,7 +789,7 @@ The full C implementation can be found in :c:func:`!type_getattro` and Invocation from super --------------------- -The logic for super's dotted lookup is in the :meth:`__getattribute__` method for +The logic for super's dotted lookup is in the :meth:`~object.__getattribute__` method for object returned by :func:`super`. A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` @@ -806,21 +806,21 @@ The full C implementation can be found in :c:func:`!super_getattro` in Summary of invocation logic --------------------------- -The mechanism for descriptors is embedded in the :meth:`__getattribute__` +The mechanism for descriptors is embedded in the :meth:`~object.__getattribute__` methods for :class:`object`, :class:`type`, and :func:`super`. The important points to remember are: -* Descriptors are invoked by the :meth:`__getattribute__` method. +* Descriptors are invoked by the :meth:`~object.__getattribute__` method. * Classes inherit this machinery from :class:`object`, :class:`type`, or :func:`super`. -* Overriding :meth:`__getattribute__` prevents automatic descriptor calls +* Overriding :meth:`~object.__getattribute__` prevents automatic descriptor calls because all the descriptor logic is in that method. -* :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make - different calls to :meth:`__get__`. The first includes the instance and may +* :meth:`!object.__getattribute__` and :meth:`!type.__getattribute__` make + different calls to :meth:`~object.__get__`. The first includes the instance and may include the class. The second puts in ``None`` for the instance and always includes the class. @@ -835,16 +835,16 @@ Automatic name notification Sometimes it is desirable for a descriptor to know what class variable name it was assigned to. When a new class is created, the :class:`type` metaclass scans the dictionary of the new class. If any of the entries are descriptors -and if they define :meth:`__set_name__`, that method is called with two +and if they define :meth:`~object.__set_name__`, that method is called with two arguments. The *owner* is the class where the descriptor is used, and the *name* is the class variable the descriptor was assigned to. The implementation details are in :c:func:`!type_new` and :c:func:`!set_names` in :source:`Objects/typeobject.c`. -Since the update logic is in :meth:`type.__new__`, notifications only take +Since the update logic is in :meth:`!type.__new__`, notifications only take place at the time of class creation. If descriptors are added to the class -afterwards, :meth:`__set_name__` will need to be called manually. +afterwards, :meth:`~object.__set_name__` will need to be called manually. ORM example @@ -873,7 +873,7 @@ care of lookups or updates: conn.execute(self.store, [value, obj.key]) conn.commit() -We can use the :class:`Field` class to define `models +We can use the :class:`!Field` class to define `models `_ that describe the schema for each table in a database: @@ -1140,7 +1140,7 @@ to wrap access to the value attribute in a property data descriptor: self.recalc() return self._value -Either the built-in :func:`property` or our :func:`Property` equivalent would +Either the built-in :func:`property` or our :func:`!Property` equivalent would work in this example. @@ -1187,7 +1187,7 @@ roughly equivalent to: return self To support automatic creation of methods, functions include the -:meth:`__get__` method for binding methods during attribute access. This +:meth:`~object.__get__` method for binding methods during attribute access. This means that functions are non-data descriptors that return bound methods during dotted lookup from an instance. Here's how it works: @@ -1231,19 +1231,19 @@ The function has a :term:`qualified name` attribute to support introspection: 'D.f' Accessing the function through the class dictionary does not invoke -:meth:`__get__`. Instead, it just returns the underlying function object:: +:meth:`~object.__get__`. Instead, it just returns the underlying function object:: >>> D.__dict__['f'] -Dotted access from a class calls :meth:`__get__` which just returns the +Dotted access from a class calls :meth:`~object.__get__` which just returns the underlying function unchanged:: >>> D.f The interesting behavior occurs during dotted access from an instance. The -dotted lookup calls :meth:`__get__` which returns a bound method object:: +dotted lookup calls :meth:`~object.__get__` which returns a bound method object:: >>> d = D() >>> d.f @@ -1268,7 +1268,7 @@ Kinds of methods Non-data descriptors provide a simple mechanism for variations on the usual patterns of binding functions into methods. -To recap, functions have a :meth:`__get__` method so that they can be converted +To recap, functions have a :meth:`~object.__get__` method so that they can be converted to a method when accessed as attributes. The non-data descriptor transforms an ``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``cls.f(*args)`` becomes ``f(*args)``. @@ -1671,7 +1671,7 @@ by member descriptors: 'Emulate member_repr() in Objects/descrobject.c' return f'' -The :meth:`type.__new__` method takes care of adding member objects to class +The :meth:`!type.__new__` method takes care of adding member objects to class variables: .. testcode:: @@ -1722,7 +1722,7 @@ Python: ) super().__delattr__(name) -To use the simulation in a real class, just inherit from :class:`Object` and +To use the simulation in a real class, just inherit from :class:`!Object` and set the :term:`metaclass` to :class:`Type`: .. testcode:: diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 66929b4104d8de..7713aede6d564a 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -64,12 +64,12 @@ The *type* of an enumeration member is the enum it belongs to:: >>> isinstance(Weekday.FRIDAY, Weekday) True -Enum members have an attribute that contains just their :attr:`name`:: +Enum members have an attribute that contains just their :attr:`!name`:: >>> print(Weekday.TUESDAY.name) TUESDAY -Likewise, they have an attribute for their :attr:`value`:: +Likewise, they have an attribute for their :attr:`!value`:: >>> Weekday.WEDNESDAY.value @@ -77,17 +77,18 @@ Likewise, they have an attribute for their :attr:`value`:: Unlike many languages that treat enumerations solely as name/value pairs, Python Enums can have behavior added. For example, :class:`datetime.date` -has two methods for returning the weekday: :meth:`weekday` and :meth:`isoweekday`. +has two methods for returning the weekday: +:meth:`~datetime.date.weekday` and :meth:`~datetime.date.isoweekday`. The difference is that one of them counts from 0-6 and the other from 1-7. -Rather than keep track of that ourselves we can add a method to the :class:`Weekday` -enum to extract the day from the :class:`date` instance and return the matching +Rather than keep track of that ourselves we can add a method to the :class:`!Weekday` +enum to extract the day from the :class:`~datetime.date` instance and return the matching enum member:: @classmethod def from_date(cls, date): return cls(date.isoweekday()) -The complete :class:`Weekday` enum now looks like this:: +The complete :class:`!Weekday` enum now looks like this:: >>> class Weekday(Enum): ... MONDAY = 1 @@ -110,7 +111,7 @@ Now we can find out what today is! Observe:: Of course, if you're reading this on some other day, you'll see that day instead. -This :class:`Weekday` enum is great if our variable only needs one day, but +This :class:`!Weekday` enum is great if our variable only needs one day, but what if we need several? Maybe we're writing a function to plot chores during a week, and don't want to use a :class:`list` -- we could use a different type of :class:`Enum`:: @@ -128,7 +129,7 @@ of :class:`Enum`:: We've changed two things: we're inherited from :class:`Flag`, and the values are all powers of 2. -Just like the original :class:`Weekday` enum above, we can have a single selection:: +Just like the original :class:`!Weekday` enum above, we can have a single selection:: >>> first_week_day = Weekday.MONDAY >>> first_week_day @@ -203,7 +204,7 @@ If you want to access enum members by *name*, use item access:: >>> Color['GREEN'] -If you have an enum member and need its :attr:`name` or :attr:`value`:: +If you have an enum member and need its :attr:`!name` or :attr:`!value`:: >>> member = Color.RED >>> member.name @@ -284,7 +285,7 @@ If the exact value is unimportant you can use :class:`auto`:: >>> [member.value for member in Color] [1, 2, 3] -The values are chosen by :func:`_generate_next_value_`, which can be +The values are chosen by :func:`~Enum._generate_next_value_`, which can be overridden:: >>> class AutoName(Enum): @@ -303,7 +304,7 @@ overridden:: .. note:: - The :meth:`_generate_next_value_` method must be defined before any members. + The :meth:`~Enum._generate_next_value_` method must be defined before any members. Iteration --------- @@ -424,18 +425,18 @@ Then:: The rules for what is allowed are as follows: names that start and end with a single underscore are reserved by enum and cannot be used; all other attributes defined within an enumeration will become members of this -enumeration, with the exception of special methods (:meth:`__str__`, -:meth:`__add__`, etc.), descriptors (methods are also descriptors), and -variable names listed in :attr:`_ignore_`. +enumeration, with the exception of special methods (:meth:`~object.__str__`, +:meth:`~object.__add__`, etc.), descriptors (methods are also descriptors), and +variable names listed in :attr:`~Enum._ignore_`. -Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__`, +Note: if your enumeration defines :meth:`~object.__new__` and/or :meth:`~object.__init__`, any value(s) given to the enum member will be passed into those methods. See `Planet`_ for an example. .. note:: - The :meth:`__new__` method, if defined, is used during creation of the Enum - members; it is then replaced by Enum's :meth:`__new__` which is used after + The :meth:`~object.__new__` method, if defined, is used during creation of the Enum + members; it is then replaced by Enum's :meth:`~object.__new__` which is used after class creation for lookup of existing members. See :ref:`new-vs-init` for more details. @@ -544,7 +545,7 @@ from that module. nested in other classes. It is possible to modify how enum members are pickled/unpickled by defining -:meth:`__reduce_ex__` in the enumeration class. The default method is by-value, +:meth:`~object.__reduce_ex__` in the enumeration class. The default method is by-value, but enums with complicated values may want to use by-name:: >>> import enum @@ -580,7 +581,7 @@ values. The last two options enable assigning arbitrary values to enumerations; the others auto-assign increasing integers starting with 1 (use the ``start`` parameter to specify a different starting value). A new class derived from :class:`Enum` is returned. In other words, the above -assignment to :class:`Animal` is equivalent to:: +assignment to :class:`!Animal` is equivalent to:: >>> class Animal(Enum): ... ANT = 1 @@ -891,7 +892,7 @@ simple to implement independently:: pass This demonstrates how similar derived enumerations can be defined; for example -a :class:`FloatEnum` that mixes in :class:`float` instead of :class:`int`. +a :class:`!FloatEnum` that mixes in :class:`float` instead of :class:`int`. Some rules: @@ -905,32 +906,32 @@ Some rules: additional type, all the members must have values of that type, e.g. :class:`int` above. This restriction does not apply to mix-ins which only add methods and don't specify another type. -4. When another data type is mixed in, the :attr:`value` attribute is *not the +4. When another data type is mixed in, the :attr:`~Enum.value` attribute is *not the same* as the enum member itself, although it is equivalent and will compare equal. -5. A ``data type`` is a mixin that defines :meth:`__new__`, or a +5. A ``data type`` is a mixin that defines :meth:`~object.__new__`, or a :class:`~dataclasses.dataclass` 6. %-style formatting: ``%s`` and ``%r`` call the :class:`Enum` class's - :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as + :meth:`~object.__str__` and :meth:`~object.__repr__` respectively; other codes (such as ``%i`` or ``%h`` for IntEnum) treat the enum member as its mixed-in type. 7. :ref:`Formatted string literals `, :meth:`str.format`, - and :func:`format` will use the enum's :meth:`__str__` method. + and :func:`format` will use the enum's :meth:`~object.__str__` method. .. note:: Because :class:`IntEnum`, :class:`IntFlag`, and :class:`StrEnum` are designed to be drop-in replacements for existing constants, their - :meth:`__str__` method has been reset to their data types' - :meth:`__str__` method. + :meth:`~object.__str__` method has been reset to their data types' + :meth:`~object.__str__` method. .. _new-vs-init: -When to use :meth:`__new__` vs. :meth:`__init__` ------------------------------------------------- +When to use :meth:`~object.__new__` vs. :meth:`~object.__init__` +---------------------------------------------------------------- -:meth:`__new__` must be used whenever you want to customize the actual value of +:meth:`~object.__new__` must be used whenever you want to customize the actual value of the :class:`Enum` member. Any other modifications may go in either -:meth:`__new__` or :meth:`__init__`, with :meth:`__init__` being preferred. +:meth:`~object.__new__` or :meth:`~object.__init__`, with :meth:`~object.__init__` being preferred. For example, if you want to pass several items to the constructor, but only want one of them to be the value:: @@ -969,11 +970,11 @@ Finer Points Supported ``__dunder__`` names """""""""""""""""""""""""""""" -:attr:`__members__` is a read-only ordered mapping of ``member_name``:``member`` +:attr:`~enum.EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` items. It is only available on the class. -:meth:`__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`_value_` appropriately. Once +:meth:`~object.__new__`, if specified, must create and return the enum members; it is +also a very good idea to set the member's :attr:`~Enum._value_` appropriately. Once all the members are created it is no longer used. @@ -1009,7 +1010,7 @@ Supported ``_sunder_`` names .. versionadded:: 3.7 ``_ignore_`` .. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` -To help keep Python 2 / Python 3 code in sync an :attr:`_order_` attribute can +To help keep Python 2 / Python 3 code in sync an :attr:`~Enum._order_` attribute can be provided. It will be checked against the actual order of the enumeration and raise an error if the two do not match:: @@ -1027,7 +1028,7 @@ and raise an error if the two do not match:: .. note:: - In Python 2 code the :attr:`_order_` attribute is necessary as definition + In Python 2 code the :attr:`~Enum._order_` attribute is necessary as definition order is lost before it can be recorded. @@ -1216,12 +1217,12 @@ Enum Classes ^^^^^^^^^^^^ The :class:`EnumType` metaclass is responsible for providing the -:meth:`__contains__`, :meth:`__dir__`, :meth:`__iter__` and other methods that +:meth:`~object.__contains__`, :meth:`~object.__dir__`, :meth:`~object.__iter__` and other methods that allow one to do things with an :class:`Enum` class that fail on a typical class, such as ``list(Color)`` or ``some_enum_var in Color``. :class:`EnumType` is responsible for ensuring that various other methods on the final :class:`Enum` -class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`, -:meth:`__str__` and :meth:`__repr__`). +class are correct (such as :meth:`~object.__new__`, :meth:`~object.__getnewargs__`, +:meth:`~object.__str__` and :meth:`~object.__repr__`). Flag Classes ^^^^^^^^^^^^ @@ -1236,7 +1237,7 @@ Enum Members (aka instances) The most interesting thing about enum members is that they are singletons. :class:`EnumType` creates them all while it is creating the enum class itself, -and then puts a custom :meth:`__new__` in place to ensure that no new ones are +and then puts a custom :meth:`~object.__new__` in place to ensure that no new ones are ever instantiated by returning only the existing member instances. Flag Members @@ -1284,7 +1285,7 @@ is. There are several ways to define this type of simple enumeration: - use instances of :class:`auto` for the value - use instances of :class:`object` as the value - use a descriptive string as the value -- use a tuple as the value and a custom :meth:`__new__` to replace the +- use a tuple as the value and a custom :meth:`~object.__new__` to replace the tuple with an :class:`int` value Using any of these methods signifies to the user that these values are not @@ -1320,7 +1321,7 @@ Using :class:`object` would look like:: > This is also a good example of why you might want to write your own -:meth:`__repr__`:: +:meth:`~object.__repr__`:: >>> class Color(Enum): ... RED = object() @@ -1348,10 +1349,10 @@ Using a string as the value would look like:: -Using a custom :meth:`__new__` -"""""""""""""""""""""""""""""" +Using a custom :meth:`~object.__new__` +"""""""""""""""""""""""""""""""""""""" -Using an auto-numbering :meth:`__new__` would look like:: +Using an auto-numbering :meth:`~object.__new__` would look like:: >>> class AutoNumber(Enum): ... def __new__(cls): @@ -1397,8 +1398,8 @@ to handle any extra arguments:: .. note:: - The :meth:`__new__` method, if defined, is used during creation of the Enum - members; it is then replaced by Enum's :meth:`__new__` which is used after + The :meth:`~object.__new__` method, if defined, is used during creation of the Enum + members; it is then replaced by Enum's :meth:`~object.__new__` which is used after class creation for lookup of existing members. .. warning:: @@ -1504,7 +1505,7 @@ Supports having more than one value per member:: Planet ^^^^^^ -If :meth:`__new__` or :meth:`__init__` is defined, the value of the enum member +If :meth:`~object.__new__` or :meth:`~object.__init__` is defined, the value of the enum member will be passed to those methods:: >>> class Planet(Enum): @@ -1535,7 +1536,7 @@ will be passed to those methods:: TimePeriod ^^^^^^^^^^ -An example to show the :attr:`_ignore_` attribute in use:: +An example to show the :attr:`~Enum._ignore_` attribute in use:: >>> from datetime import timedelta >>> class Period(timedelta, Enum): diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 6abe93d71ad529..83eba8cfea3969 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -6,8 +6,8 @@ C API Extension Support for Free Threading ****************************************** -Starting with the 3.13 release, CPython has experimental support for running -with the :term:`global interpreter lock` (GIL) disabled in a configuration +Starting with the 3.13 release, CPython has support for running with +the :term:`global interpreter lock` (GIL) disabled in a configuration called :term:`free threading`. This document describes how to adapt C API extensions to support free threading. @@ -23,6 +23,14 @@ You can use it to enable code that only runs under the free-threaded build:: /* code that only runs in the free-threaded build */ #endif +.. note:: + + On Windows, this macro is not defined automatically, but must be specified + to the compiler when building. The :func:`sysconfig.get_config_var` function + can be used to determine whether the current running interpreter had the + macro defined. + + Module Initialization ===================== @@ -37,9 +45,12 @@ single-phase initialization. Multi-Phase Initialization .......................... -Extensions that use multi-phase initialization (i.e., -:c:func:`PyModuleDef_Init`) should add a :c:data:`Py_mod_gil` slot in the -module definition. If your extension supports older versions of CPython, +Extensions that use :ref:`multi-phase initialization ` +(functions like :c:func:`PyModuleDef_Init`, +:c:func:`PyModExport_* ` export hook, +:c:func:`PyModule_FromSlotsAndSpec`) should add a +:c:data:`Py_mod_gil` slot in the module definition. +If your extension supports older versions of CPython, you should guard the slot with a :c:data:`PY_VERSION_HEX` check. :: @@ -52,18 +63,12 @@ you should guard the slot with a :c:data:`PY_VERSION_HEX` check. {0, NULL} }; - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - .m_slots = module_slots, - ... - }; - Single-Phase Initialization ........................... -Extensions that use single-phase initialization (i.e., -:c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to +Extensions that use legacy :ref:`single-phase initialization ` +(that is, :c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to indicate that they support running with the GIL disabled. The function is only defined in the free-threaded build, so you should guard the call with ``#ifdef Py_GIL_DISABLED`` to avoid compilation errors in the regular build. @@ -96,8 +101,10 @@ Most of the C API is thread-safe, but there are some exceptions. * **Struct Fields**: Accessing fields in Python C API objects or structs directly is not thread-safe if the field may be concurrently modified. -* **Macros**: Accessor macros like :c:macro:`PyList_GET_ITEM` and - :c:macro:`PyList_SET_ITEM` do not perform any error checking or locking. +* **Macros**: Accessor macros like :c:macro:`PyList_GET_ITEM`, + :c:macro:`PyList_SET_ITEM`, and macros like + :c:macro:`PySequence_Fast_GET_SIZE` that use the object returned by + :c:func:`PySequence_Fast` do not perform any error checking or locking. These macros are not thread-safe if the container object may be modified concurrently. * **Borrowed References**: C API functions that return @@ -151,6 +158,8 @@ that return :term:`strong references `. +===================================+===================================+ | :c:func:`PyList_GetItem` | :c:func:`PyList_GetItemRef` | +-----------------------------------+-----------------------------------+ +| :c:func:`PyList_GET_ITEM` | :c:func:`PyList_GetItemRef` | ++-----------------------------------+-----------------------------------+ | :c:func:`PyDict_GetItem` | :c:func:`PyDict_GetItemRef` | +-----------------------------------+-----------------------------------+ | :c:func:`PyDict_GetItemWithError` | :c:func:`PyDict_GetItemRef` | @@ -161,12 +170,14 @@ that return :term:`strong references `. +-----------------------------------+-----------------------------------+ | :c:func:`PyDict_Next` | none (see :ref:`PyDict_Next`) | +-----------------------------------+-----------------------------------+ -| :c:func:`PyWeakref_GetObject` | :c:func:`PyWeakref_GetRef` | +| :c:func:`!PyWeakref_GetObject` | :c:func:`PyWeakref_GetRef` | +-----------------------------------+-----------------------------------+ -| :c:func:`PyWeakref_GET_OBJECT` | :c:func:`PyWeakref_GetRef` | +| :c:func:`!PyWeakref_GET_OBJECT` | :c:func:`PyWeakref_GetRef` | +-----------------------------------+-----------------------------------+ | :c:func:`PyImport_AddModule` | :c:func:`PyImport_AddModuleRef` | +-----------------------------------+-----------------------------------+ +| :c:func:`PyCell_GET` | :c:func:`PyCell_Get` | ++-----------------------------------+-----------------------------------+ Not all APIs that return borrowed references are problematic. For example, :c:func:`PyTuple_GetItem` is safe because tuples are immutable. @@ -189,7 +200,7 @@ Memory Allocation APIs Python's memory management C API provides functions in three different :ref:`allocation domains `: "raw", "mem", and "object". For thread-safety, the free-threaded build requires that only Python objects -are allocated using the object domain, and that all Python object are +are allocated using the object domain, and that all Python objects are allocated using that domain. This differs from the prior Python versions, where this was only a best practice and not a hard requirement. @@ -239,6 +250,141 @@ depend on your extension, but some common patterns include: `thread-local storage `_. +Critical Sections +================= + +.. _critical-sections: + +In the free-threaded build, CPython provides a mechanism called "critical +sections" to protect data that would otherwise be protected by the GIL. +While extension authors may not interact with the internal critical section +implementation directly, understanding their behavior is crucial when using +certain C API functions or managing shared state in the free-threaded build. + +What Are Critical Sections? +........................... + +Conceptually, critical sections act as a deadlock avoidance layer built on +top of simple mutexes. Each thread maintains a stack of active critical +sections. When a thread needs to acquire a lock associated with a critical +section (e.g., implicitly when calling a thread-safe C API function like +:c:func:`PyDict_SetItem`, or explicitly using macros), it attempts to acquire +the underlying mutex. + +Using Critical Sections +....................... + +The primary APIs for using critical sections are: + +* :c:macro:`Py_BEGIN_CRITICAL_SECTION` and :c:macro:`Py_END_CRITICAL_SECTION` - + For locking a single object + +* :c:macro:`Py_BEGIN_CRITICAL_SECTION2` and :c:macro:`Py_END_CRITICAL_SECTION2` + - For locking two objects simultaneously + +These macros must be used in matching pairs and must appear in the same C +scope, since they establish a new local scope. These macros are no-ops in +non-free-threaded builds, so they can be safely added to code that needs to +support both build types. + +A common use of a critical section would be to lock an object while accessing +an internal attribute of it. For example, if an extension type has an internal +count field, you could use a critical section while reading or writing that +field:: + + // read the count, returns new reference to internal count value + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(obj); + result = Py_NewRef(obj->count); + Py_END_CRITICAL_SECTION(); + return result; + + // write the count, consumes reference from new_count + Py_BEGIN_CRITICAL_SECTION(obj); + obj->count = new_count; + Py_END_CRITICAL_SECTION(); + + +How Critical Sections Work +.......................... + +Unlike traditional locks, critical sections do not guarantee exclusive access +throughout their entire duration. If a thread would block while holding a +critical section (e.g., by acquiring another lock or performing I/O), the +critical section is temporarily suspended—all locks are released—and then +resumed when the blocking operation completes. + +This behavior is similar to what happens with the GIL when a thread makes a +blocking call. The key differences are: + +* Critical sections operate on a per-object basis rather than globally + +* Critical sections follow a stack discipline within each thread (the "begin" and + "end" macros enforce this since they must be paired and within the same scope) + +* Critical sections automatically release and reacquire locks around potential + blocking operations + +Deadlock Avoidance +.................. + +Critical sections help avoid deadlocks in two ways: + +1. If a thread tries to acquire a lock that's already held by another thread, + it first suspends all of its active critical sections, temporarily releasing + their locks + +2. When the blocking operation completes, only the top-most critical section is + reacquired first + +This means you cannot rely on nested critical sections to lock multiple objects +at once, as the inner critical section may suspend the outer ones. Instead, use +:c:macro:`Py_BEGIN_CRITICAL_SECTION2` to lock two objects simultaneously. + +Note that the locks described above are only :c:type:`PyMutex` based locks. +The critical section implementation does not know about or affect other locking +mechanisms that might be in use, like POSIX mutexes. Also note that while +blocking on any :c:type:`PyMutex` causes the critical sections to be +suspended, only the mutexes that are part of the critical sections are +released. If :c:type:`PyMutex` is used without a critical section, it will +not be released and therefore does not get the same deadlock avoidance. + +Important Considerations +........................ + +* Critical sections may temporarily release their locks, allowing other threads + to modify the protected data. Be careful about making assumptions about the + state of the data after operations that might block. + +* Because locks can be temporarily released (suspended), entering a critical + section does not guarantee exclusive access to the protected resource + throughout the section's duration. If code within a critical section calls + another function that blocks (e.g., acquires another lock, performs blocking + I/O), all locks held by the thread via critical sections will be released. + This is similar to how the GIL can be released during blocking calls. + +* Only the lock(s) associated with the most recently entered (top-most) + critical section are guaranteed to be held at any given time. Locks for + outer, nested critical sections might have been suspended. + +* You can lock at most two objects simultaneously with these APIs. If you need + to lock more objects, you'll need to restructure your code. + +* While critical sections will not deadlock if you attempt to lock the same + object twice, they are less efficient than purpose-built reentrant locks for + this use case. + +* When using :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, the order of the objects + doesn't affect correctness (the implementation handles deadlock avoidance), + but it's good practice to always lock objects in a consistent order. + +* Remember that the critical section macros are primarily for protecting access + to *Python objects* that might be involved in internal CPython operations + susceptible to the deadlock scenarios described above. For protecting purely + internal extension state, standard mutexes or other synchronization + primitives might be more appropriate. + + Building Extensions for the Free-Threaded Build =============================================== @@ -248,8 +394,9 @@ The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. * `pypa/manylinux `_ supports the free-threaded build, with the ``t`` suffix, such as ``python3.13t``. * `pypa/cibuildwheel `_ supports the - free-threaded build if you set - `CIBW_FREE_THREADED_SUPPORT `_. + free-threaded build on Python 3.13 and 3.14. On Python 3.14, free-threaded + wheels will be built by default. On Python 3.13, you will need to set + `CIBW_ENABLE to cpython-freethreading `_. Limited C API and Stable ABI ............................ diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index b21e3287ecaa3f..380c2be04957d5 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -1,18 +1,19 @@ .. _freethreading-python-howto: -********************************************** -Python experimental support for free threading -********************************************** +********************************* +Python support for free threading +********************************* -Starting with the 3.13 release, CPython has experimental support for a build of +Starting with the 3.13 release, CPython has support for a build of Python called :term:`free threading` where the :term:`global interpreter lock` (GIL) is disabled. Free-threaded execution allows for full utilization of the available processing power by running threads in parallel on available CPU cores. While not all software will benefit from this automatically, programs designed with threading in mind will run faster on multi-core hardware. -**The free-threaded mode is experimental** and work is ongoing to improve it: -expect some bugs and a substantial single-threaded performance hit. +Some third-party packages, in particular ones +with an :term:`extension module`, may not be ready for use in a +free-threaded build, and will re-enable the :term:`GIL`. This document describes the implications of free threading for Python code. See :ref:`freethreading-extensions-howto` for information on @@ -32,7 +33,7 @@ optionally support installing free-threaded Python binaries. The installers are available at https://www.python.org/downloads/. For information on other platforms, see the `Installing a Free-Threaded Python -`_, a +`_, a community-maintained installation guide for installing free-threaded Python. When building CPython from source, the :option:`--disable-gil` configure option @@ -43,7 +44,7 @@ Identifying free-threaded Python ================================ To check if the current interpreter supports free-threading, :option:`python -VV <-V>` -and :attr:`sys.version` contain "experimental free-threading build". +and :data:`sys.version` contain "free-threading build". The new :func:`sys._is_gil_enabled` function can be used to check whether the GIL is actually disabled in the running process. @@ -98,57 +99,69 @@ This section describes known limitations of the free-threaded CPython build. Immortalization --------------- -The free-threaded build of the 3.13 release makes some objects :term:`immortal`. +In the free-threaded build, some objects are :term:`immortal`. Immortal objects are not deallocated and have reference counts that are never modified. This is done to avoid reference count contention that would prevent efficient multi-threaded scaling. -An object will be made immortal when a new thread is started for the first time -after the main thread is running. The following objects are immortalized: +As of the 3.14 release, immortalization is limited to: -* :ref:`function ` objects declared at the module level -* :ref:`method ` descriptors -* :ref:`code ` objects -* :term:`module` objects and their dictionaries -* :ref:`classes ` (type objects) - -Because immortal objects are never deallocated, applications that create many -objects of these types may see increased memory usage. This is expected to be -addressed in the 3.14 release. - -Additionally, numeric and string literals in the code as well as strings -returned by :func:`sys.intern` are also immortalized. This behavior is -expected to remain in the 3.14 free-threaded build. +* Code constants: numeric literals, string literals, and tuple literals + composed of other constants. +* Strings interned by :func:`sys.intern`. Frame objects ------------- -It is not safe to access :ref:`frame ` objects from other -threads and doing so may cause your program to crash . This means that -:func:`sys._current_frames` is generally not safe to use in a free-threaded -build. Functions like :func:`inspect.currentframe` and :func:`sys._getframe` -are generally safe as long as the resulting frame object is not passed to -another thread. +It is not safe to access :attr:`frame.f_locals` from a :ref:`frame ` +object if that frame is currently executing in another thread, and doing so may +crash the interpreter. + Iterators --------- -Sharing the same iterator object between multiple threads is generally not -safe and threads may see duplicate or missing elements when iterating or crash -the interpreter. +It is generally not thread-safe to access the same iterator object from +multiple threads concurrently, and threads may see duplicate or missing +elements. Single-threaded performance --------------------------- The free-threaded build has additional overhead when executing Python code -compared to the default GIL-enabled build. In 3.13, this overhead is about -40% on the `pyperformance `_ suite. -Programs that spend most of their time in C extensions or I/O will see -less of an impact. The largest impact is because the specializing adaptive -interpreter (:pep:`659`) is disabled in the free-threaded build. We expect -to re-enable it in a thread-safe way in the 3.14 release. This overhead is -expected to be reduced in upcoming Python release. We are aiming for an -overhead of 10% or less on the pyperformance suite compared to the default -GIL-enabled build. +compared to the default GIL-enabled build. The amount of overhead depends +on the workload and hardware. On the pyperformance benchmark suite, the +average overhead ranges from about 1% on macOS aarch64 to 8% on x86-64 Linux +systems. + + +Behavioral changes +================== + +This section describes CPython behavioural changes with the free-threaded +build. + + +Context variables +----------------- + +In the free-threaded build, the flag :data:`~sys.flags.thread_inherit_context` +is set to true by default which causes threads created with +:class:`threading.Thread` to start with a copy of the +:class:`~contextvars.Context()` of the caller of +:meth:`~threading.Thread.start`. In the default GIL-enabled build, the flag +defaults to false so threads start with an +empty :class:`~contextvars.Context()`. + + +Warning filters +--------------- + +In the free-threaded build, the flag :data:`~sys.flags.context_aware_warnings` +is set to true by default. In the default GIL-enabled build, the flag defaults +to false. If the flag is true then the :class:`warnings.catch_warnings` +context manager uses a context variable for warning filters. If the flag is +false then :class:`~warnings.catch_warnings` modifies the global filters list, +which is not thread-safe. See the :mod:`warnings` module for more details. diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 1f0608fb0fc53f..552514063c95ab 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -4,7 +4,7 @@ Functional Programming HOWTO ******************************** -:Author: A. M. Kuchling +:Author: \A. M. Kuchling :Release: 0.32 In this document, we'll take a tour of Python's features suitable for @@ -372,8 +372,8 @@ have the form:: for expr2 in sequence2 if condition2 for expr3 in sequence3 - ... if condition3 + ... for exprN in sequenceN if conditionN ) @@ -602,7 +602,7 @@ generators: raise an exception inside the generator; the exception is raised by the ``yield`` expression where the generator's execution is paused. -* :meth:`~generator.close` raises a :exc:`GeneratorExit` exception inside the +* :meth:`~generator.close` sends a :exc:`GeneratorExit` exception to the generator to terminate the iteration. On receiving this exception, the generator's code must either raise :exc:`GeneratorExit` or :exc:`StopIteration`; catching the exception and doing anything else is @@ -1217,7 +1217,7 @@ flow inside a program. The book uses Scheme for its examples, but many of the design approaches described in these chapters are applicable to functional-style Python code. -https://www.defmacro.org/ramblings/fp.html: A general introduction to functional +https://defmacro.org/2006/06/19/fp.html: A general introduction to functional programming that uses Java examples and has a lengthy historical introduction. https://en.wikipedia.org/wiki/Functional_programming: General Wikipedia entry diff --git a/Doc/howto/gdb_helpers.rst b/Doc/howto/gdb_helpers.rst index 53bbf7ddaa2ab9..33d1fbf8cd9e9e 100644 --- a/Doc/howto/gdb_helpers.rst +++ b/Doc/howto/gdb_helpers.rst @@ -136,7 +136,7 @@ enabled:: at Objects/unicodeobject.c:551 #7 0x0000000000440d94 in PyUnicodeUCS2_FromString (u=0x5c2b8d "__lltrace__") at Objects/unicodeobject.c:569 #8 0x0000000000584abd in PyDict_GetItemString (v= - {'Yuck': , '__builtins__': , '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': , 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__cached__': None, '__name__': '__main__', 'z': , '__doc__': None}, key= + {'Yuck': , '__builtins__': , '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': , 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__name__': '__main__', 'z': , '__doc__': None}, key= 0x5c2b8d "__lltrace__") at Objects/dictobject.c:2171 Notice how the dictionary argument to ``PyDict_GetItemString`` is displayed @@ -180,7 +180,7 @@ regular machine-level integer:: (gdb) p some_python_integer $4 = 42 -The internal structure can be revealed with a cast to :c:expr:`PyLongObject *`: +The internal structure can be revealed with a cast to :c:expr:`PyLongObject *`:: (gdb) p *(PyLongObject*)some_python_integer $5 = {ob_base = {ob_base = {ob_refcnt = 8, ob_type = 0x3dad39f5e0}, ob_size = 1}, diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index c09f92c9528ee1..81fc7e63f35bd7 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -1,3 +1,5 @@ +.. _how-tos: + *************** Python HOWTOs *************** @@ -11,6 +13,7 @@ Python Library Reference. :maxdepth: 1 :hidden: + a-conceptual-overview-of-asyncio.rst cporting.rst curses.rst descriptor.rst @@ -34,9 +37,11 @@ Python Library Reference. mro.rst free-threading-python.rst free-threading-extensions.rst + remote_debugging.rst General: +* :ref:`a-conceptual-overview-of-asyncio` * :ref:`annotations-howto` * :ref:`argparse-tutorial` * :ref:`descriptorhowto` @@ -66,3 +71,4 @@ Debugging and profiling: * :ref:`gdb` * :ref:`instrumentation` * :ref:`perf_profiling` +* :ref:`remote-debugging` diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 6e03ef20a21fa3..b3db1189e5dcbc 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -269,6 +269,8 @@ should instead read: (assuming a :ref:`debug build ` of CPython 3.6) +.. _static-markers: + Available static markers ------------------------ diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index a636e06bda8344..6092c75f48fdef 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -168,7 +168,7 @@ possible, consider explicit locking. If it is necessary to use process-global state, the simplest way to avoid issues with multiple interpreters is to explicitly prevent a module from being loaded more than once per process—see -`Opt-Out: Limiting to One Module Object per Process`_. +:ref:`isolating-extensions-optout`. Managing Per-Module State @@ -207,6 +207,8 @@ An example of a module with per-module state is currently available as example module initialization shown at the bottom of the file. +.. _isolating-extensions-optout: + Opt-Out: Limiting to One Module Object per Process -------------------------------------------------- @@ -215,21 +217,36 @@ multiple interpreters correctly. If this is not yet the case for your module, you can explicitly make your module loadable only once per process. For example:: + // A process-wide flag static int loaded = 0; + // Mutex to provide thread safety (only needed for free-threaded Python) + static PyMutex modinit_mutex = {0}; + static int exec_module(PyObject* module) { + PyMutex_Lock(&modinit_mutex); if (loaded) { + PyMutex_Unlock(&modinit_mutex); PyErr_SetString(PyExc_ImportError, "cannot load module more than once per process"); return -1; } loaded = 1; + PyMutex_Unlock(&modinit_mutex); // ... rest of initialization } +If your module's :c:member:`PyModuleDef.m_clear` function is able to prepare +for future re-initialization, it should clear the ``loaded`` flag. +In this case, your module won't support multiple instances existing +*concurrently*, but it will, for example, support being loaded after +Python runtime shutdown (:c:func:`Py_FinalizeEx`) and re-initialization +(:c:func:`Py_Initialize`). + + Module State Access from Functions ---------------------------------- @@ -336,7 +353,7 @@ garbage collection protocol. That is, heap types should: - Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. -- Define a traverse function using ``Py_tp_traverse``, which +- Define a traverse function using :c:data:`Py_tp_traverse`, which visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). Please refer to the documentation of @@ -436,7 +453,7 @@ Avoiding ``PyObject_New`` GC-tracked objects need to be allocated using GC-aware functions. -If you use use :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`: +If you use :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`: - Get and call type's :c:member:`~PyTypeObject.tp_alloc` slot, if possible. That is, replace ``TYPE *o = PyObject_New(TYPE, typeobj)`` with:: @@ -609,8 +626,7 @@ Open Issues Several issues around per-module state and heap types are still open. -Discussions about improving the situation are best held on the `capi-sig -mailing list `__. +Discussions about improving the situation are best held on the `discuss forum under c-api tag `__. Per-Class Scope diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 321ec0c0f73871..52537a91df542c 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -626,6 +626,19 @@ which, when run, will produce: of each message with the handler's level, and only passes a message to a handler if it's appropriate to do so. +.. versionchanged:: 3.14 + The :class:`QueueListener` can be started (and stopped) via the + :keyword:`with` statement. For example: + + .. code-block:: python + + with QueueListener(que, handler) as listener: + # The queue listener automatically starts + # when the 'with' block is entered. + pass + # The queue listener automatically stops once + # the 'with' block is exited. + .. _network-logging: Sending and receiving logging events across a network @@ -825,9 +838,9 @@ To test these files, do the following in a POSIX environment: which will lead to records being written to the log. #. Inspect the log files in the :file:`run` subdirectory. You should see the - most recent log lines in files matching the pattern :file:`app.log*`. They won't be in - any particular order, since they have been handled concurrently by different - worker processes in a non-deterministic way. + most recent log lines in files matching the pattern :file:`app.log*`. They + won't be in any particular order, since they have been handled concurrently + by different worker processes in a non-deterministic way. #. You can shut down the listener and the web application by running ``venv/bin/supervisorctl -c supervisor.conf shutdown``. @@ -835,6 +848,19 @@ To test these files, do the following in a POSIX environment: You may need to tweak the configuration files in the unlikely event that the configured ports clash with something else in your test environment. +The default configuration uses a TCP socket on port 9020. You can use a Unix +Domain socket instead of a TCP socket by doing the following: + +#. In :file:`listener.json`, add a ``socket`` key with the path to the domain + socket you want to use. If this key is present, the listener listens on the + corresponding domain socket and not on a TCP socket (the ``port`` key is + ignored). + +#. In :file:`webapp.json`, change the socket handler configuration dictionary + so that the ``host`` value is the path to the domain socket, and set the + ``port`` value to ``null``. + + .. currentmodule:: logging .. _context-info: @@ -1267,11 +1293,8 @@ to adapt in your own applications. You could also write your own handler which uses the :class:`~multiprocessing.Lock` class from the :mod:`multiprocessing` module to serialize access to the -file from your processes. The existing :class:`FileHandler` and subclasses do -not make use of :mod:`multiprocessing` at present, though they may do so in the -future. Note that at present, the :mod:`multiprocessing` module does not provide -working lock functionality on all platforms (see -https://bugs.python.org/issue3770). +file from your processes. The stdlib :class:`FileHandler` and subclasses do +not make use of :mod:`multiprocessing`. .. currentmodule:: logging.handlers @@ -4055,6 +4078,104 @@ lines. With this approach, you get better output: WARNING:demo: 1/0 WARNING:demo:ZeroDivisionError: division by zero +How to uniformly handle newlines in logging output +-------------------------------------------------- + +Usually, messages that are logged (say to console or file) consist of a single +line of text. However, sometimes there is a need to handle messages with +multiple lines - whether because a logging format string contains newlines, or +logged data contains newlines. If you want to handle such messages uniformly, so +that each line in the logged message appears uniformly formatted as if it was +logged separately, you can do this using a handler mixin, as in the following +snippet: + +.. code-block:: python + + # Assume this is in a module mymixins.py + import copy + + class MultilineMixin: + def emit(self, record): + s = record.getMessage() + if '\n' not in s: + super().emit(record) + else: + lines = s.splitlines() + rec = copy.copy(record) + rec.args = None + for line in lines: + rec.msg = line + super().emit(rec) + +You can use the mixin as in the following script: + +.. code-block:: python + + import logging + + from mymixins import MultilineMixin + + logger = logging.getLogger(__name__) + + class StreamHandler(MultilineMixin, logging.StreamHandler): + pass + + if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)-9s %(message)s', + handlers = [StreamHandler()]) + logger.debug('Single line') + logger.debug('Multiple lines:\nfool me once ...') + logger.debug('Another single line') + logger.debug('Multiple lines:\n%s', 'fool me ...\ncan\'t get fooled again') + +The script, when run, prints something like: + +.. code-block:: text + + 2025-07-02 13:54:47,234 DEBUG Single line + 2025-07-02 13:54:47,234 DEBUG Multiple lines: + 2025-07-02 13:54:47,234 DEBUG fool me once ... + 2025-07-02 13:54:47,234 DEBUG Another single line + 2025-07-02 13:54:47,234 DEBUG Multiple lines: + 2025-07-02 13:54:47,234 DEBUG fool me ... + 2025-07-02 13:54:47,234 DEBUG can't get fooled again + +If, on the other hand, you are concerned about `log injection +`_, you can use a +formatter which escapes newlines, as per the following example: + +.. code-block:: python + + import logging + + logger = logging.getLogger(__name__) + + class EscapingFormatter(logging.Formatter): + def format(self, record): + s = super().format(record) + return s.replace('\n', r'\n') + + if __name__ == '__main__': + h = logging.StreamHandler() + h.setFormatter(EscapingFormatter('%(asctime)s %(levelname)-9s %(message)s')) + logging.basicConfig(level=logging.DEBUG, handlers = [h]) + logger.debug('Single line') + logger.debug('Multiple lines:\nfool me once ...') + logger.debug('Another single line') + logger.debug('Multiple lines:\n%s', 'fool me ...\ncan\'t get fooled again') + +You can, of course, use whatever escaping scheme makes the most sense for you. +The script, when run, should produce output like this: + +.. code-block:: text + + 2025-07-09 06:47:33,783 DEBUG Single line + 2025-07-09 06:47:33,783 DEBUG Multiple lines:\nfool me once ... + 2025-07-09 06:47:33,783 DEBUG Another single line + 2025-07-09 06:47:33,783 DEBUG Multiple lines:\nfool me ...\ncan't get fooled again + +Escaping behaviour can't be the stdlib default , as it would break backwards +compatibility. .. patterns-to-avoid: diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 3182d5664ab6ec..b7225ff1c2cbfc 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -127,7 +127,7 @@ that; formatting options will also be explained later. Notice that in this example, we use functions directly on the ``logging`` module, like ``logging.debug``, rather than creating a logger and calling -functions on it. These functions operation on the root logger, but can be useful +functions on it. These functions operate on the root logger, but can be useful as they will call :func:`~logging.basicConfig` for you if it has not been called yet, like in this example. In larger programs you'll usually want to control the logging configuration explicitly however - so for that reason as well as others, it's @@ -302,10 +302,10 @@ reading the following sections. If you're ready for that, grab some of your favourite beverage and carry on. If your logging needs are simple, then use the above examples to incorporate -logging into your own scripts, and if you run into problems or don't -understand something, please post a question on the comp.lang.python Usenet -group (available at https://groups.google.com/g/comp.lang.python) and you -should receive help before too long. +logging into your own scripts, and if you run into problems or don't understand +something, please post a question in the Help category of the `Python +discussion forum `_ and you should receive +help before too long. Still here? You can carry on reading the next few sections, which provide a slightly more advanced/in-depth tutorial than the basic one above. After that, diff --git a/Doc/howto/mro.rst b/Doc/howto/mro.rst index 46db516e16dae4..0872bedcd3a2d3 100644 --- a/Doc/howto/mro.rst +++ b/Doc/howto/mro.rst @@ -398,7 +398,7 @@ with inheritance diagram We see that class G inherits from F and E, with F *before* E: therefore we would expect the attribute *G.remember2buy* to be inherited by -*F.rembermer2buy* and not by *E.remember2buy*: nevertheless Python 2.2 +*F.remember2buy* and not by *E.remember2buy*: nevertheless Python 2.2 gives >>> G.remember2buy # doctest: +SKIP diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index 06459d1b222964..fc4772bbccab57 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -2,34 +2,35 @@ .. _perf_profiling: -============================================== -Python support for the Linux ``perf`` profiler -============================================== +======================================================== +Python support for the ``perf map`` compatible profilers +======================================================== :author: Pablo Galindo -`The Linux perf profiler `_ -is a very powerful tool that allows you to profile and obtain -information about the performance of your application. -``perf`` also has a very vibrant ecosystem of tools -that aid with the analysis of the data that it produces. +`The Linux perf profiler `_ and +`samply `_ are powerful tools that allow you to +profile and obtain information about the performance of your application. +Both tools have vibrant ecosystems that aid with the analysis of the data they produce. -The main problem with using the ``perf`` profiler with Python applications is that -``perf`` only gets information about native symbols, that is, the names of +The main problem with using these profilers with Python applications is that +they only get information about native symbols, that is, the names of functions and procedures written in C. This means that the names and file names -of Python functions in your code will not appear in the output of ``perf``. +of Python functions in your code will not appear in the profiler output. Since Python 3.12, the interpreter can run in a special mode that allows Python -functions to appear in the output of the ``perf`` profiler. When this mode is +functions to appear in the output of compatible profilers. When this mode is enabled, the interpreter will interpose a small piece of code compiled on the -fly before the execution of every Python function and it will teach ``perf`` the +fly before the execution of every Python function and it will teach the profiler the relationship between this piece of code and the associated Python function using :doc:`perf map files <../c-api/perfmaps>`. .. note:: - Support for the ``perf`` profiler is currently only available for Linux on - select architectures. Check the output of the ``configure`` build step or + Support for profiling is available on Linux and macOS on select architectures. + Perf is available on Linux, while samply can be used on both Linux and macOS. + samply support on macOS is available starting from Python 3.15. + Check the output of the ``configure`` build step or check the output of ``python -m sysconfig | grep HAVE_PERF_TRAMPOLINE`` to see if your system is supported. @@ -92,7 +93,7 @@ Then we can use ``perf report`` to analyze the data: | | | | | | | |--51.67%--_PyEval_EvalFrameDefault | | | | | - | | | | |--11.52%--_PyLong_Add + | | | | |--11.52%--_PyCompactLong_Add | | | | | | | | | | | |--2.97%--_PyObject_Malloc ... @@ -142,12 +143,37 @@ Instead, if we run the same experiment with ``perf`` support enabled we get: | | | | | | | |--51.81%--_PyEval_EvalFrameDefault | | | | | - | | | | |--13.77%--_PyLong_Add + | | | | |--13.77%--_PyCompactLong_Add | | | | | | | | | | | |--3.26%--_PyObject_Malloc +Using the samply profiler +------------------------- + +samply is a modern profiler that can be used as an alternative to perf. +It uses the same perf map files that Python generates, making it compatible +with Python's profiling support. samply is particularly useful on macOS +where perf is not available. + +To use samply with Python, first install it following the instructions at +https://github.com/mstange/samply, then run:: + + $ samply record PYTHONPERFSUPPORT=1 python my_script.py + +This will open a web interface where you can analyze the profiling data +interactively. The advantage of samply is that it provides a modern +web-based interface for analyzing profiling data and works on both Linux +and macOS. + +On macOS, samply support requires Python 3.15 or later. Also on macOS, samply +can't profile signed Python executables due to restrictions by macOS. You can +profile with Python binaries that you've compiled yourself, or which are +unsigned or locally-signed (such as anything installed by Homebrew). In +order to attach to running processes on macOS, run ``samply setup`` once (and +every time samply is updated) to self-sign the samply binary. + How to enable ``perf`` profiling support ---------------------------------------- @@ -162,12 +188,12 @@ the :option:`!-X` option takes precedence over the environment variable. Example, using the environment variable:: - $ PYTHONPERFSUPPORT=1 perf record -F 9999 -g -o perf.data python script.py + $ PYTHONPERFSUPPORT=1 perf record -F 9999 -g -o perf.data python my_script.py $ perf report -g -i perf.data Example, using the :option:`!-X` option:: - $ perf record -F 9999 -g -o perf.data python -X perf script.py + $ perf record -F 9999 -g -o perf.data python -X perf my_script.py $ perf report -g -i perf.data Example, using the :mod:`sys` APIs in file :file:`example.py`: @@ -236,7 +262,7 @@ When using the perf JIT mode, you need an extra step before you can run ``perf report``. You need to call the ``perf inject`` command to inject the JIT information into the ``perf.data`` file.:: - $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperf_jit my_script.py + $ perf record -F 9999 -g -k 1 --call-graph dwarf -o perf.data python -Xperf_jit my_script.py $ perf inject -i perf.data --jit --output perf.jit.data $ perf report -g -i perf.jit.data @@ -254,13 +280,28 @@ files in the current directory which are ELF images for all the JIT trampolines that were created by Python. .. warning:: - Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take + When using ``--call-graph dwarf``, the ``perf`` tool will take snapshots of the stack of the process being profiled and save the - information in the ``perf.data`` file. By default the size of the stack dump - is 8192 bytes but the user can change the size by passing the size after - comma like ``--call-graph dwarf,4096``. The size of the stack dump is - important because if the size is too small ``perf`` will not be able to - unwind the stack and the output will be incomplete. On the other hand, if - the size is too big, then ``perf`` won't be able to sample the process as - frequently as it would like as the overhead will be higher. + information in the ``perf.data`` file. By default, the size of the stack dump + is 8192 bytes, but you can change the size by passing it after + a comma like ``--call-graph dwarf,16384``. + + The size of the stack dump is important because if the size is too small + ``perf`` will not be able to unwind the stack and the output will be + incomplete. On the other hand, if the size is too big, then ``perf`` won't + be able to sample the process as frequently as it would like as the overhead + will be higher. + + The stack size is particularly important when profiling Python code compiled + with low optimization levels (like ``-O0``), as these builds tend to have + larger stack frames. If you are compiling Python with ``-O0`` and not seeing + Python functions in your profiling output, try increasing the stack dump + size to 65528 bytes (the maximum):: + + $ perf record -F 9999 -g -k 1 --call-graph dwarf,65528 -o perf.data python -Xperf_jit my_script.py + + Different compilation flags can significantly impact stack sizes: + - Builds with ``-O0`` typically have much larger stack frames than those with ``-O1`` or higher + - Adding optimizations (``-O1``, ``-O2``, etc.) typically reduces stack size + - Frame pointers (``-fno-omit-frame-pointer``) generally provide more reliable stack unwinding diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index 5e2f9a9d1837fe..7486a378dbb06f 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -738,9 +738,12 @@ given location, they can obviously be matched an infinite number of times. different: ``\A`` still matches only at the beginning of the string, but ``^`` may match at any location inside the string that follows a newline character. -``\Z`` +``\z`` Matches only at the end of the string. +``\Z`` + The same as ``\z``. For compatibility with old Python versions. + ``\b`` Word boundary. This is a zero-width assertion that matches only at the beginning or end of a word. A word is defined as a sequence of alphanumeric @@ -1013,7 +1016,9 @@ extension. This regular expression matches ``foo.bar`` and Now, consider complicating the problem a bit; what if you want to match filenames where the extension is not ``bat``? Some incorrect attempts: -``.*[.][^b].*$`` The first attempt above tries to exclude ``bat`` by requiring +``.*[.][^b].*$`` + +The first attempt above tries to exclude ``bat`` by requiring that the first character of the extension is not a ``b``. This is wrong, because the pattern also doesn't match ``foo.bar``. @@ -1040,7 +1045,9 @@ confusing. A negative lookahead cuts through all this confusion: -``.*[.](?!bat$)[^.]*$`` The negative lookahead means: if the expression ``bat`` +``.*[.](?!bat$)[^.]*$`` + +The negative lookahead means: if the expression ``bat`` doesn't match at this point, try the rest of the pattern; if ``bat$`` does match, the whole pattern will fail. The trailing ``$`` is required to ensure that something like ``sample.batch``, where the extension only starts with diff --git a/Doc/howto/remote_debugging.rst b/Doc/howto/remote_debugging.rst new file mode 100644 index 00000000000000..dfe0176b75a020 --- /dev/null +++ b/Doc/howto/remote_debugging.rst @@ -0,0 +1,626 @@ +.. _remote-debugging: + +Remote debugging attachment protocol +==================================== + +This protocol enables external tools to attach to a running CPython process and +execute Python code remotely. + +Most platforms require elevated privileges to attach to another Python process. + +Disabling remote debugging +-------------------------- + +To disable remote debugging support, use any of the following: + +* Set the :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable to ``1`` before + starting the interpreter. +* Use the :option:`-X disable_remote_debug` command-line option. +* Compile Python with the :option:`--without-remote-debug` build flag. + +.. _permission-requirements: + +Permission requirements +======================= + +Attaching to a running Python process for remote debugging requires elevated +privileges on most platforms. The specific requirements and troubleshooting +steps depend on your operating system: + +.. rubric:: Linux + +The tracer process must have the ``CAP_SYS_PTRACE`` capability or equivalent +privileges. You can only trace processes you own and can signal. Tracing may +fail if the process is already being traced, or if it is running with +set-user-ID or set-group-ID. Security modules like Yama may further restrict +tracing. + +To temporarily relax ptrace restrictions (until reboot), run: + + ``echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope`` + +.. note:: + + Disabling ``ptrace_scope`` reduces system hardening and should only be done + in trusted environments. + +If running inside a container, use ``--cap-add=SYS_PTRACE`` or +``--privileged``, and run as root if needed. + +Try re-running the command with elevated privileges: + + ``sudo -E !!`` + + +.. rubric:: macOS + +To attach to another process, you typically need to run your debugging tool +with elevated privileges. This can be done by using ``sudo`` or running as +root. + +Even when attaching to processes you own, macOS may block debugging unless +the debugger is run with root privileges due to system security restrictions. + + +.. rubric:: Windows + +To attach to another process, you usually need to run your debugging tool +with administrative privileges. Start the command prompt or terminal as +Administrator. + +Some processes may still be inaccessible even with Administrator rights, +unless you have the ``SeDebugPrivilege`` privilege enabled. + +To resolve file or folder access issues, adjust the security permissions: + + 1. Right-click the file or folder and select **Properties**. + 2. Go to the **Security** tab to view users and groups with access. + 3. Click **Edit** to modify permissions. + 4. Select your user account. + 5. In **Permissions**, check **Read** or **Full control** as needed. + 6. Click **Apply**, then **OK** to confirm. + + +.. note:: + + Ensure you've satisfied all :ref:`permission-requirements` before proceeding. + +This section describes the low-level protocol that enables external tools to +inject and execute a Python script within a running CPython process. + +This mechanism forms the basis of the :func:`sys.remote_exec` function, which +instructs a remote Python process to execute a ``.py`` file. However, this +section does not document the usage of that function. Instead, it provides a +detailed explanation of the underlying protocol, which takes as input the +``pid`` of a target Python process and the path to a Python source file to be +executed. This information supports independent reimplementation of the +protocol, regardless of programming language. + +.. warning:: + + The execution of the injected script depends on the interpreter reaching a + safe evaluation point. As a result, execution may be delayed depending on + the runtime state of the target process. + +Once injected, the script is executed by the interpreter within the target +process the next time a safe evaluation point is reached. This approach enables +remote execution capabilities without modifying the behavior or structure of +the running Python application. + +Subsequent sections provide a step-by-step description of the protocol, +including techniques for locating interpreter structures in memory, safely +accessing internal fields, and triggering code execution. Platform-specific +variations are noted where applicable, and example implementations are included +to clarify each operation. + +Locating the PyRuntime structure +================================ + +CPython places the ``PyRuntime`` structure in a dedicated binary section to +help external tools find it at runtime. The name and format of this section +vary by platform. For example, ``.PyRuntime`` is used on ELF systems, and +``__DATA,__PyRuntime`` is used on macOS. Tools can find the offset of this +structure by examining the binary on disk. + +The ``PyRuntime`` structure contains CPython’s global interpreter state and +provides access to other internal data, including the list of interpreters, +thread states, and debugger support fields. + +To work with a remote Python process, a debugger must first find the memory +address of the ``PyRuntime`` structure in the target process. This address +can’t be hardcoded or calculated from a symbol name, because it depends on +where the operating system loaded the binary. + +The method for finding ``PyRuntime`` depends on the platform, but the steps are +the same in general: + +1. Find the base address where the Python binary or shared library was loaded + in the target process. +2. Use the on-disk binary to locate the offset of the ``.PyRuntime`` section. +3. Add the section offset to the base address to compute the address in memory. + +The sections below explain how to do this on each supported platform and +include example code. + +.. rubric:: Linux (ELF) + +To find the ``PyRuntime`` structure on Linux: + +1. Read the process’s memory map (for example, ``/proc//maps``) to find + the address where the Python executable or ``libpython`` was loaded. +2. Parse the ELF section headers in the binary to get the offset of the + ``.PyRuntime`` section. +3. Add that offset to the base address from step 1 to get the memory address of + ``PyRuntime``. + +The following is an example implementation:: + + def find_py_runtime_linux(pid: int) -> int: + # Step 1: Try to find the Python executable in memory + binary_path, base_address = find_mapped_binary( + pid, name_contains="python" + ) + + # Step 2: Fallback to shared library if executable is not found + if binary_path is None: + binary_path, base_address = find_mapped_binary( + pid, name_contains="libpython" + ) + + # Step 3: Parse ELF headers to get .PyRuntime section offset + section_offset = parse_elf_section_offset( + binary_path, ".PyRuntime" + ) + + # Step 4: Compute PyRuntime address in memory + return base_address + section_offset + + +On Linux systems, there are two main approaches to read memory from another +process. The first is through the ``/proc`` filesystem, specifically by reading from +``/proc/[pid]/mem`` which provides direct access to the process's memory. This +requires appropriate permissions - either being the same user as the target +process or having root access. The second approach is using the +``process_vm_readv()`` system call which provides a more efficient way to copy +memory between processes. While ptrace's ``PTRACE_PEEKTEXT`` operation can also be +used to read memory, it is significantly slower as it only reads one word at a +time and requires multiple context switches between the tracer and tracee +processes. + +For parsing ELF sections, the process involves reading and interpreting the ELF +file format structures from the binary file on disk. The ELF header contains a +pointer to the section header table. Each section header contains metadata about +a section including its name (stored in a separate string table), offset, and +size. To find a specific section like .PyRuntime, you need to walk through these +headers and match the section name. The section header then provides the offset +where that section exists in the file, which can be used to calculate its +runtime address when the binary is loaded into memory. + +You can read more about the ELF file format in the `ELF specification +`_. + + +.. rubric:: macOS (Mach-O) + +To find the ``PyRuntime`` structure on macOS: + +1. Call ``task_for_pid()`` to get the ``mach_port_t`` task port for the target + process. This handle is needed to read memory using APIs like + ``mach_vm_read_overwrite`` and ``mach_vm_region``. +2. Scan the memory regions to find the one containing the Python executable or + ``libpython``. +3. Load the binary file from disk and parse the Mach-O headers to find the + section named ``PyRuntime`` in the ``__DATA`` segment. On macOS, symbol + names are automatically prefixed with an underscore, so the ``PyRuntime`` + symbol appears as ``_PyRuntime`` in the symbol table, but the section name + is not affected. + +The following is an example implementation:: + + def find_py_runtime_macos(pid: int) -> int: + # Step 1: Get access to the process's memory + handle = get_memory_access_handle(pid) + + # Step 2: Try to find the Python executable in memory + binary_path, base_address = find_mapped_binary( + handle, name_contains="python" + ) + + # Step 3: Fallback to libpython if the executable is not found + if binary_path is None: + binary_path, base_address = find_mapped_binary( + handle, name_contains="libpython" + ) + + # Step 4: Parse Mach-O headers to get __DATA,__PyRuntime section offset + section_offset = parse_macho_section_offset( + binary_path, "__DATA", "__PyRuntime" + ) + + # Step 5: Compute the PyRuntime address in memory + return base_address + section_offset + +On macOS, accessing another process's memory requires using Mach-O specific APIs +and file formats. The first step is obtaining a ``task_port`` handle via +``task_for_pid()``, which provides access to the target process's memory space. +This handle enables memory operations through APIs like +``mach_vm_read_overwrite()``. + +The process memory can be examined using ``mach_vm_region()`` to scan through the +virtual memory space, while ``proc_regionfilename()`` helps identify which binary +files are loaded at each memory region. When the Python binary or library is +found, its Mach-O headers need to be parsed to locate the ``PyRuntime`` structure. + +The Mach-O format organizes code and data into segments and sections. The +``PyRuntime`` structure lives in a section named ``__PyRuntime`` within the +``__DATA`` segment. The actual runtime address calculation involves finding the +``__TEXT`` segment which serves as the binary's base address, then locating the +``__DATA`` segment containing our target section. The final address is computed by +combining the base address with the appropriate section offsets from the Mach-O +headers. + +Note that accessing another process's memory on macOS typically requires +elevated privileges - either root access or special security entitlements +granted to the debugging process. + + +.. rubric:: Windows (PE) + +To find the ``PyRuntime`` structure on Windows: + +1. Use the ToolHelp API to enumerate all modules loaded in the target process. + This is done using functions such as `CreateToolhelp32Snapshot + `_, + `Module32First + `_, + and `Module32Next + `_. +2. Identify the module corresponding to :file:`python.exe` or + :file:`python{XY}.dll`, where ``X`` and ``Y`` are the major and minor + version numbers of the Python version, and record its base address. +3. Locate the ``PyRuntim`` section. Due to the PE format's 8-character limit + on section names (defined as ``IMAGE_SIZEOF_SHORT_NAME``), the original + name ``PyRuntime`` is truncated. This section contains the ``PyRuntime`` + structure. +4. Retrieve the section’s relative virtual address (RVA) and add it to the base + address of the module. + +The following is an example implementation:: + + def find_py_runtime_windows(pid: int) -> int: + # Step 1: Try to find the Python executable in memory + binary_path, base_address = find_loaded_module( + pid, name_contains="python" + ) + + # Step 2: Fallback to shared pythonXY.dll if the executable is not + # found + if binary_path is None: + binary_path, base_address = find_loaded_module( + pid, name_contains="python3" + ) + + # Step 3: Parse PE section headers to get the RVA of the PyRuntime + # section. The section name appears as "PyRuntim" due to the + # 8-character limit defined by the PE format (IMAGE_SIZEOF_SHORT_NAME). + section_rva = parse_pe_section_offset(binary_path, "PyRuntim") + + # Step 4: Compute PyRuntime address in memory + return base_address + section_rva + + +On Windows, accessing another process's memory requires using the Windows API +functions like ``CreateToolhelp32Snapshot()`` and ``Module32First()/Module32Next()`` +to enumerate loaded modules. The ``OpenProcess()`` function provides a handle to +access the target process's memory space, enabling memory operations through +``ReadProcessMemory()``. + +The process memory can be examined by enumerating loaded modules to find the +Python binary or DLL. When found, its PE headers need to be parsed to locate the +``PyRuntime`` structure. + +The PE format organizes code and data into sections. The ``PyRuntime`` structure +lives in a section named "PyRuntim" (truncated from "PyRuntime" due to PE's +8-character name limit). The actual runtime address calculation involves finding +the module's base address from the module entry, then locating our target +section in the PE headers. The final address is computed by combining the base +address with the section's virtual address from the PE section headers. + +Note that accessing another process's memory on Windows typically requires +appropriate privileges - either administrative access or the ``SeDebugPrivilege`` +privilege granted to the debugging process. + + +Reading _Py_DebugOffsets +======================== + +Once the address of the ``PyRuntime`` structure has been determined, the next +step is to read the ``_Py_DebugOffsets`` structure located at the beginning of +the ``PyRuntime`` block. + +This structure provides version-specific field offsets that are needed to +safely read interpreter and thread state memory. These offsets vary between +CPython versions and must be checked before use to ensure they are compatible. + +To read and check the debug offsets, follow these steps: + +1. Read memory from the target process starting at the ``PyRuntime`` address, + covering the same number of bytes as the ``_Py_DebugOffsets`` structure. + This structure is located at the very start of the ``PyRuntime`` memory + block. Its layout is defined in CPython’s internal headers and stays the + same within a given minor version, but may change in major versions. + +2. Check that the structure contains valid data: + + - The ``cookie`` field must match the expected debug marker. + - The ``version`` field must match the version of the Python interpreter + used by the debugger. + - If either the debugger or the target process is using a pre-release + version (for example, an alpha, beta, or release candidate), the versions + must match exactly. + - The ``free_threaded`` field must have the same value in both the debugger + and the target process. + +3. If the structure is valid, the offsets it contains can be used to locate + fields in memory. If any check fails, the debugger should stop the operation + to avoid reading memory in the wrong format. + +The following is an example implementation that reads and checks +``_Py_DebugOffsets``:: + + def read_debug_offsets(pid: int, py_runtime_addr: int) -> DebugOffsets: + # Step 1: Read memory from the target process at the PyRuntime address + data = read_process_memory( + pid, address=py_runtime_addr, size=DEBUG_OFFSETS_SIZE + ) + + # Step 2: Deserialize the raw bytes into a _Py_DebugOffsets structure + debug_offsets = parse_debug_offsets(data) + + # Step 3: Validate the contents of the structure + if debug_offsets.cookie != EXPECTED_COOKIE: + raise RuntimeError("Invalid or missing debug cookie") + if debug_offsets.version != LOCAL_PYTHON_VERSION: + raise RuntimeError( + "Mismatch between caller and target Python versions" + ) + if debug_offsets.free_threaded != LOCAL_FREE_THREADED: + raise RuntimeError("Mismatch in free-threaded configuration") + + return debug_offsets + + + +.. warning:: + + **Process suspension recommended** + + To avoid race conditions and ensure memory consistency, it is strongly + recommended that the target process be suspended before performing any + operations that read or write internal interpreter state. The Python runtime + may concurrently mutate interpreter data structures—such as creating or + destroying threads—during normal execution. This can result in invalid + memory reads or writes. + + A debugger may suspend execution by attaching to the process with ``ptrace`` + or by sending a ``SIGSTOP`` signal. Execution should only be resumed after + debugger-side memory operations are complete. + + .. note:: + + Some tools, such as profilers or sampling-based debuggers, may operate on + a running process without suspension. In such cases, tools must be + explicitly designed to handle partially updated or inconsistent memory. + For most debugger implementations, suspending the process remains the + safest and most robust approach. + + +Locating the interpreter and thread state +========================================= + +Before code can be injected and executed in a remote Python process, the +debugger must choose a thread in which to schedule execution. This is necessary +because the control fields used to perform remote code injection are located in +the ``_PyRemoteDebuggerSupport`` structure, which is embedded in a +``PyThreadState`` object. These fields are modified by the debugger to request +execution of injected scripts. + +The ``PyThreadState`` structure represents a thread running inside a Python +interpreter. It maintains the thread’s evaluation context and contains the +fields required for debugger coordination. Locating a valid ``PyThreadState`` +is therefore a key prerequisite for triggering execution remotely. + +A thread is typically selected based on its role or ID. In most cases, the main +thread is used, but some tools may target a specific thread by its native +thread ID. Once the target thread is chosen, the debugger must locate both the +interpreter and the associated thread state structures in memory. + +The relevant internal structures are defined as follows: + +- ``PyInterpreterState`` represents an isolated Python interpreter instance. + Each interpreter maintains its own set of imported modules, built-in state, + and thread state list. Although most Python applications use a single + interpreter, CPython supports multiple interpreters in the same process. + +- ``PyThreadState`` represents a thread running within an interpreter. It + contains execution state and the control fields used by the debugger. + +To locate a thread: + +1. Use the offset ``runtime_state.interpreters_head`` to obtain the address of + the first interpreter in the ``PyRuntime`` structure. This is the entry point + to the linked list of active interpreters. + +2. Use the offset ``interpreter_state.threads_main`` to access the main thread + state associated with the selected interpreter. This is typically the most + reliable thread to target. + +3. Optionally, use the offset ``interpreter_state.threads_head`` to iterate + through the linked list of all thread states. Each ``PyThreadState`` + structure contains a ``native_thread_id`` field, which may be compared to + a target thread ID to find a specific thread. + +4. Once a valid ``PyThreadState`` has been found, its address can be used in + later steps of the protocol, such as writing debugger control fields and + scheduling execution. + +The following is an example implementation that locates the main thread state:: + + def find_main_thread_state( + pid: int, py_runtime_addr: int, debug_offsets: DebugOffsets, + ) -> int: + # Step 1: Read interpreters_head from PyRuntime + interp_head_ptr = ( + py_runtime_addr + debug_offsets.runtime_state.interpreters_head + ) + interp_addr = read_pointer(pid, interp_head_ptr) + if interp_addr == 0: + raise RuntimeError("No interpreter found in the target process") + + # Step 2: Read the threads_main pointer from the interpreter + threads_main_ptr = ( + interp_addr + debug_offsets.interpreter_state.threads_main + ) + thread_state_addr = read_pointer(pid, threads_main_ptr) + if thread_state_addr == 0: + raise RuntimeError("Main thread state is not available") + + return thread_state_addr + +The following example demonstrates how to locate a thread by its native thread +ID:: + + def find_thread_by_id( + pid: int, + interp_addr: int, + debug_offsets: DebugOffsets, + target_tid: int, + ) -> int: + # Start at threads_head and walk the linked list + thread_ptr = read_pointer( + pid, + interp_addr + debug_offsets.interpreter_state.threads_head + ) + + while thread_ptr: + native_tid_ptr = ( + thread_ptr + debug_offsets.thread_state.native_thread_id + ) + native_tid = read_int(pid, native_tid_ptr) + if native_tid == target_tid: + return thread_ptr + thread_ptr = read_pointer( + pid, + thread_ptr + debug_offsets.thread_state.next + ) + + raise RuntimeError("Thread with the given ID was not found") + + +Once a valid thread state has been located, the debugger can proceed with +modifying its control fields and scheduling execution, as described in the next +section. + +Writing control information +=========================== + +Once a valid ``PyThreadState`` structure has been identified, the debugger may +modify control fields within it to schedule the execution of a specified Python +script. These control fields are checked periodically by the interpreter, and +when set correctly, they trigger the execution of remote code at a safe point +in the evaluation loop. + +Each ``PyThreadState`` contains a ``_PyRemoteDebuggerSupport`` structure used +for communication between the debugger and the interpreter. The locations of +its fields are defined by the ``_Py_DebugOffsets`` structure and include the +following: + +- ``debugger_script_path``: A fixed-size buffer that holds the full path to a + Python source file (``.py``). This file must be accessible and readable by + the target process when execution is triggered. + +- ``debugger_pending_call``: An integer flag. Setting this to ``1`` tells the + interpreter that a script is ready to be executed. + +- ``eval_breaker``: A field checked by the interpreter during execution. + Setting bit 5 (``_PY_EVAL_PLEASE_STOP_BIT``, value ``1U << 5``) in this + field causes the interpreter to pause and check for debugger activity. + +To complete the injection, the debugger must perform the following steps: + +1. Write the full script path into the ``debugger_script_path`` buffer. +2. Set ``debugger_pending_call`` to ``1``. +3. Read the current value of ``eval_breaker``, set bit 5 + (``_PY_EVAL_PLEASE_STOP_BIT``), and write the updated value back. This + signals the interpreter to check for debugger activity. + +The following is an example implementation:: + + def inject_script( + pid: int, + thread_state_addr: int, + debug_offsets: DebugOffsets, + script_path: str + ) -> None: + # Compute the base offset of _PyRemoteDebuggerSupport + support_base = ( + thread_state_addr + + debug_offsets.debugger_support.remote_debugger_support + ) + + # Step 1: Write the script path into debugger_script_path + script_path_ptr = ( + support_base + + debug_offsets.debugger_support.debugger_script_path + ) + write_string(pid, script_path_ptr, script_path) + + # Step 2: Set debugger_pending_call to 1 + pending_ptr = ( + support_base + + debug_offsets.debugger_support.debugger_pending_call + ) + write_int(pid, pending_ptr, 1) + + # Step 3: Set _PY_EVAL_PLEASE_STOP_BIT (bit 5, value 1 << 5) in + # eval_breaker + eval_breaker_ptr = ( + thread_state_addr + + debug_offsets.debugger_support.eval_breaker + ) + breaker = read_int(pid, eval_breaker_ptr) + breaker |= (1 << 5) + write_int(pid, eval_breaker_ptr, breaker) + + +Once these fields are set, the debugger may resume the process (if it was +suspended). The interpreter will process the request at the next safe +evaluation point, load the script from disk, and execute it. + +It is the responsibility of the debugger to ensure that the script file remains +present and accessible to the target process during execution. + +.. note:: + + Script execution is asynchronous. The script file cannot be deleted + immediately after injection. The debugger should wait until the injected + script has produced an observable effect before removing the file. + This effect depends on what the script is designed to do. For example, + a debugger might wait until the remote process connects back to a socket + before removing the script. Once such an effect is observed, it is safe to + assume the file is no longer needed. + +Summary +======= + +To inject and execute a Python script in a remote process: + +1. Locate the ``PyRuntime`` structure in the target process’s memory. +2. Read and validate the ``_Py_DebugOffsets`` structure at the beginning of + ``PyRuntime``. +3. Use the offsets to locate a valid ``PyThreadState``. +4. Write the path to a Python script into ``debugger_script_path``. +5. Set the ``debugger_pending_call`` flag to ``1``. +6. Set ``_PY_EVAL_PLEASE_STOP_BIT`` in the ``eval_breaker`` field. +7. Resume the process (if suspended). The script will execute at the next safe + evaluation point. diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 254fe729355353..243cc27bac7025 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -352,6 +352,8 @@ If you don't include such a comment, the default encoding used will be UTF-8 as already mentioned. See also :pep:`263` for more information. +.. _unicode-properties: + Unicode Properties ------------------ diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 33a2a7ea89ea07..4e77d2cb407f72 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -15,7 +15,7 @@ Introduction You may also find useful the following article on fetching web resources with Python: - * `Basic Authentication `_ + * `Basic Authentication `__ A tutorial on *Basic Authentication*, with examples in Python. @@ -245,75 +245,27 @@ codes in the 100--299 range indicate success, you will usually only see error codes in the 400--599 range. :attr:`http.server.BaseHTTPRequestHandler.responses` is a useful dictionary of -response codes in that shows all the response codes used by :rfc:`2616`. The -dictionary is reproduced here for convenience :: +response codes that shows all the response codes used by :rfc:`2616`. +An excerpt from the dictionary is shown below :: - # Table mapping response codes to messages; entries have the - # form {code: (shortmessage, longmessage)}. responses = { - 100: ('Continue', 'Request received, please continue'), - 101: ('Switching Protocols', - 'Switching to new protocol; obey Upgrade header'), - - 200: ('OK', 'Request fulfilled, document follows'), - 201: ('Created', 'Document created, URL follows'), - 202: ('Accepted', - 'Request accepted, processing continues off-line'), - 203: ('Non-Authoritative Information', 'Request fulfilled from cache'), - 204: ('No Content', 'Request fulfilled, nothing follows'), - 205: ('Reset Content', 'Clear input form for further input.'), - 206: ('Partial Content', 'Partial content follows.'), - - 300: ('Multiple Choices', - 'Object has several resources -- see URI list'), - 301: ('Moved Permanently', 'Object moved permanently -- see URI list'), - 302: ('Found', 'Object moved temporarily -- see URI list'), - 303: ('See Other', 'Object moved -- see Method and URL list'), - 304: ('Not Modified', - 'Document has not changed since given time'), - 305: ('Use Proxy', - 'You must use proxy specified in Location to access this ' - 'resource.'), - 307: ('Temporary Redirect', - 'Object moved temporarily -- see URI list'), - - 400: ('Bad Request', - 'Bad request syntax or unsupported method'), - 401: ('Unauthorized', - 'No permission -- see authorization schemes'), - 402: ('Payment Required', - 'No payment -- see charging schemes'), - 403: ('Forbidden', - 'Request forbidden -- authorization will not help'), - 404: ('Not Found', 'Nothing matches the given URI'), - 405: ('Method Not Allowed', - 'Specified method is invalid for this server.'), - 406: ('Not Acceptable', 'URI not available in preferred format.'), - 407: ('Proxy Authentication Required', 'You must authenticate with ' - 'this proxy before proceeding.'), - 408: ('Request Timeout', 'Request timed out; try again later.'), - 409: ('Conflict', 'Request conflict.'), - 410: ('Gone', - 'URI no longer exists and has been permanently removed.'), - 411: ('Length Required', 'Client must specify Content-Length.'), - 412: ('Precondition Failed', 'Precondition in headers is false.'), - 413: ('Request Entity Too Large', 'Entity is too large.'), - 414: ('Request-URI Too Long', 'URI is too long.'), - 415: ('Unsupported Media Type', 'Entity body in unsupported format.'), - 416: ('Requested Range Not Satisfiable', - 'Cannot satisfy request range.'), - 417: ('Expectation Failed', - 'Expect condition could not be satisfied.'), - - 500: ('Internal Server Error', 'Server got itself in trouble'), - 501: ('Not Implemented', - 'Server does not support this operation'), - 502: ('Bad Gateway', 'Invalid responses from another server/proxy.'), - 503: ('Service Unavailable', - 'The server cannot process the request due to a high load'), - 504: ('Gateway Timeout', - 'The gateway server did not receive a timely response'), - 505: ('HTTP Version Not Supported', 'Cannot fulfill request.'), + ... + : ('OK', 'Request fulfilled, document follows'), + ... + : ('Forbidden', + 'Request forbidden -- authorization will ' + 'not help'), + : ('Not Found', + 'Nothing matches the given URI'), + ... + : ("I'm a Teapot", + 'Server refuses to brew coffee because ' + 'it is a teapot'), + ... + : ('Service Unavailable', + 'The server cannot process the ' + 'request due to a high load'), + ... } When an error is raised the server responds by returning an HTTP error code diff --git a/Doc/includes/capi-extension/spammodule-01.c b/Doc/includes/capi-extension/spammodule-01.c new file mode 100644 index 00000000000000..86c9840359d9c7 --- /dev/null +++ b/Doc/includes/capi-extension/spammodule-01.c @@ -0,0 +1,55 @@ +/* This file needs to be kept in sync with the tutorial + * at Doc/extending/first-extension-module.rst + */ + +/// Includes + +#include +#include // for system() + +/// Implementation of spam.system + +static PyObject * +spam_system(PyObject *self, PyObject *arg) +{ + const char *command = PyUnicode_AsUTF8(arg); + if (command == NULL) { + return NULL; + } + int status = system(command); + PyObject *result = PyLong_FromLong(status); + return result; +} + +/// Module method table + +static PyMethodDef spam_methods[] = { + { + .ml_name="system", + .ml_meth=spam_system, + .ml_flags=METH_O, + .ml_doc="Execute a shell command.", + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +/// Module slot table + +static PyModuleDef_Slot spam_slots[] = { + {Py_mod_name, "spam"}, + {Py_mod_doc, "A wonderful module with an example function"}, + {Py_mod_methods, spam_methods}, + {0, NULL} +}; + +/// Export hook prototype + +PyMODEXPORT_FUNC PyModExport_spam(void); + +/// Module export hook + +PyMODEXPORT_FUNC +PyModExport_spam(void) +{ + return spam_slots; +} diff --git a/Doc/includes/email-alternative.py b/Doc/includes/email-alternative.py index 26b302b495c7ac..afe2b4fbb5eb3f 100644 --- a/Doc/includes/email-alternative.py +++ b/Doc/includes/email-alternative.py @@ -36,7 +36,7 @@ recette sera sûrement un très bon repas.

- + """.format(asparagus_cid=asparagus_cid[1:-1]), subtype='html') diff --git a/Doc/includes/newtypes/custom.c b/Doc/includes/newtypes/custom.c index 5253f879360210..039a1a7219349c 100644 --- a/Doc/includes/newtypes/custom.c +++ b/Doc/includes/newtypes/custom.c @@ -16,28 +16,37 @@ static PyTypeObject CustomType = { .tp_new = PyType_GenericNew, }; -static PyModuleDef custommodule = { +static int +custom_module_exec(PyObject *m) +{ + if (PyType_Ready(&CustomType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot custom_module_slots[] = { + {Py_mod_exec, custom_module_exec}, + // Just use this while using static types + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef custom_module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = custom_module_slots, }; PyMODINIT_FUNC PyInit_custom(void) { - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&custom_module); } diff --git a/Doc/includes/newtypes/custom2.c b/Doc/includes/newtypes/custom2.c index 768ce29fab9ff0..1ff8e707d1b0a0 100644 --- a/Doc/includes/newtypes/custom2.c +++ b/Doc/includes/newtypes/custom2.c @@ -10,11 +10,12 @@ typedef struct { } CustomObject; static void -Custom_dealloc(CustomObject *self) +Custom_dealloc(PyObject *op) { + CustomObject *self = (CustomObject *) op; Py_XDECREF(self->first); Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject *) self); + Py_TYPE(self)->tp_free(self); } static PyObject * @@ -39,8 +40,9 @@ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) +Custom_init(PyObject *op, PyObject *args, PyObject *kwds) { + CustomObject *self = (CustomObject *) op; static char *kwlist[] = {"first", "last", "number", NULL}; PyObject *first = NULL, *last = NULL; @@ -69,8 +71,9 @@ static PyMemberDef Custom_members[] = { }; static PyObject * -Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) +Custom_name(PyObject *op, PyObject *Py_UNUSED(dummy)) { + CustomObject *self = (CustomObject *) op; if (self->first == NULL) { PyErr_SetString(PyExc_AttributeError, "first"); return NULL; @@ -83,7 +86,7 @@ Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) } static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, + {"name", Custom_name, METH_NOARGS, "Return the name, combining the first and last name" }, {NULL} /* Sentinel */ @@ -97,34 +100,42 @@ static PyTypeObject CustomType = { .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_new = Custom_new, - .tp_init = (initproc) Custom_init, - .tp_dealloc = (destructor) Custom_dealloc, + .tp_init = Custom_init, + .tp_dealloc = Custom_dealloc, .tp_members = Custom_members, .tp_methods = Custom_methods, }; -static PyModuleDef custommodule = { - .m_base =PyModuleDef_HEAD_INIT, +static int +custom_module_exec(PyObject *m) +{ + if (PyType_Ready(&CustomType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot custom_module_slots[] = { + {Py_mod_exec, custom_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef custom_module = { + .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom2", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = custom_module_slots, }; PyMODINIT_FUNC PyInit_custom2(void) { - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&custom_module); } diff --git a/Doc/includes/newtypes/custom3.c b/Doc/includes/newtypes/custom3.c index 7d969adfa7c9cc..22f50eb0e1de89 100644 --- a/Doc/includes/newtypes/custom3.c +++ b/Doc/includes/newtypes/custom3.c @@ -10,11 +10,12 @@ typedef struct { } CustomObject; static void -Custom_dealloc(CustomObject *self) +Custom_dealloc(PyObject *op) { + CustomObject *self = (CustomObject *) op; Py_XDECREF(self->first); Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject *) self); + Py_TYPE(self)->tp_free(self); } static PyObject * @@ -39,8 +40,9 @@ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) +Custom_init(PyObject *op, PyObject *args, PyObject *kwds) { + CustomObject *self = (CustomObject *) op; static char *kwlist[] = {"first", "last", "number", NULL}; PyObject *first = NULL, *last = NULL; @@ -65,14 +67,16 @@ static PyMemberDef Custom_members[] = { }; static PyObject * -Custom_getfirst(CustomObject *self, void *closure) +Custom_getfirst(PyObject *op, void *closure) { + CustomObject *self = (CustomObject *) op; return Py_NewRef(self->first); } static int -Custom_setfirst(CustomObject *self, PyObject *value, void *closure) +Custom_setfirst(PyObject *op, PyObject *value, void *closure) { + CustomObject *self = (CustomObject *) op; if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); return -1; @@ -87,14 +91,16 @@ Custom_setfirst(CustomObject *self, PyObject *value, void *closure) } static PyObject * -Custom_getlast(CustomObject *self, void *closure) +Custom_getlast(PyObject *op, void *closure) { + CustomObject *self = (CustomObject *) op; return Py_NewRef(self->last); } static int -Custom_setlast(CustomObject *self, PyObject *value, void *closure) +Custom_setlast(PyObject *op, PyObject *value, void *closure) { + CustomObject *self = (CustomObject *) op; if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute"); return -1; @@ -109,21 +115,22 @@ Custom_setlast(CustomObject *self, PyObject *value, void *closure) } static PyGetSetDef Custom_getsetters[] = { - {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, + {"first", Custom_getfirst, Custom_setfirst, "first name", NULL}, - {"last", (getter) Custom_getlast, (setter) Custom_setlast, + {"last", Custom_getlast, Custom_setlast, "last name", NULL}, {NULL} /* Sentinel */ }; static PyObject * -Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) +Custom_name(PyObject *op, PyObject *Py_UNUSED(dummy)) { + CustomObject *self = (CustomObject *) op; return PyUnicode_FromFormat("%S %S", self->first, self->last); } static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, + {"name", Custom_name, METH_NOARGS, "Return the name, combining the first and last name" }, {NULL} /* Sentinel */ @@ -137,35 +144,43 @@ static PyTypeObject CustomType = { .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, .tp_new = Custom_new, - .tp_init = (initproc) Custom_init, - .tp_dealloc = (destructor) Custom_dealloc, + .tp_init = Custom_init, + .tp_dealloc = Custom_dealloc, .tp_members = Custom_members, .tp_methods = Custom_methods, .tp_getset = Custom_getsetters, }; -static PyModuleDef custommodule = { +static int +custom_module_exec(PyObject *m) +{ + if (PyType_Ready(&CustomType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot custom_module_slots[] = { + {Py_mod_exec, custom_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef custom_module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom3", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = custom_module_slots, }; PyMODINIT_FUNC PyInit_custom3(void) { - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&custom_module); } diff --git a/Doc/includes/newtypes/custom4.c b/Doc/includes/newtypes/custom4.c index a7b8de44a57c90..07585aff5987f3 100644 --- a/Doc/includes/newtypes/custom4.c +++ b/Doc/includes/newtypes/custom4.c @@ -10,27 +10,29 @@ typedef struct { } CustomObject; static int -Custom_traverse(CustomObject *self, visitproc visit, void *arg) +Custom_traverse(PyObject *op, visitproc visit, void *arg) { + CustomObject *self = (CustomObject *) op; Py_VISIT(self->first); Py_VISIT(self->last); return 0; } static int -Custom_clear(CustomObject *self) +Custom_clear(PyObject *op) { + CustomObject *self = (CustomObject *) op; Py_CLEAR(self->first); Py_CLEAR(self->last); return 0; } static void -Custom_dealloc(CustomObject *self) +Custom_dealloc(PyObject *op) { - PyObject_GC_UnTrack(self); - Custom_clear(self); - Py_TYPE(self)->tp_free((PyObject *) self); + PyObject_GC_UnTrack(op); + (void)Custom_clear(op); + Py_TYPE(op)->tp_free(op); } static PyObject * @@ -55,8 +57,9 @@ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } static int -Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) +Custom_init(PyObject *op, PyObject *args, PyObject *kwds) { + CustomObject *self = (CustomObject *) op; static char *kwlist[] = {"first", "last", "number", NULL}; PyObject *first = NULL, *last = NULL; @@ -81,14 +84,16 @@ static PyMemberDef Custom_members[] = { }; static PyObject * -Custom_getfirst(CustomObject *self, void *closure) +Custom_getfirst(PyObject *op, void *closure) { + CustomObject *self = (CustomObject *) op; return Py_NewRef(self->first); } static int -Custom_setfirst(CustomObject *self, PyObject *value, void *closure) +Custom_setfirst(PyObject *op, PyObject *value, void *closure) { + CustomObject *self = (CustomObject *) op; if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); return -1; @@ -103,14 +108,16 @@ Custom_setfirst(CustomObject *self, PyObject *value, void *closure) } static PyObject * -Custom_getlast(CustomObject *self, void *closure) +Custom_getlast(PyObject *op, void *closure) { + CustomObject *self = (CustomObject *) op; return Py_NewRef(self->last); } static int -Custom_setlast(CustomObject *self, PyObject *value, void *closure) +Custom_setlast(PyObject *op, PyObject *value, void *closure) { + CustomObject *self = (CustomObject *) op; if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute"); return -1; @@ -125,21 +132,22 @@ Custom_setlast(CustomObject *self, PyObject *value, void *closure) } static PyGetSetDef Custom_getsetters[] = { - {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, + {"first", Custom_getfirst, Custom_setfirst, "first name", NULL}, - {"last", (getter) Custom_getlast, (setter) Custom_setlast, + {"last", Custom_getlast, Custom_setlast, "last name", NULL}, {NULL} /* Sentinel */ }; static PyObject * -Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) +Custom_name(PyObject *op, PyObject *Py_UNUSED(dummy)) { + CustomObject *self = (CustomObject *) op; return PyUnicode_FromFormat("%S %S", self->first, self->last); } static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, + {"name", Custom_name, METH_NOARGS, "Return the name, combining the first and last name" }, {NULL} /* Sentinel */ @@ -153,37 +161,45 @@ static PyTypeObject CustomType = { .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, .tp_new = Custom_new, - .tp_init = (initproc) Custom_init, - .tp_dealloc = (destructor) Custom_dealloc, - .tp_traverse = (traverseproc) Custom_traverse, - .tp_clear = (inquiry) Custom_clear, + .tp_init = Custom_init, + .tp_dealloc = Custom_dealloc, + .tp_traverse = Custom_traverse, + .tp_clear = Custom_clear, .tp_members = Custom_members, .tp_methods = Custom_methods, .tp_getset = Custom_getsetters, }; -static PyModuleDef custommodule = { +static int +custom_module_exec(PyObject *m) +{ + if (PyType_Ready(&CustomType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot custom_module_slots[] = { + {Py_mod_exec, custom_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef custom_module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom4", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = custom_module_slots, }; PyMODINIT_FUNC PyInit_custom4(void) { - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&custom_module); } diff --git a/Doc/includes/newtypes/sublist.c b/Doc/includes/newtypes/sublist.c index d8aba463f30ba2..b784456a4ef667 100644 --- a/Doc/includes/newtypes/sublist.c +++ b/Doc/includes/newtypes/sublist.c @@ -7,61 +7,71 @@ typedef struct { } SubListObject; static PyObject * -SubList_increment(SubListObject *self, PyObject *unused) +SubList_increment(PyObject *op, PyObject *Py_UNUSED(dummy)) { + SubListObject *self = (SubListObject *) op; self->state++; return PyLong_FromLong(self->state); } static PyMethodDef SubList_methods[] = { - {"increment", (PyCFunction) SubList_increment, METH_NOARGS, + {"increment", SubList_increment, METH_NOARGS, PyDoc_STR("increment state counter")}, {NULL}, }; static int -SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) +SubList_init(PyObject *op, PyObject *args, PyObject *kwds) { - if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) + SubListObject *self = (SubListObject *) op; + if (PyList_Type.tp_init(op, args, kwds) < 0) return -1; self->state = 0; return 0; } static PyTypeObject SubListType = { - PyVarObject_HEAD_INIT(NULL, 0) + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "sublist.SubList", .tp_doc = PyDoc_STR("SubList objects"), .tp_basicsize = sizeof(SubListObject), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - .tp_init = (initproc) SubList_init, + .tp_init = SubList_init, .tp_methods = SubList_methods, }; -static PyModuleDef sublistmodule = { - PyModuleDef_HEAD_INIT, +static int +sublist_module_exec(PyObject *m) +{ + SubListType.tp_base = &PyList_Type; + if (PyType_Ready(&SubListType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot sublist_module_slots[] = { + {Py_mod_exec, sublist_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef sublist_module = { + .m_base = PyModuleDef_HEAD_INIT, .m_name = "sublist", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = sublist_module_slots, }; PyMODINIT_FUNC PyInit_sublist(void) { - PyObject *m; - SubListType.tp_base = &PyList_Type; - if (PyType_Ready(&SubListType) < 0) - return NULL; - - m = PyModule_Create(&sublistmodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&sublist_module); } diff --git a/Doc/includes/optional-module.rst b/Doc/includes/optional-module.rst new file mode 100644 index 00000000000000..262e73f2eaa09f --- /dev/null +++ b/Doc/includes/optional-module.rst @@ -0,0 +1,9 @@ +This is an :term:`optional module`. +If it is missing from your copy of CPython, +look for documentation from your distributor (that is, +whoever provided Python to you). +If you are the distributor, see :ref:`optional-module-requirements`. + +.. Similar notes appear in the docs of the modules: + - zipfile + - tarfile diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index ec939c28831c33..0d1d85ce9741b2 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -54,11 +54,11 @@ typedef struct _typeobject { iternextfunc tp_iternext; /* Attribute descriptor and subclassing stuff */ - struct PyMethodDef *tp_methods; - struct PyMemberDef *tp_members; - struct PyGetSetDef *tp_getset; + PyMethodDef *tp_methods; + PyMemberDef *tp_members; + PyGetSetDef *tp_getset; // Strong reference on a heap type, borrowed reference on a static type - struct _typeobject *tp_base; + PyTypeObject *tp_base; PyObject *tp_dict; descrgetfunc tp_descr_get; descrsetfunc tp_descr_set; @@ -70,12 +70,14 @@ typedef struct _typeobject { inquiry tp_is_gc; /* For PyObject_IS_GC */ PyObject *tp_bases; PyObject *tp_mro; /* method resolution order */ - PyObject *tp_cache; - PyObject *tp_subclasses; - PyObject *tp_weaklist; + PyObject *tp_cache; /* no longer used */ + void *tp_subclasses; /* for static builtin types this is an index */ + PyObject *tp_weaklist; /* not used for static builtin types */ destructor tp_del; - /* Type attribute cache version tag. Added in version 2.6 */ + /* Type attribute cache version tag. Added in version 2.6. + * If zero, the cache is invalid and must be initialized. + */ unsigned int tp_version_tag; destructor tp_finalize; @@ -83,4 +85,11 @@ typedef struct _typeobject { /* bitset of which type-watchers care about this type */ unsigned char tp_watched; + + /* Number of tp_version_tag values used. + * Set to _Py_ATTR_CACHE_UNUSED if the attribute cache is + * disabled for this type (e.g. due to custom MRO entries). + * Otherwise, limited to MAX_VERSIONS_PER_CLASS (defined elsewhere). + */ + uint16_t tp_versions_used; } PyTypeObject; diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index a46c1caefe4d8a..3a485a43a5a751 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -188,7 +188,7 @@ switch:: Once the Development & Deployment part of PPUG is fleshed out, some of those sections should be linked from new questions here (most notably, we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) + https://packaging.python.org/en/latest/guides/index-mirrors-and-caches/) Common installation issues diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 4f3b663006fb28..5d916b30112d3c 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -37,38 +37,52 @@ No feature description will ever be deleted from :mod:`__future__`. Since its introduction in Python 2.1 the following features have found their way into the language using this mechanism: -+------------------+-------------+--------------+---------------------------------------------+ -| feature | optional in | mandatory in | effect | -+==================+=============+==============+=============================================+ -| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | -| | | | *Statically Nested Scopes* | -+------------------+-------------+--------------+---------------------------------------------+ -| generators | 2.2.0a1 | 2.3 | :pep:`255`: | -| | | | *Simple Generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| division | 2.2.0a2 | 3.0 | :pep:`238`: | -| | | | *Changing the Division Operator* | -+------------------+-------------+--------------+---------------------------------------------+ -| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | -| | | | *Imports: Multi-Line and Absolute/Relative* | -+------------------+-------------+--------------+---------------------------------------------+ -| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | -| | | | *The "with" Statement* | -+------------------+-------------+--------------+---------------------------------------------+ -| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | -| | | | *Make print a function* | -+------------------+-------------+--------------+---------------------------------------------+ -| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | -| | | | *Bytes literals in Python 3000* | -+------------------+-------------+--------------+---------------------------------------------+ -| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | -| | | | *StopIteration handling inside generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | Never [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations*, | -| | | | :pep:`649`: *Deferred evaluation of | -| | | | annotations using descriptors* | -+------------------+-------------+--------------+---------------------------------------------+ + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * feature + * optional in + * mandatory in + * effect + * * .. data:: nested_scopes + * 2.1.0b1 + * 2.2 + * :pep:`227`: *Statically Nested Scopes* + * * .. data:: generators + * 2.2.0a1 + * 2.3 + * :pep:`255`: *Simple Generators* + * * .. data:: division + * 2.2.0a2 + * 3.0 + * :pep:`238`: *Changing the Division Operator* + * * .. data:: absolute_import + * 2.5.0a1 + * 3.0 + * :pep:`328`: *Imports: Multi-Line and Absolute/Relative* + * * .. data:: with_statement + * 2.5.0a1 + * 2.6 + * :pep:`343`: *The “with” Statement* + * * .. data:: print_function + * 2.6.0a2 + * 3.0 + * :pep:`3105`: *Make print a function* + * * .. data:: unicode_literals + * 2.6.0a2 + * 3.0 + * :pep:`3112`: *Bytes literals in Python 3000* + * * .. data:: generator_stop + * 3.5.0b1 + * 3.7 + * :pep:`479`: *StopIteration handling inside generators* + * * .. data:: annotations + * 3.7.0b1 + * Never [1]_ + * :pep:`563`: *Postponed evaluation of annotations*, + :pep:`649`: *Deferred evaluation of annotations using descriptors* .. XXX Adding a new entry? Remember to update simple_stmts.rst, too. diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst index 647ff9da04d10d..4407ba2f7714dd 100644 --- a/Doc/library/__main__.rst +++ b/Doc/library/__main__.rst @@ -292,10 +292,7 @@ Here is an example module that consumes the ``__main__`` namespace:: if not did_user_define_their_name(): raise ValueError('Define the variable `my_name`!') - if '__file__' in dir(__main__): - print(__main__.my_name, "found in file", __main__.__file__) - else: - print(__main__.my_name) + print(__main__.my_name) Example usage of this module could be as follows:: @@ -330,7 +327,7 @@ status code 0, indicating success: .. code-block:: shell-session $ python start.py - Dinsdale found in file /path/to/start.py + Dinsdale Note that importing ``__main__`` doesn't cause any issues with unintentionally running top-level code meant for script use which is put in the @@ -361,8 +358,5 @@ defined in the REPL becomes part of the ``__main__`` scope:: >>> namely.print_user_name() Jabberwocky -Note that in this case the ``__main__`` scope doesn't contain a ``__file__`` -attribute as it's interactive. - The ``__main__`` scope is used in the implementation of :mod:`pdb` and :mod:`rlcompleter`. diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index ed29ac70035597..47f5eabb6f2180 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -120,13 +120,16 @@ This module defines the following constants and functions: Its value may be used to uniquely identify this particular thread system-wide (until the thread terminates, after which the value may be recycled by the OS). - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD. + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD, GNU/kFreeBSD, Solaris. .. versionadded:: 3.8 .. versionchanged:: 3.13 Added support for GNU/kFreeBSD. + .. versionchanged:: 3.15 + Added support for Solaris. + .. function:: stack_size([size]) diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 38d744e97d087d..49e541a9d9b1cb 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -141,18 +141,18 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: MyIterable.register(Foo) The ABC ``MyIterable`` defines the standard iterable method, - :meth:`~iterator.__iter__`, as an abstract method. The implementation given + :meth:`~object.__iter__`, as an abstract method. The implementation given here can still be called from subclasses. The :meth:`!get_iterator` method is also part of the ``MyIterable`` abstract base class, but it does not have to be overridden in non-abstract derived classes. The :meth:`__subclasshook__` class method defined here says that any class - that has an :meth:`~iterator.__iter__` method in its + that has an :meth:`~object.__iter__` method in its :attr:`~object.__dict__` (or in that of one of its base classes, accessed via the :attr:`~type.__mro__` list) is considered a ``MyIterable`` too. Finally, the last line makes ``Foo`` a virtual subclass of ``MyIterable``, - even though it does not define an :meth:`~iterator.__iter__` method (it uses + even though it does not define an :meth:`~object.__iter__` method (it uses the old-style iterable protocol, defined in terms of :meth:`~object.__len__` and :meth:`~object.__getitem__`). Note that this will not make ``get_iterator`` available as a method of ``Foo``, so it is provided separately. diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst new file mode 100644 index 00000000000000..a756d679036ecb --- /dev/null +++ b/Doc/library/aifc.rst @@ -0,0 +1,15 @@ +:mod:`!aifc` --- Read and write AIFF and AIFC files +=================================================== + +.. module:: aifc + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!aifc` module was +`Python 3.12 `_. diff --git a/Doc/library/allos.rst b/Doc/library/allos.rst index 0223c1054ea5d8..1aed340b2527ac 100644 --- a/Doc/library/allos.rst +++ b/Doc/library/allos.rst @@ -15,14 +15,9 @@ but they are available on most other systems as well. Here's an overview: os.rst io.rst time.rst - argparse.rst logging.rst logging.config.rst logging.handlers.rst - getpass.rst - curses.rst - curses.ascii.rst - curses.panel.rst platform.rst errno.rst ctypes.rst diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 37490456d13312..40f2a6dc30460b 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -4,6 +4,7 @@ .. module:: annotationlib :synopsis: Functionality for introspecting annotations +.. versionadded:: 3.14 **Source code:** :source:`Lib/annotationlib.py` @@ -40,11 +41,15 @@ The :func:`get_annotations` function is the main entry point for retrieving annotations. Given a function, class, or module, it returns an annotations dictionary in the requested format. This module also provides functionality for working directly with the :term:`annotate function` -that is used to evaluate annotations, such as :func:`get_annotate_function` +that is used to evaluate annotations, such as :func:`get_annotate_from_class_namespace` and :func:`call_annotate_function`, as well as the :func:`call_evaluate_function` function for working with :term:`evaluate functions `. +.. caution:: + + Most functionality in this module can execute arbitrary code; see + :ref:`the security section ` for more information. .. seealso:: @@ -127,16 +132,27 @@ Classes Values are the result of evaluating the annotation expressions. - .. attribute:: FORWARDREF + .. attribute:: VALUE_WITH_FAKE_GLOBALS :value: 2 + Special value used to signal that an annotate function is being + evaluated in a special environment with fake globals. When passed this + value, annotate functions should either return the same value as for + the :attr:`Format.VALUE` format, or raise :exc:`NotImplementedError` + to signal that they do not support execution in this environment. + This format is only used internally and should not be passed to + the functions in this module. + + .. attribute:: FORWARDREF + :value: 3 + Values are real annotation values (as per :attr:`Format.VALUE` format) for defined values, and :class:`ForwardRef` proxies for undefined - values. Real objects may contain references to, :class:`ForwardRef` + values. Real objects may contain references to :class:`ForwardRef` proxy objects. .. attribute:: STRING - :value: 3 + :value: 4 Values are the text string of the annotation as it appears in the source code, up to modifications including, but not restricted to, @@ -161,14 +177,37 @@ Classes :class:`~ForwardRef`. The string may not be exactly equivalent to the original source. - .. method:: evaluate(*, globals=None, locals=None, type_params=None, owner=None) + .. method:: evaluate(*, owner=None, globals=None, locals=None, type_params=None, format=Format.VALUE) Evaluate the forward reference, returning its value. - This may throw an exception, such as :exc:`NameError`, if the forward - reference refers to names that do not exist. The arguments to this + If the *format* argument is :attr:`~Format.VALUE` (the default), + this method may throw an exception, such as :exc:`NameError`, if the forward + reference refers to a name that cannot be resolved. The arguments to this method can be used to provide bindings for names that would otherwise - be undefined. + be undefined. If the *format* argument is :attr:`~Format.FORWARDREF`, + the method will never throw an exception, but may return a :class:`~ForwardRef` + instance. For example, if the forward reference object contains the code + ``list[undefined]``, where ``undefined`` is a name that is not defined, + evaluating it with the :attr:`~Format.FORWARDREF` format will return + ``list[ForwardRef('undefined')]``. If the *format* argument is + :attr:`~Format.STRING`, the method will return :attr:`~ForwardRef.__forward_arg__`. + + The *owner* parameter provides the preferred mechanism for passing scope + information to this method. The owner of a :class:`~ForwardRef` is the + object that contains the annotation from which the :class:`~ForwardRef` + derives, such as a module object, type object, or function object. + + The *globals*, *locals*, and *type_params* parameters provide a more precise + mechanism for influencing the names that are available when the :class:`~ForwardRef` + is evaluated. *globals* and *locals* are passed to :func:`eval`, representing + the global and local namespaces in which the name is evaluated. + The *type_params* parameter is relevant for objects created using the native + syntax for :ref:`generic classes ` and :ref:`functions `. + It is a tuple of :ref:`type parameters ` that are in scope + while the forward reference is being evaluated. For example, if evaluating a + :class:`~ForwardRef` retrieved from an annotation found in the class namespace + of a generic class ``C``, *type_params* should be set to ``C.__type_params__``. :class:`~ForwardRef` instances returned by :func:`get_annotations` retain references to information about the scope they originated from, @@ -177,19 +216,9 @@ Classes means may not have any information about their scope, so passing arguments to this method may be necessary to evaluate them successfully. - *globals* and *locals* are passed to :func:`eval`, representing - the global and local namespaces in which the name is evaluated. - *type_params*, if given, must be a tuple of - :ref:`type parameters ` that are in scope while the forward - reference is being evaluated. *owner* is the object that owns the - annotation from which the forward reference derives, usually a function, - class, or module. - - .. important:: - - Once a :class:`~ForwardRef` instance has been evaluated, it caches - the evaluated value, and future calls to :meth:`evaluate` will return - the cached value, regardless of the parameters passed in. + If no *owner*, *globals*, *locals*, or *type_params* are provided and the + :class:`~ForwardRef` does not contain information about its origin, + empty globals and locals dictionaries are used. .. versionadded:: 3.14 @@ -201,7 +230,7 @@ Functions Convert an annotations dict containing runtime values to a dict containing only strings. If the values are not already strings, - they are converted using :func:`value_to_string`. + they are converted using :func:`type_repr`. This is meant as a helper for user-provided annotate functions that support the :attr:`~Format.STRING` format but do not have access to the code creating the annotations. @@ -287,15 +316,13 @@ Functions .. versionadded:: 3.14 -.. function:: get_annotate_function(obj) +.. function:: get_annotate_from_class_namespace(namespace) - Retrieve the :term:`annotate function` for *obj*. Return :const:`!None` - if *obj* does not have an annotate function. - - This is usually equivalent to accessing the :attr:`~object.__annotate__` - attribute of *obj*, but direct access to the attribute may return the wrong - object in certain situations involving metaclasses. This function should be - used instead of accessing the attribute directly. + Retrieve the :term:`annotate function` from a class namespace dictionary *namespace*. + Return :const:`!None` if the namespace does not contain an annotate function. + This is primarily useful before the class has been fully created (e.g., in a metaclass); + after the class exists, the annotate function can be retrieved with ``cls.__annotate__``. + See :ref:`below ` for an example using this function in a metaclass. .. versionadded:: 3.14 @@ -304,11 +331,37 @@ Functions Compute the annotations dict for an object. *obj* may be a callable, class, module, or other object with - :attr:`~object.__annotate__` and :attr:`~object.__annotations__` attributes. - Passing in an object of any other type raises :exc:`TypeError`. + :attr:`~object.__annotate__` or :attr:`~object.__annotations__` attributes. + Passing any other object raises :exc:`TypeError`. The *format* parameter controls the format in which annotations are returned, and must be a member of the :class:`Format` enum or its integer equivalent. + The different formats work as follows: + + * VALUE: :attr:`!object.__annotations__` is tried first; if that does not exist, + the :attr:`!object.__annotate__` function is called if it exists. + + * FORWARDREF: If :attr:`!object.__annotations__` exists and can be evaluated successfully, + it is used; otherwise, the :attr:`!object.__annotate__` function is called. If it + does not exist either, :attr:`!object.__annotations__` is tried again and any error + from accessing it is re-raised. + + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.FORWARDREF`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + + * STRING: If :attr:`!object.__annotate__` exists, it is called first; + otherwise, :attr:`!object.__annotations__` is used and stringified + using :func:`annotations_to_string`. + + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.STRING`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE` + with the result converted using :func:`annotations_to_string`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. Returns a dict. :func:`!get_annotations` returns a new dict every time it's called; calling it twice on the same object will return two @@ -369,7 +422,7 @@ Functions .. versionadded:: 3.14 -.. function:: value_to_string(value) +.. function:: type_repr(value) Convert an arbitrary Python value to a format suitable for use by the :attr:`~Format.STRING` format. This calls :func:`repr` for most @@ -383,3 +436,210 @@ Functions .. versionadded:: 3.14 + +Recipes +------- + +.. _annotationlib-metaclass: + +Using annotations in a metaclass +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A :ref:`metaclass ` may want to inspect or even modify the annotations +in a class body during class creation. Doing so requires retrieving annotations +from the class namespace dictionary. For classes created with +``from __future__ import annotations``, the annotations will be in the ``__annotations__`` +key of the dictionary. For other classes with annotations, +:func:`get_annotate_from_class_namespace` can be used to get the +annotate function, and :func:`call_annotate_function` can be used to call it and +retrieve the annotations. Using the :attr:`~Format.FORWARDREF` format will usually +be best, because this allows the annotations to refer to names that cannot yet be +resolved when the class is created. + +To modify the annotations, it is best to create a wrapper annotate function +that calls the original annotate function, makes any necessary adjustments, and +returns the result. + +Below is an example of a metaclass that filters out all :class:`typing.ClassVar` +annotations from the class and puts them in a separate attribute: + +.. code-block:: python + + import annotationlib + import typing + + class ClassVarSeparator(type): + def __new__(mcls, name, bases, ns): + if "__annotations__" in ns: # from __future__ import annotations + annotations = ns["__annotations__"] + classvar_keys = { + key for key, value in annotations.items() + # Use string comparison for simplicity; a more robust solution + # could use annotationlib.ForwardRef.evaluate + if value.startswith("ClassVar") + } + classvars = {key: annotations[key] for key in classvar_keys} + ns["__annotations__"] = { + key: value for key, value in annotations.items() + if key not in classvar_keys + } + wrapped_annotate = None + elif annotate := annotationlib.get_annotate_from_class_namespace(ns): + annotations = annotationlib.call_annotate_function( + annotate, format=annotationlib.Format.FORWARDREF + ) + classvar_keys = { + key for key, value in annotations.items() + if typing.get_origin(value) is typing.ClassVar + } + classvars = {key: annotations[key] for key in classvar_keys} + + def wrapped_annotate(format): + annos = annotationlib.call_annotate_function(annotate, format, owner=typ) + return {key: value for key, value in annos.items() if key not in classvar_keys} + + else: # no annotations + classvars = {} + wrapped_annotate = None + typ = super().__new__(mcls, name, bases, ns) + + if wrapped_annotate is not None: + # Wrap the original __annotate__ with a wrapper that removes ClassVars + typ.__annotate__ = wrapped_annotate + typ.classvars = classvars # Store the ClassVars in a separate attribute + return typ + + +Limitations of the ``STRING`` format +------------------------------------ + +The :attr:`~Format.STRING` format is meant to approximate the source code +of the annotation, but the implementation strategy used means that it is not +always possible to recover the exact source code. + +First, the stringifier of course cannot recover any information that is not present in +the compiled code, including comments, whitespace, parenthesization, and operations that +get simplified by the compiler. + +Second, the stringifier can intercept almost all operations that involve names looked +up in some scope, but it cannot intercept operations that operate fully on constants. +As a corollary, this also means it is not safe to request the ``STRING`` format on +untrusted code: Python is powerful enough that it is possible to achieve arbitrary +code execution even with no access to any globals or builtins. For example: + +.. code-block:: pycon + + >>> def f(x: (1).__class__.__base__.__subclasses__()[-1].__init__.__builtins__["print"]("Hello world")): pass + ... + >>> annotationlib.get_annotations(f, format=annotationlib.Format.STRING) + Hello world + {'x': 'None'} + +.. note:: + This particular example works as of the time of writing, but it relies on + implementation details and is not guaranteed to work in the future. + +Among the different kinds of expressions that exist in Python, +as represented by the :mod:`ast` module, some expressions are supported, +meaning that the ``STRING`` format can generally recover the original source code; +others are unsupported, meaning that they may result in incorrect output or an error. + +The following are supported (sometimes with caveats): + +* :class:`ast.BinOp` +* :class:`ast.UnaryOp` + + * :class:`ast.Invert` (``~``), :class:`ast.UAdd` (``+``), and :class:`ast.USub` (``-``) are supported + * :class:`ast.Not` (``not``) is not supported + +* :class:`ast.Dict` (except when using ``**`` unpacking) +* :class:`ast.Set` +* :class:`ast.Compare` + + * :class:`ast.Eq` and :class:`ast.NotEq` are supported + * :class:`ast.Lt`, :class:`ast.LtE`, :class:`ast.Gt`, and :class:`ast.GtE` are supported, but the operand may be flipped + * :class:`ast.Is`, :class:`ast.IsNot`, :class:`ast.In`, and :class:`ast.NotIn` are not supported + +* :class:`ast.Call` (except when using ``**`` unpacking) +* :class:`ast.Constant` (though not the exact representation of the constant; for example, escape + sequences in strings are lost; hexadecimal numbers are converted to decimal) +* :class:`ast.Attribute` (assuming the value is not a constant) +* :class:`ast.Subscript` (assuming the value is not a constant) +* :class:`ast.Starred` (``*`` unpacking) +* :class:`ast.Name` +* :class:`ast.List` +* :class:`ast.Tuple` +* :class:`ast.Slice` + +The following are unsupported, but throw an informative error when encountered by the +stringifier: + +* :class:`ast.FormattedValue` (f-strings; error is not detected if conversion specifiers like ``!r`` + are used) +* :class:`ast.JoinedStr` (f-strings) + +The following are unsupported and result in incorrect output: + +* :class:`ast.BoolOp` (``and`` and ``or``) +* :class:`ast.IfExp` +* :class:`ast.Lambda` +* :class:`ast.ListComp` +* :class:`ast.SetComp` +* :class:`ast.DictComp` +* :class:`ast.GeneratorExp` + +The following are disallowed in annotation scopes and therefore not relevant: + +* :class:`ast.NamedExpr` (``:=``) +* :class:`ast.Await` +* :class:`ast.Yield` +* :class:`ast.YieldFrom` + + +Limitations of the ``FORWARDREF`` format +---------------------------------------- + +The :attr:`~Format.FORWARDREF` format aims to produce real values as much +as possible, with anything that cannot be resolved replaced with +:class:`ForwardRef` objects. It is affected by broadly the same Limitations +as the :attr:`~Format.STRING` format: annotations that perform operations on +literals or that use unsupported expression types may raise exceptions when +evaluated using the :attr:`~Format.FORWARDREF` format. + +Below are a few examples of the behavior with unsupported expressions: + +.. code-block:: pycon + + >>> from annotationlib import get_annotations, Format + >>> def zerodiv(x: 1 / 0): ... + >>> get_annotations(zerodiv, format=Format.STRING) + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> get_annotations(zerodiv, format=Format.FORWARDREF) + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> def ifexp(x: 1 if y else 0): ... + >>> get_annotations(ifexp, format=Format.STRING) + {'x': '1'} + +.. _annotationlib-security: + +Security implications of introspecting annotations +-------------------------------------------------- + +Much of the functionality in this module involves executing code related to annotations, +which can then do arbitrary things. For example, +:func:`get_annotations` may call an arbitrary :term:`annotate function`, and +:meth:`ForwardRef.evaluate` may call :func:`eval` on an arbitrary string. Code contained +in an annotation might make arbitrary system calls, enter an infinite loop, or perform any +other operation. This is also true for any access of the :attr:`~object.__annotations__` attribute, +and for various functions in the :mod:`typing` module that work with annotations, such as +:func:`typing.get_type_hints`. + +Any security issue arising from this also applies immediately after importing +code that may contain untrusted annotations: importing code can always cause arbitrary operations +to be performed. However, it is unsafe to accept strings or other input from an untrusted source and +pass them to any of the APIs for introspecting annotations, for example by editing an +``__annotations__`` dictionary or directly creating a :class:`ForwardRef` object. diff --git a/Doc/library/archiving.rst b/Doc/library/archiving.rst index c9284949af4972..da0b3f8c3e7693 100644 --- a/Doc/library/archiving.rst +++ b/Doc/library/archiving.rst @@ -5,13 +5,15 @@ Data Compression and Archiving ****************************** The modules described in this chapter support data compression with the zlib, -gzip, bzip2 and lzma algorithms, and the creation of ZIP- and tar-format +gzip, bzip2, lzma, and zstd algorithms, and the creation of ZIP- and tar-format archives. See also :ref:`archiving-operations` provided by the :mod:`shutil` module. .. toctree:: + compression.rst + compression.zstd.rst zlib.rst gzip.rst bz2.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 940698218534e0..f4109fe0e5f2bf 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -11,6 +11,18 @@ **Source code:** :source:`Lib/argparse.py` +.. note:: + + While :mod:`argparse` is the default recommended standard library module + for implementing basic command line applications, authors with more + exacting requirements for exactly how their command line applications + behave may find it doesn't provide the necessary level of control. + Refer to :ref:`choosing-an-argument-parser` for alternatives to + consider when ``argparse`` doesn't support behaviors that the application + requires (such as entirely disabling support for interspersed options and + positional arguments, or accepting option parameter values that start + with ``-`` even when they correspond to another defined option). + -------------- .. sidebar:: Tutorial @@ -50,8 +62,8 @@ the extracted data in a :class:`argparse.Namespace` object:: print(args.filename, args.count, args.verbose) .. note:: - If you're looking a guide about how to upgrade optparse code - to argparse, see :ref:`Upgrading Optparse Code `. + If you're looking for a guide about how to upgrade :mod:`optparse` code + to :mod:`!argparse`, see :ref:`Upgrading Optparse Code `. ArgumentParser objects ---------------------- @@ -62,7 +74,7 @@ ArgumentParser objects prefix_chars='-', fromfile_prefix_chars=None, \ argument_default=None, conflict_handler='error', \ add_help=True, allow_abbrev=True, exit_on_error=True, \ - suggest_on_error=False) + *, suggest_on_error=True, color=True) Create a new :class:`ArgumentParser` object. All parameters should be passed as keyword arguments. Each parameter has its own more detailed description @@ -99,14 +111,15 @@ ArgumentParser objects * add_help_ - Add a ``-h/--help`` option to the parser (default: ``True``) * allow_abbrev_ - Allows long options to be abbreviated if the - abbreviation is unambiguous. (default: ``True``) + abbreviation is unambiguous (default: ``True``) - * exit_on_error_ - Determines whether or not ArgumentParser exits with + * exit_on_error_ - Determines whether or not :class:`!ArgumentParser` exits with error info when an error occurs. (default: ``True``) * suggest_on_error_ - Enables suggestions for mistyped argument choices - and subparser names (default: ``False``) + and subparser names (default: ``True``) + * color_ - Allow color output (default: ``True``) .. versionchanged:: 3.5 *allow_abbrev* parameter was added. @@ -118,6 +131,12 @@ ArgumentParser objects .. versionchanged:: 3.9 *exit_on_error* parameter was added. + .. versionchanged:: 3.14 + *suggest_on_error* and *color* parameters were added. + + .. versionchanged:: 3.15 + *suggest_on_error* default changed to ``True``. + The following sections describe how each of these are used. @@ -192,6 +211,12 @@ arguments it contains. The default message can be overridden with the The ``%(prog)s`` format specifier is available to fill in the program name in your usage messages. +When a custom usage message is specified for the main parser, you may also want to +consider passing the ``prog`` argument to :meth:`~ArgumentParser.add_subparsers` +or the ``prog`` and the ``usage`` arguments to +:meth:`~_SubParsersAction.add_parser`, to ensure consistent command prefixes and +usage information across subparsers. + .. _description: @@ -381,7 +406,7 @@ Most command-line options will use ``-`` as the prefix, e.g. ``-f/--foo``. Parsers that need to support different or additional prefix characters, e.g. for options like ``+f`` or ``/foo``, may specify them using the ``prefix_chars=`` argument -to the ArgumentParser constructor:: +to the :class:`ArgumentParser` constructor:: >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='-+') >>> parser.add_argument('+f') @@ -412,12 +437,18 @@ arguments they contain. For example:: >>> parser.parse_args(['-f', 'foo', '@args.txt']) Namespace(f='bar') -Arguments read from a file must by default be one per line (but see also +Arguments read from a file must be one per line by default (but see also :meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they were in the same place as the original file referencing argument on the command line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. +.. note:: + + Empty lines are treated as empty strings (``''``), which are allowed as values but + not as arguments. Empty lines that are read as arguments will result in an + "unrecognized arguments" error. + :class:`ArgumentParser` uses :term:`filesystem encoding and error handler` to read the file containing arguments. @@ -512,9 +543,9 @@ string was overridden. add_help ^^^^^^^^ -By default, ArgumentParser objects add an option which simply displays +By default, :class:`ArgumentParser` objects add an option which simply displays the parser's help message. If ``-h`` or ``--help`` is supplied at the command -line, the ArgumentParser help will be printed. +line, the :class:`!ArgumentParser` help will be printed. Occasionally, it may be useful to disable the addition of this help option. This can be achieved by passing ``False`` as the ``add_help=`` argument to @@ -568,36 +599,86 @@ suggest_on_error ^^^^^^^^^^^^^^^^ By default, when a user passes an invalid argument choice or subparser name, -:class:`ArgumentParser` will exit with error info and list the permissible -argument choices (if specified) or subparser names as part of the error message. - -If the user would like to enable suggestions for mistyped argument choices and -subparser names, the feature can be enabled by setting ``suggest_on_error`` to -``True``. Note that this only applies for arguments when the choices specified -are strings:: - - >>> parser = argparse.ArgumentParser(description='Process some integers.', suggest_on_error=True) +:class:`ArgumentParser` will exit with error info and provide suggestions for +mistyped arguments. The error message will list the permissible argument +choices (if specified) or subparser names, along with a "maybe you meant" +suggestion if a close match is found. Note that this only applies for arguments +when the choices specified are strings:: + + >>> parser = argparse.ArgumentParser(description='Process some integers.', + suggest_on_error=True) >>> parser.add_argument('--action', choices=['sum', 'max']) >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', ... help='an integer for the accumulator') >>> parser.parse_args(['--action', 'sumn', 1, 2, 3]) tester.py: error: argument --action: invalid choice: 'sumn', maybe you meant 'sum'? (choose from 'sum', 'max') +You can disable suggestions by setting ``suggest_on_error`` to ``False``:: + + >>> parser = argparse.ArgumentParser(description='Process some integers.', + suggest_on_error=False) + .. versionadded:: 3.14 +.. versionchanged:: 3.15 + Changed default value of ``suggest_on_error`` from ``False`` to ``True``. + +color +^^^^^ + +By default, the help message is printed in color using `ANSI escape sequences +`__. +If you want plain text help messages, you can disable this :ref:`in your local +environment `, or in the argument parser itself +by setting ``color`` to ``False``:: + + >>> parser = argparse.ArgumentParser(description='Process some integers.', + ... color=False) + >>> parser.add_argument('--action', choices=['sum', 'max']) + >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', + ... help='an integer for the accumulator') + >>> parser.parse_args(['--help']) + +Note that when ``color=True``, colored output depends on both environment +variables and terminal capabilities. However, if ``color=False``, colored +output is always disabled, even if environment variables like ``FORCE_COLOR`` +are set. + +.. versionadded:: 3.14 + +To highlight inline code in your description or epilog text, you can use +backticks:: + + >>> parser = argparse.ArgumentParser( + ... formatter_class=argparse.RawDescriptionHelpFormatter, + ... epilog='''Examples: + ... `python -m myapp --verbose` + ... `python -m myapp --config settings.json` + ... ''') + +When colors are enabled, the text inside backticks will be displayed in a +distinct color to help examples stand out. When colors are disabled, backticks +are preserved as-is, which is readable in plain text. + +.. note:: + + Backtick markup only applies to description and epilog text. It does not + apply to individual argument ``help`` strings. + +.. versionadded:: 3.15 The add_argument() method ------------------------- -.. method:: ArgumentParser.add_argument(name or flags..., [action], [nargs], \ +.. method:: ArgumentParser.add_argument(name or flags..., *, [action], [nargs], \ [const], [default], [type], [choices], [required], \ [help], [metavar], [dest], [deprecated]) Define how a single command-line argument should be parsed. Each parameter has its own more detailed description below, but in short they are: - * `name or flags`_ - Either a name or a list of option strings, e.g. ``foo`` - or ``-f, --foo``. + * `name or flags`_ - Either a name or a list of option strings, e.g. ``'foo'`` + or ``'-f', '--foo'``. * action_ - The basic type of action to be taken when this argument is encountered at the command line. @@ -662,7 +743,7 @@ be positional:: usage: PROG [-h] [-f FOO] bar PROG: error: the following arguments are required: bar -By default, argparse automatically handles the internal naming and +By default, :mod:`!argparse` automatically handles the internal naming and display names of arguments, simplifying the process without requiring additional configuration. As such, you do not need to specify the dest_ and metavar_ parameters. @@ -672,7 +753,7 @@ upper-cased name. For example:: >>> parser = argparse.ArgumentParser(prog='PROG') >>> parser.add_argument('--foo-bar') - >>> parser.parse_args(['--foo-bar', 'FOO-BAR'] + >>> parser.parse_args(['--foo-bar', 'FOO-BAR']) Namespace(foo_bar='FOO-BAR') >>> parser.print_help() usage: [-h] [--foo-bar FOO-BAR] @@ -707,9 +788,9 @@ how the command-line arguments should be handled. The supplied actions are: Namespace(foo=42) * ``'store_true'`` and ``'store_false'`` - These are special cases of - ``'store_const'`` used for storing the values ``True`` and ``False`` - respectively. In addition, they create default values of ``False`` and - ``True`` respectively:: + ``'store_const'`` that respectively store the values ``True`` and ``False`` + with default values of ``False`` and + ``True``:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', action='store_true') @@ -718,19 +799,19 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--foo --bar'.split()) Namespace(foo=True, bar=False, baz=True) -* ``'append'`` - This stores a list, and appends each argument value to the - list. It is useful to allow an option to be specified multiple times. - If the default value is non-empty, the default elements will be present - in the parsed value for the option, with any values from the - command line appended after those default values. Example usage:: +* ``'append'`` - This appends each argument value to a list. + It is useful for allowing an option to be specified multiple times. + If the default value is a non-empty list, the parsed value will start + with the default list's elements and any values from the command line + will be appended after those default values. Example usage:: >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='append') + >>> parser.add_argument('--foo', action='append', default=['0']) >>> parser.parse_args('--foo 1 --foo 2'.split()) - Namespace(foo=['1', '2']) + Namespace(foo=['0', '1', '2']) -* ``'append_const'`` - This stores a list, and appends the value specified by - the const_ keyword argument to the list; note that the const_ keyword +* ``'append_const'`` - This appends the value specified by + the const_ keyword argument to a list; note that the const_ keyword argument defaults to ``None``. The ``'append_const'`` action is typically useful when multiple arguments need to store constants to the same list. For example:: @@ -741,8 +822,8 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--str --int'.split()) Namespace(types=[, ]) -* ``'extend'`` - This stores a list and appends each item from the multi-value - argument list to it. +* ``'extend'`` - This appends each item from a multi-value + argument to a list. The ``'extend'`` action is typically used with the nargs_ keyword argument value ``'+'`` or ``'*'``. Note that when nargs_ is ``None`` (the default) or ``'?'``, each @@ -756,7 +837,7 @@ how the command-line arguments should be handled. The supplied actions are: .. versionadded:: 3.8 -* ``'count'`` - This counts the number of times a keyword argument occurs. For +* ``'count'`` - This counts the number of times an argument occurs. For example, this is useful for increasing verbosity levels:: >>> parser = argparse.ArgumentParser() @@ -781,25 +862,16 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args(['--version']) PROG 2.0 -Only actions that consume command-line arguments (e.g. ``'store'``, -``'append'`` or ``'extend'``) can be used with positional arguments. - -You may also specify an arbitrary action by passing an Action subclass or -other object that implements the same interface. The ``BooleanOptionalAction`` -is available in ``argparse`` and adds support for boolean actions such as -``--foo`` and ``--no-foo``:: - - >>> import argparse - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) - >>> parser.parse_args(['--no-foo']) - Namespace(foo=False) - -.. versionadded:: 3.9 +You may also specify an arbitrary action by passing an :class:`Action` subclass +(e.g. :class:`BooleanOptionalAction`) or other object that implements the same +interface. Only actions that consume command-line arguments (e.g. ``'store'``, +``'append'``, ``'extend'``, or custom actions with non-zero ``nargs``) can be used +with positional arguments. The recommended way to create a custom action is to extend :class:`Action`, -overriding the ``__call__`` method and optionally the ``__init__`` and -``format_usage`` methods. +overriding the :meth:`!__call__` method and optionally the :meth:`!__init__` and +:meth:`!format_usage` methods. You can also register custom actions using the +:meth:`~ArgumentParser.register` method and reference them by their registered name. An example of a custom action:: @@ -829,7 +901,7 @@ For more details, see :class:`Action`. nargs ^^^^^ -ArgumentParser objects usually associate a single command-line argument with a +:class:`ArgumentParser` objects usually associate a single command-line argument with a single action to be taken. The ``nargs`` keyword argument associates a different number of command-line arguments with a single action. See also :ref:`specifying-ambiguous-arguments`. The supported values are: @@ -894,7 +966,7 @@ See also :ref:`specifying-ambiguous-arguments`. The supported values are: .. index:: single: + (plus); in argparse module -* ``'+'``. Just like ``'*'``, all command-line args present are gathered into a +* ``'+'``. Just like ``'*'``, all command-line arguments present are gathered into a list. Additionally, an error message will be generated if there wasn't at least one command-line argument present. For example:: @@ -934,8 +1006,8 @@ the various :class:`ArgumentParser` actions. The two most common uses of it are (like ``-f`` or ``--foo``) and ``nargs='?'``. This creates an optional argument that can be followed by zero or one command-line arguments. When parsing the command line, if the option string is encountered with no - command-line argument following it, the value of ``const`` will be assumed to - be ``None`` instead. See the nargs_ description for examples. + command-line argument following it, the value from ``const`` will be used. + See the nargs_ description for examples. .. versionchanged:: 3.11 ``const=None`` by default, including when ``action='append_const'`` or @@ -1018,10 +1090,11 @@ necessary type-checking and type conversions to be performed. If the type_ keyword is used with the default_ keyword, the type converter is only applied if the default is a string. -The argument to ``type`` can be any callable that accepts a single string. +The argument to ``type`` can be a callable that accepts a single string or +the name of a registered type (see :meth:`~ArgumentParser.register`) If the function raises :exc:`ArgumentTypeError`, :exc:`TypeError`, or :exc:`ValueError`, the exception is caught and a nicely formatted error -message is displayed. No other exception types are handled. +message is displayed. Other exception types are not handled. Common built-in types and functions can be used as type converters: @@ -1094,16 +1167,21 @@ if the argument was not one of the acceptable values:: game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', 'paper', 'scissors') -Note that inclusion in the *choices* sequence is checked after any type_ -conversions have been performed, so the type of the objects in the *choices* -sequence should match the type_ specified. - Any sequence can be passed as the *choices* value, so :class:`list` objects, :class:`tuple` objects, and custom sequences are all supported. Use of :class:`enum.Enum` is not recommended because it is difficult to control its appearance in usage, help, and error messages. +Note that *choices* are checked after any type_ +conversions have been performed, so objects in *choices* +should match the type_ specified. This can make *choices* +appear unfamiliar in usage, help, or error messages. + +To keep *choices* user-friendly, consider a custom type wrapper that +converts and formats values, or omit type_ and handle conversion in +your application code. + Formatted choices override the default *metavar* which is normally derived from *dest*. This is usually what you want because the user never sees the *dest* parameter. If this display isn't desirable (perhaps because there are @@ -1115,7 +1193,7 @@ many choices), just specify an explicit metavar_. required ^^^^^^^^ -In general, the :mod:`argparse` module assumes that flags like ``-f`` and ``--bar`` +In general, the :mod:`!argparse` module assumes that flags like ``-f`` and ``--bar`` indicate *optional* arguments, which can always be omitted at the command line. To make an option *required*, ``True`` can be specified for the ``required=`` keyword argument to :meth:`~ArgumentParser.add_argument`:: @@ -1168,7 +1246,7 @@ specifiers include the program name, ``%(prog)s`` and most keyword arguments to As the help string supports %-formatting, if you want a literal ``%`` to appear in the help string, you must escape it as ``%%``. -:mod:`argparse` supports silencing the help entry for certain options, by +:mod:`!argparse` supports silencing the help entry for certain options, by setting the ``help`` value to ``argparse.SUPPRESS``:: >>> parser = argparse.ArgumentParser(prog='frobble') @@ -1186,7 +1264,7 @@ metavar ^^^^^^^ When :class:`ArgumentParser` generates help messages, it needs some way to refer -to each expected argument. By default, ArgumentParser objects use the dest_ +to each expected argument. By default, :class:`!ArgumentParser` objects use the dest_ value as the "name" of each object. By default, for positional argument actions, the dest_ value is used directly, and for optional argument actions, the dest_ value is uppercased. So, a single positional argument with @@ -1265,8 +1343,12 @@ attribute is determined by the ``dest`` keyword argument of For optional argument actions, the value of ``dest`` is normally inferred from the option strings. :class:`ArgumentParser` generates the value of ``dest`` by -taking the first long option string and stripping away the initial ``--`` -string. If no long option strings were supplied, ``dest`` will be derived from +taking the first double-dash long option string and stripping away the initial +``-`` characters. +If no double-dash long option strings were supplied, ``dest`` will be derived +from the first single-dash long option string by stripping the initial ``-`` +character. +If no long option strings were supplied, ``dest`` will be derived from the first short option string by stripping the initial ``-`` character. Any internal ``-`` characters will be converted to ``_`` characters to make sure the string is a valid attribute name. The examples below illustrate this @@ -1274,11 +1356,12 @@ behavior:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('-f', '--foo-bar', '--foo') + >>> parser.add_argument('-q', '-quz') >>> parser.add_argument('-x', '-y') - >>> parser.parse_args('-f 1 -x 2'.split()) - Namespace(foo_bar='1', x='2') - >>> parser.parse_args('--foo 1 -y 2'.split()) - Namespace(foo_bar='1', x='2') + >>> parser.parse_args('-f 1 -q 2 -x 3'.split()) + Namespace(foo_bar='1', quz='2', x='3') + >>> parser.parse_args('--foo 1 -quz 2 -y 3'.split()) + Namespace(foo_bar='1', quz='2', x='2') ``dest`` allows a custom attribute name to be provided:: @@ -1287,6 +1370,9 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') +.. versionchanged:: 3.15 + Single-dash long option now takes precedence over short options. + .. _deprecated: @@ -1318,7 +1404,7 @@ printed to :data:`sys.stderr` when the argument is used:: Action classes ^^^^^^^^^^^^^^ -Action classes implement the Action API, a callable which returns a callable +:class:`!Action` classes implement the Action API, a callable which returns a callable which processes arguments from the command-line. Any object which follows this API may be passed as the ``action`` parameter to :meth:`~ArgumentParser.add_argument`. @@ -1327,40 +1413,70 @@ this API may be passed as the ``action`` parameter to type=None, choices=None, required=False, help=None, \ metavar=None) - Action objects are used by an ArgumentParser to represent the information + :class:`!Action` objects are used by an :class:`ArgumentParser` to represent the information needed to parse a single argument from one or more strings from the - command line. The Action class must accept the two positional arguments + command line. The :class:`!Action` class must accept the two positional arguments plus any keyword arguments passed to :meth:`ArgumentParser.add_argument` except for the ``action`` itself. - Instances of Action (or return value of any callable to the ``action`` - parameter) should have attributes "dest", "option_strings", "default", "type", - "required", "help", etc. defined. The easiest way to ensure these attributes - are defined is to call ``Action.__init__``. + Instances of :class:`!Action` (or return value of any callable to the + ``action`` parameter) should have attributes :attr:`!dest`, + :attr:`!option_strings`, :attr:`!default`, :attr:`!type`, :attr:`!required`, + :attr:`!help`, etc. defined. The easiest way to ensure these attributes + are defined is to call :meth:`!Action.__init__`. + + .. method:: __call__(parser, namespace, values, option_string=None) + + :class:`!Action` instances should be callable, so subclasses must override the + :meth:`!__call__` method, which should accept four parameters: + + * *parser* - The :class:`ArgumentParser` object which contains this action. + + * *namespace* - The :class:`Namespace` object that will be returned by + :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this + object using :func:`setattr`. - Action instances should be callable, so subclasses must override the - ``__call__`` method, which should accept four parameters: + * *values* - The associated command-line arguments, with any type conversions + applied. Type conversions are specified with the type_ keyword argument to + :meth:`~ArgumentParser.add_argument`. - * *parser* - The ArgumentParser object which contains this action. + * *option_string* - The option string that was used to invoke this action. + The ``option_string`` argument is optional, and will be absent if the action + is associated with a positional argument. - * *namespace* - The :class:`Namespace` object that will be returned by - :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this - object using :func:`setattr`. + The :meth:`!__call__` method may perform arbitrary actions, but will typically set + attributes on the ``namespace`` based on ``dest`` and ``values``. - * *values* - The associated command-line arguments, with any type conversions - applied. Type conversions are specified with the type_ keyword argument to - :meth:`~ArgumentParser.add_argument`. + .. method:: format_usage() - * *option_string* - The option string that was used to invoke this action. - The ``option_string`` argument is optional, and will be absent if the action - is associated with a positional argument. + :class:`!Action` subclasses can define a :meth:`!format_usage` method that takes no argument + and return a string which will be used when printing the usage of the program. + If such method is not provided, a sensible default will be used. - The ``__call__`` method may perform arbitrary actions, but will typically set - attributes on the ``namespace`` based on ``dest`` and ``values``. +.. class:: BooleanOptionalAction - Action subclasses can define a ``format_usage`` method that takes no argument - and return a string which will be used when printing the usage of the program. - If such method is not provided, a sensible default will be used. + A subclass of :class:`Action` for handling boolean flags with positive + and negative options. Adding a single argument such as ``--foo`` automatically + creates both ``--foo`` and ``--no-foo`` options, storing ``True`` and ``False`` + respectively:: + + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) + >>> parser.parse_args(['--no-foo']) + Namespace(foo=False) + + Single-dash long options are also supported. + For example, negative option ``-nofoo`` is automatically added for + positive option ``-foo``. + But no additional options are added for short options such as ``-f``. + + .. versionadded:: 3.9 + + .. versionchanged:: 3.15 + Added support for single-dash options. + + Added support for alternate prefix_chars_. The parse_args() method @@ -1373,7 +1489,7 @@ The parse_args() method Previous calls to :meth:`add_argument` determine exactly what objects are created and how they are assigned. See the documentation for - :meth:`add_argument` for details. + :meth:`!add_argument` for details. * args_ - List of strings to parse. The default is taken from :data:`sys.argv`. @@ -1529,7 +1645,7 @@ This feature can be disabled by setting :ref:`allow_abbrev` to ``False``. Beyond ``sys.argv`` ^^^^^^^^^^^^^^^^^^^ -Sometimes it may be useful to have an ArgumentParser parse arguments other than those +Sometimes it may be useful to have an :class:`ArgumentParser` parse arguments other than those of :data:`sys.argv`. This can be accomplished by passing a list of strings to :meth:`~ArgumentParser.parse_args`. This is useful for testing at the interactive prompt:: @@ -1584,12 +1700,12 @@ The Namespace object Other utilities --------------- -Sub-commands +Subcommands ^^^^^^^^^^^^ -.. method:: ArgumentParser.add_subparsers([title], [description], [prog], \ +.. method:: ArgumentParser.add_subparsers(*, [title], [description], [prog], \ [parser_class], [action], \ - [option_strings], [dest], [required], \ + [dest], [required], \ [help], [metavar]) Many programs split up their functionality into a number of subcommands, @@ -1598,11 +1714,11 @@ Sub-commands this way can be a particularly good idea when a program performs several different functions which require different kinds of command-line arguments. :class:`ArgumentParser` supports the creation of such subcommands with the - :meth:`add_subparsers` method. The :meth:`add_subparsers` method is normally + :meth:`!add_subparsers` method. The :meth:`!add_subparsers` method is normally called with no arguments and returns a special action object. This object has a single method, :meth:`~_SubParsersAction.add_parser`, which takes a - command name and any :class:`ArgumentParser` constructor arguments, and - returns an :class:`ArgumentParser` object that can be modified as usual. + command name and any :class:`!ArgumentParser` constructor arguments, and + returns an :class:`!ArgumentParser` object that can be modified as usual. Description of parameters: @@ -1613,17 +1729,17 @@ Sub-commands * *description* - description for the sub-parser group in help output, by default ``None`` - * *prog* - usage information that will be displayed with sub-command help, + * *prog* - usage information that will be displayed with subcommand help, by default the name of the program and any positional arguments before the subparser argument * *parser_class* - class which will be used to create sub-parser instances, by - default the class of the current parser (e.g. ArgumentParser) + default the class of the current parser (e.g. :class:`ArgumentParser`) * action_ - the basic type of action to be taken when this argument is encountered at the command line - * dest_ - name of the attribute under which sub-command name will be + * dest_ - name of the attribute under which subcommand name will be stored; by default ``None`` and no value is stored * required_ - Whether or not a subcommand must be provided, by default @@ -1799,7 +1915,11 @@ Sub-commands Namespace(subparser_name='2', y='frobble') .. versionchanged:: 3.7 - New *required* keyword argument. + New *required* keyword-only parameter. + + .. versionchanged:: 3.14 + Subparser's *prog* is no longer affected by a custom usage message in + the main parser. FileType objects @@ -1852,7 +1972,7 @@ Argument groups "positional arguments" and "options" when displaying help messages. When there is a better conceptual grouping of arguments than this default one, appropriate groups can be created using the - :meth:`add_argument_group` method:: + :meth:`!add_argument_group` method:: >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) >>> group = parser.add_argument_group('group') @@ -1869,7 +1989,7 @@ Argument groups has an :meth:`~ArgumentParser.add_argument` method just like a regular :class:`ArgumentParser`. When an argument is added to the group, the parser treats it just like a normal argument, but displays the argument in a - separate group for help messages. The :meth:`add_argument_group` method + separate group for help messages. The :meth:`!add_argument_group` method accepts *title* and *description* arguments which can be used to customize this display:: @@ -1899,15 +2019,14 @@ Argument groups Note that any arguments not in your user-defined groups will end up back in the usual "positional arguments" and "optional arguments" sections. - .. versionchanged:: 3.11 - Calling :meth:`add_argument_group` on an argument group is deprecated. - This feature was never supported and does not always work correctly. - The function exists on the API by accident through inheritance and - will be removed in the future. + .. deprecated-removed:: 3.11 3.14 + Calling :meth:`add_argument_group` on an argument group now raises an + exception. This nesting was never supported, often failed to work + correctly, and was unintentionally exposed through inheritance. .. deprecated:: 3.14 - Passing prefix_chars_ to :meth:`add_argument_group` - is now deprecated. + Passing prefix_chars_ to :meth:`add_argument_group` + is now deprecated. Mutual exclusion @@ -1915,7 +2034,7 @@ Mutual exclusion .. method:: ArgumentParser.add_mutually_exclusive_group(required=False) - Create a mutually exclusive group. :mod:`argparse` will make sure that only + Create a mutually exclusive group. :mod:`!argparse` will make sure that only one of the arguments in the mutually exclusive group was present on the command line:: @@ -1966,11 +2085,11 @@ Mutual exclusion --foo FOO foo help --bar BAR bar help - .. versionchanged:: 3.11 - Calling :meth:`add_argument_group` or :meth:`add_mutually_exclusive_group` - on a mutually exclusive group is deprecated. These features were never - supported and do not always work correctly. The functions exist on the - API by accident through inheritance and will be removed in the future. + .. deprecated-removed:: 3.11 3.14 + Calling :meth:`add_argument_group` or :meth:`add_mutually_exclusive_group` + on a mutually exclusive group now raises an exception. This nesting was + never supported, often failed to work correctly, and was unintentionally + exposed through inheritance. Parser defaults @@ -1990,7 +2109,9 @@ Parser defaults >>> parser.parse_args(['736']) Namespace(bar=42, baz='badger', foo=736) - Note that parser-level defaults always override argument-level defaults:: + Note that defaults can be set at both the parser level using :meth:`set_defaults` + and at the argument level using :meth:`add_argument`. If both are called for the + same argument, the last default set for an argument is used:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default='bar') @@ -2052,12 +2173,15 @@ Partial parsing .. method:: ArgumentParser.parse_known_args(args=None, namespace=None) - Sometimes a script may only parse a few of the command-line arguments, passing - the remaining arguments on to another script or program. In these cases, the - :meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like - :meth:`~ArgumentParser.parse_args` except that it does not produce an error when - extra arguments are present. Instead, it returns a two item tuple containing - the populated namespace and the list of remaining argument strings. + Sometimes a script only needs to handle a specific set of command-line + arguments, leaving any unrecognized arguments for another script or program. + In these cases, the :meth:`~ArgumentParser.parse_known_args` method can be + useful. + + This method works similarly to :meth:`~ArgumentParser.parse_args`, but it does + not raise an error for extra, unrecognized arguments. Instead, it parses the + known arguments and returns a two item tuple that contains the populated + namespace and the list of any unrecognized arguments. :: @@ -2128,7 +2252,7 @@ Intermixed parsing and :meth:`~ArgumentParser.parse_known_intermixed_args` methods support this parsing style. - These parsers do not support all the argparse features, and will raise + These parsers do not support all the :mod:`!argparse` features, and will raise exceptions if unsupported features are used. In particular, subparsers, and mutually exclusive groups that include both optionals and positionals are not supported. @@ -2156,6 +2280,34 @@ Intermixed parsing .. versionadded:: 3.7 +Registering custom types or actions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. method:: ArgumentParser.register(registry_name, value, object) + + Sometimes it's desirable to use a custom string in error messages to provide + more user-friendly output. In these cases, :meth:`!register` can be used to + register custom actions or types with a parser and allow you to reference the + type by their registered name instead of their callable name. + + The :meth:`!register` method accepts three arguments - a *registry_name*, + specifying the internal registry where the object will be stored (e.g., + ``action``, ``type``), *value*, which is the key under which the object will + be registered, and object, the callable to be registered. + + The following example shows how to register a custom type with a parser:: + + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> parser.register('type', 'hexadecimal integer', lambda s: int(s, 16)) + >>> parser.add_argument('--foo', type='hexadecimal integer') + _StoreAction(option_strings=['--foo'], dest='foo', nargs=None, const=None, default=None, type='hexadecimal integer', choices=None, required=False, help=None, metavar=None, deprecated=False) + >>> parser.parse_args(['--foo', '0xFA']) + Namespace(foo=250) + >>> parser.parse_args(['--foo', '1.2']) + usage: PROG [-h] [--foo FOO] + PROG: error: argument --foo: invalid 'hexadecimal integer' value: '1.2' + Exceptions ---------- diff --git a/Doc/library/array.rst b/Doc/library/array.rst index e0b1eb89cf6c05..5592bd7089ba49 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -9,7 +9,7 @@ -------------- This module defines an object type which can compactly represent an array of -basic values: characters, integers, floating-point numbers. Arrays are sequence +basic values: characters, integers, floating-point numbers. Arrays are mutable :term:`sequence` types and behave very much like lists, except that the type of objects stored in them is constrained. The type is specified at object creation time by using a :dfn:`type code`, which is a single character. The following type codes are @@ -24,7 +24,7 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'u'`` | wchar_t | Unicode character | 2 | \(1) | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'w'`` | Py_UCS4 | Unicode character | 4 | | +| ``'w'`` | Py_UCS4 | Unicode character | 4 | \(2) | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'h'`` | signed short | int | 2 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -60,6 +60,9 @@ Notes: .. deprecated-removed:: 3.3 3.16 Please migrate to ``'w'`` typecode. +(2) + .. versionadded:: 3.13 + The actual representation of values is determined by the machine architecture (strictly speaking, by the C implementation). The actual size can be accessed @@ -90,7 +93,7 @@ The module defines the following type: otherwise, the initializer's iterator is passed to the :meth:`extend` method to add initial items to the array. - Array objects support the ordinary sequence operations of indexing, slicing, + Array objects support the ordinary :ref:`mutable ` :term:`sequence` operations of indexing, slicing, concatenation, and multiplication. When using slice assignment, the assigned value must be an array object with the same type code; in all other cases, :exc:`TypeError` is raised. Array objects also implement the buffer interface, diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 22d8c87cb58e78..bf37540e5faf42 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1,4 +1,4 @@ -:mod:`!ast` --- Abstract Syntax Trees +:mod:`!ast` --- Abstract syntax trees ===================================== .. module:: ast @@ -29,7 +29,7 @@ compiled into a Python code object using the built-in :func:`compile` function. .. _abstract-grammar: -Abstract Grammar +Abstract grammar ---------------- The abstract grammar is currently defined as follows: @@ -139,12 +139,13 @@ Node classes The :meth:`~object.__repr__` output of :class:`~ast.AST` nodes includes the values of the node fields. -.. deprecated:: 3.8 +.. deprecated-removed:: 3.8 3.14 - Old classes :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, - :class:`!ast.NameConstant` and :class:`!ast.Ellipsis` are still available, - but they will be removed in future Python releases. In the meantime, - instantiating them will return an instance of a different class. + Previous versions of Python provided the AST classes :class:`!ast.Num`, + :class:`!ast.Str`, :class:`!ast.Bytes`, :class:`!ast.NameConstant` and + :class:`!ast.Ellipsis`, which were deprecated in Python 3.8. These classes + were removed in Python 3.14, and their functionality has been replaced with + :class:`ast.Constant`. .. deprecated:: 3.9 @@ -252,12 +253,11 @@ Root nodes >>> print(ast.dump(ast.parse('(int, str) -> List[int]', mode='func_type'), indent=4)) FunctionType( argtypes=[ - Name(id='int', ctx=Load()), - Name(id='str', ctx=Load())], + Name(id='int'), + Name(id='str')], returns=Subscript( - value=Name(id='List', ctx=Load()), - slice=Name(id='int', ctx=Load()), - ctx=Load())) + value=Name(id='List'), + slice=Name(id='int'))) .. versionadded:: 3.8 @@ -268,9 +268,9 @@ Literals .. class:: Constant(value) A constant value. The ``value`` attribute of the ``Constant`` literal contains the - Python object it represents. The values represented can be simple types - such as a number, string or ``None``, but also immutable container types - (tuples and frozensets) if all of their elements are constant. + Python object it represents. The values represented can be instances of :class:`str`, + :class:`bytes`, :class:`int`, :class:`float`, :class:`complex`, and :class:`bool`, + and the constants :data:`None` and :data:`Ellipsis`. .. doctest:: @@ -290,9 +290,9 @@ Literals * ``conversion`` is an integer: * -1: no formatting - * 115: ``!s`` string formatting - * 114: ``!r`` repr formatting - * 97: ``!a`` ascii formatting + * 97 (``ord('a')``): ``!a`` :func:`ASCII ` formatting + * 114 (``ord('r')``): ``!r`` :func:`repr` formatting + * 115 (``ord('s')``): ``!s`` :func:`string ` formatting * ``format_spec`` is a :class:`JoinedStr` node representing the formatting of the value, or ``None`` if no format was specified. Both @@ -312,20 +312,77 @@ Literals values=[ Constant(value='sin('), FormattedValue( - value=Name(id='a', ctx=Load()), + value=Name(id='a'), conversion=-1), Constant(value=') is '), FormattedValue( value=Call( - func=Name(id='sin', ctx=Load()), + func=Name(id='sin'), args=[ - Name(id='a', ctx=Load())]), + Name(id='a')]), conversion=-1, format_spec=JoinedStr( values=[ Constant(value='.3')]))])) +.. class:: TemplateStr(values, /) + + .. versionadded:: 3.14 + + Node representing a template string literal, comprising a series of + :class:`Interpolation` and :class:`Constant` nodes. + These nodes may be any order, and do not need to be interleaved. + + .. doctest:: + + >>> expr = ast.parse('t"{name} finished {place:ordinal}"', mode='eval') + >>> print(ast.dump(expr, indent=4)) + Expression( + body=TemplateStr( + values=[ + Interpolation( + value=Name(id='name'), + str='name', + conversion=-1), + Constant(value=' finished '), + Interpolation( + value=Name(id='place'), + str='place', + conversion=-1, + format_spec=JoinedStr( + values=[ + Constant(value='ordinal')]))])) + +.. class:: Interpolation(value, str, conversion, format_spec=None) + + .. versionadded:: 3.14 + + Node representing a single interpolation field in a template string literal. + + * ``value`` is any expression node (such as a literal, a variable, or a + function call). + This has the same meaning as ``FormattedValue.value``. + * ``str`` is a constant containing the text of the interpolation expression. + + If ``str`` is set to ``None``, then ``value`` is used to generate code + when calling :func:`ast.unparse`. This no longer guarantees that the + generated code is identical to the original and is intended for code + generation. + * ``conversion`` is an integer: + + * -1: no conversion + * 97 (``ord('a')``): ``!a`` :func:`ASCII ` conversion + * 114 (``ord('r')``): ``!r`` :func:`repr` conversion + * 115 (``ord('s')``): ``!s`` :func:`string ` conversion + + This has the same meaning as ``FormattedValue.conversion``. + * ``format_spec`` is a :class:`JoinedStr` node representing the formatting + of the value, or ``None`` if no format was specified. Both + ``conversion`` and ``format_spec`` can be set at the same time. + This has the same meaning as ``FormattedValue.format_spec``. + + .. class:: List(elts, ctx) Tuple(elts, ctx) @@ -341,16 +398,14 @@ Literals elts=[ Constant(value=1), Constant(value=2), - Constant(value=3)], - ctx=Load())) + Constant(value=3)])) >>> print(ast.dump(ast.parse('(1, 2, 3)', mode='eval'), indent=4)) Expression( body=Tuple( elts=[ Constant(value=1), Constant(value=2), - Constant(value=3)], - ctx=Load())) + Constant(value=3)])) .. class:: Set(elts) @@ -388,7 +443,7 @@ Literals None], values=[ Constant(value=1), - Name(id='d', ctx=Load())])) + Name(id='d')])) Variables @@ -414,7 +469,7 @@ Variables Module( body=[ Expr( - value=Name(id='a', ctx=Load()))]) + value=Name(id='a'))]) >>> print(ast.dump(ast.parse('a = 1'), indent=4)) Module( @@ -452,7 +507,7 @@ Variables value=Name(id='b', ctx=Store()), ctx=Store())], ctx=Store())], - value=Name(id='it', ctx=Load()))]) + value=Name(id='it'))]) .. _ast-expressions: @@ -475,7 +530,7 @@ Expressions Expr( value=UnaryOp( op=USub(), - operand=Name(id='a', ctx=Load())))]) + operand=Name(id='a')))]) .. class:: UnaryOp(op, operand) @@ -498,7 +553,7 @@ Expressions Expression( body=UnaryOp( op=Not(), - operand=Name(id='x', ctx=Load()))) + operand=Name(id='x'))) .. class:: BinOp(left, op, right) @@ -511,9 +566,9 @@ Expressions >>> print(ast.dump(ast.parse('x + y', mode='eval'), indent=4)) Expression( body=BinOp( - left=Name(id='x', ctx=Load()), + left=Name(id='x'), op=Add(), - right=Name(id='y', ctx=Load()))) + right=Name(id='y'))) .. class:: Add @@ -549,8 +604,8 @@ Expressions body=BoolOp( op=Or(), values=[ - Name(id='x', ctx=Load()), - Name(id='y', ctx=Load())])) + Name(id='x'), + Name(id='y')])) .. class:: And @@ -575,7 +630,7 @@ Expressions LtE(), Lt()], comparators=[ - Name(id='a', ctx=Load()), + Name(id='a'), Constant(value=10)])) @@ -609,18 +664,17 @@ Expressions >>> print(ast.dump(ast.parse('func(a, b=c, *d, **e)', mode='eval'), indent=4)) Expression( body=Call( - func=Name(id='func', ctx=Load()), + func=Name(id='func'), args=[ - Name(id='a', ctx=Load()), + Name(id='a'), Starred( - value=Name(id='d', ctx=Load()), - ctx=Load())], + value=Name(id='d'))], keywords=[ keyword( arg='b', - value=Name(id='c', ctx=Load())), + value=Name(id='c')), keyword( - value=Name(id='e', ctx=Load()))])) + value=Name(id='e'))])) .. class:: keyword(arg, value) @@ -639,9 +693,9 @@ Expressions >>> print(ast.dump(ast.parse('a if b else c', mode='eval'), indent=4)) Expression( body=IfExp( - test=Name(id='b', ctx=Load()), - body=Name(id='a', ctx=Load()), - orelse=Name(id='c', ctx=Load()))) + test=Name(id='b'), + body=Name(id='a'), + orelse=Name(id='c'))) .. class:: Attribute(value, attr, ctx) @@ -656,9 +710,8 @@ Expressions >>> print(ast.dump(ast.parse('snake.colour', mode='eval'), indent=4)) Expression( body=Attribute( - value=Name(id='snake', ctx=Load()), - attr='colour', - ctx=Load())) + value=Name(id='snake'), + attr='colour')) .. class:: NamedExpr(target, value) @@ -694,15 +747,13 @@ Subscripting >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) Expression( body=Subscript( - value=Name(id='l', ctx=Load()), + value=Name(id='l'), slice=Tuple( elts=[ Slice( lower=Constant(value=1), upper=Constant(value=2)), - Constant(value=3)], - ctx=Load()), - ctx=Load())) + Constant(value=3)]))) .. class:: Slice(lower, upper, step) @@ -716,11 +767,10 @@ Subscripting >>> print(ast.dump(ast.parse('l[1:2]', mode='eval'), indent=4)) Expression( body=Subscript( - value=Name(id='l', ctx=Load()), + value=Name(id='l'), slice=Slice( lower=Constant(value=1), - upper=Constant(value=2)), - ctx=Load())) + upper=Constant(value=2)))) Comprehensions @@ -745,11 +795,11 @@ Comprehensions ... )) Expression( body=ListComp( - elt=Name(id='x', ctx=Load()), + elt=Name(id='x'), generators=[ comprehension( target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), + iter=Name(id='numbers'), is_async=0)])) >>> print(ast.dump( ... ast.parse('{x: x**2 for x in numbers}', mode='eval'), @@ -757,15 +807,15 @@ Comprehensions ... )) Expression( body=DictComp( - key=Name(id='x', ctx=Load()), + key=Name(id='x'), value=BinOp( - left=Name(id='x', ctx=Load()), + left=Name(id='x'), op=Pow(), right=Constant(value=2)), generators=[ comprehension( target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), + iter=Name(id='numbers'), is_async=0)])) >>> print(ast.dump( ... ast.parse('{x for x in numbers}', mode='eval'), @@ -773,11 +823,11 @@ Comprehensions ... )) Expression( body=SetComp( - elt=Name(id='x', ctx=Load()), + elt=Name(id='x'), generators=[ comprehension( target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), + iter=Name(id='numbers'), is_async=0)])) @@ -798,17 +848,17 @@ Comprehensions Expression( body=ListComp( elt=Call( - func=Name(id='ord', ctx=Load()), + func=Name(id='ord'), args=[ - Name(id='c', ctx=Load())]), + Name(id='c')]), generators=[ comprehension( target=Name(id='line', ctx=Store()), - iter=Name(id='file', ctx=Load()), + iter=Name(id='file'), is_async=0), comprehension( target=Name(id='c', ctx=Store()), - iter=Name(id='line', ctx=Load()), + iter=Name(id='line'), is_async=0)])) >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), @@ -816,22 +866,22 @@ Comprehensions Expression( body=GeneratorExp( elt=BinOp( - left=Name(id='n', ctx=Load()), + left=Name(id='n'), op=Pow(), right=Constant(value=2)), generators=[ comprehension( target=Name(id='n', ctx=Store()), - iter=Name(id='it', ctx=Load()), + iter=Name(id='it'), ifs=[ Compare( - left=Name(id='n', ctx=Load()), + left=Name(id='n'), ops=[ Gt()], comparators=[ Constant(value=5)]), Compare( - left=Name(id='n', ctx=Load()), + left=Name(id='n'), ops=[ Lt()], comparators=[ @@ -842,11 +892,11 @@ Comprehensions ... indent=4)) # Async comprehension Expression( body=ListComp( - elt=Name(id='i', ctx=Load()), + elt=Name(id='i'), generators=[ comprehension( target=Name(id='i', ctx=Store()), - iter=Name(id='soc', ctx=Load()), + iter=Name(id='soc'), is_async=1)])) @@ -888,7 +938,7 @@ Statements Name(id='a', ctx=Store()), Name(id='b', ctx=Store())], ctx=Store())], - value=Name(id='c', ctx=Load()))]) + value=Name(id='c'))]) .. class:: AnnAssign(target, annotation, value, simple) @@ -911,7 +961,7 @@ Statements body=[ AnnAssign( target=Name(id='c', ctx=Store()), - annotation=Name(id='int', ctx=Load()), + annotation=Name(id='int'), simple=1)]) >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis @@ -919,7 +969,7 @@ Statements body=[ AnnAssign( target=Name(id='a', ctx=Store()), - annotation=Name(id='int', ctx=Load()), + annotation=Name(id='int'), value=Constant(value=1), simple=0)]) @@ -928,10 +978,10 @@ Statements body=[ AnnAssign( target=Attribute( - value=Name(id='a', ctx=Load()), + value=Name(id='a'), attr='b', ctx=Store()), - annotation=Name(id='int', ctx=Load()), + annotation=Name(id='int'), simple=0)]) >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation @@ -939,10 +989,10 @@ Statements body=[ AnnAssign( target=Subscript( - value=Name(id='a', ctx=Load()), + value=Name(id='a'), slice=Constant(value=1), ctx=Store()), - annotation=Name(id='int', ctx=Load()), + annotation=Name(id='int'), simple=0)]) @@ -979,8 +1029,8 @@ Statements Module( body=[ Raise( - exc=Name(id='x', ctx=Load()), - cause=Name(id='y', ctx=Load()))]) + exc=Name(id='x'), + cause=Name(id='y'))]) .. class:: Assert(test, msg) @@ -994,8 +1044,8 @@ Statements Module( body=[ Assert( - test=Name(id='x', ctx=Load()), - msg=Name(id='y', ctx=Load()))]) + test=Name(id='x'), + msg=Name(id='y'))]) .. class:: Delete(targets) @@ -1041,7 +1091,7 @@ Statements body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), - value=Name(id='int', ctx=Load()))]) + value=Name(id='int'))]) .. versionadded:: 3.12 @@ -1134,13 +1184,13 @@ Control flow Module( body=[ If( - test=Name(id='x', ctx=Load()), + test=Name(id='x'), body=[ Expr( value=Constant(value=Ellipsis))], orelse=[ If( - test=Name(id='y', ctx=Load()), + test=Name(id='y'), body=[ Expr( value=Constant(value=Ellipsis))], @@ -1174,7 +1224,7 @@ Control flow body=[ For( target=Name(id='x', ctx=Store()), - iter=Name(id='y', ctx=Load()), + iter=Name(id='y'), body=[ Expr( value=Constant(value=Ellipsis))], @@ -1190,7 +1240,7 @@ Control flow .. doctest:: - >> print(ast.dump(ast.parse(""" + >>> print(ast.dump(ast.parse(""" ... while x: ... ... ... else: @@ -1199,7 +1249,7 @@ Control flow Module( body=[ While( - test=Name(id='x', ctx=Load()), + test=Name(id='x'), body=[ Expr( value=Constant(value=Ellipsis))], @@ -1227,11 +1277,11 @@ Control flow body=[ For( target=Name(id='a', ctx=Store()), - iter=Name(id='b', ctx=Load()), + iter=Name(id='b'), body=[ If( test=Compare( - left=Name(id='a', ctx=Load()), + left=Name(id='a'), ops=[ Gt()], comparators=[ @@ -1269,12 +1319,12 @@ Control flow value=Constant(value=Ellipsis))], handlers=[ ExceptHandler( - type=Name(id='Exception', ctx=Load()), + type=Name(id='Exception'), body=[ Expr( value=Constant(value=Ellipsis))]), ExceptHandler( - type=Name(id='OtherException', ctx=Load()), + type=Name(id='OtherException'), name='e', body=[ Expr( @@ -1309,7 +1359,7 @@ Control flow value=Constant(value=Ellipsis))], handlers=[ ExceptHandler( - type=Name(id='Exception', ctx=Load()), + type=Name(id='Exception'), body=[ Expr( value=Constant(value=Ellipsis))])])]) @@ -1337,12 +1387,12 @@ Control flow body=[ Expr( value=BinOp( - left=Name(id='a', ctx=Load()), + left=Name(id='a'), op=Add(), right=Constant(value=1)))], handlers=[ ExceptHandler( - type=Name(id='TypeError', ctx=Load()), + type=Name(id='TypeError'), body=[ Pass()])])]) @@ -1375,18 +1425,18 @@ Control flow With( items=[ withitem( - context_expr=Name(id='a', ctx=Load()), + context_expr=Name(id='a'), optional_vars=Name(id='b', ctx=Store())), withitem( - context_expr=Name(id='c', ctx=Load()), + context_expr=Name(id='c'), optional_vars=Name(id='d', ctx=Store()))], body=[ Expr( value=Call( - func=Name(id='something', ctx=Load()), + func=Name(id='something'), args=[ - Name(id='b', ctx=Load()), - Name(id='d', ctx=Load())]))])]) + Name(id='b'), + Name(id='d')]))])]) Pattern matching @@ -1426,14 +1476,14 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchSequence( patterns=[ MatchAs(name='x')]), guard=Compare( - left=Name(id='x', ctx=Load()), + left=Name(id='x'), ops=[ Gt()], comparators=[ @@ -1443,7 +1493,7 @@ Pattern matching value=Constant(value=Ellipsis))]), match_case( pattern=MatchClass( - cls=Name(id='tuple', ctx=Load())), + cls=Name(id='tuple')), body=[ Expr( value=Constant(value=Ellipsis))])])]) @@ -1467,7 +1517,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchValue( @@ -1494,7 +1544,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchSingleton(value=None), @@ -1521,7 +1571,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchSequence( @@ -1554,7 +1604,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchSequence( @@ -1603,7 +1653,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchMapping( @@ -1653,11 +1703,11 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchClass( - cls=Name(id='Point2D', ctx=Load()), + cls=Name(id='Point2D'), patterns=[ MatchValue( value=Constant(value=0)), @@ -1668,7 +1718,7 @@ Pattern matching value=Constant(value=Ellipsis))]), match_case( pattern=MatchClass( - cls=Name(id='Point3D', ctx=Load()), + cls=Name(id='Point3D'), kwd_attrs=[ 'x', 'y', @@ -1709,7 +1759,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchAs( @@ -1746,7 +1796,7 @@ Pattern matching Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchOr( @@ -1761,6 +1811,43 @@ Pattern matching .. versionadded:: 3.10 + +Type annotations +^^^^^^^^^^^^^^^^ + +.. class:: TypeIgnore(lineno, tag) + + A ``# type: ignore`` comment located at *lineno*. + *tag* is the optional tag specified by the form ``# type: ignore ``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x = 1 # type: ignore', type_comments=True), indent=4)) + Module( + body=[ + Assign( + targets=[ + Name(id='x', ctx=Store())], + value=Constant(value=1))], + type_ignores=[ + TypeIgnore(lineno=1, tag='')]) + >>> print(ast.dump(ast.parse('x: bool = 1 # type: ignore[assignment]', type_comments=True), indent=4)) + Module( + body=[ + AnnAssign( + target=Name(id='x', ctx=Store()), + annotation=Name(id='bool'), + value=Constant(value=1), + simple=1)], + type_ignores=[ + TypeIgnore(lineno=1, tag='[assignment]')]) + + .. note:: + :class:`!TypeIgnore` nodes are not generated when the *type_comments* parameter + is set to ``False`` (default). See :func:`ast.parse` for more details. + + .. versionadded:: 3.8 + .. _ast-type-params: Type parameters @@ -1787,12 +1874,11 @@ aliases. type_params=[ TypeVar( name='T', - bound=Name(id='int', ctx=Load()), - default_value=Name(id='bool', ctx=Load()))], + bound=Name(id='int'), + default_value=Name(id='bool'))], value=Subscript( - value=Name(id='list', ctx=Load()), - slice=Name(id='T', ctx=Load()), - ctx=Load()))]) + value=Name(id='list'), + slice=Name(id='T')))]) .. versionadded:: 3.12 @@ -1807,7 +1893,7 @@ aliases. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[**P = (int, str)] = Callable[P, int]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[**P = [int, str]] = Callable[P, int]"), indent=4)) Module( body=[ TypeAlias( @@ -1815,19 +1901,16 @@ aliases. type_params=[ ParamSpec( name='P', - default_value=Tuple( + default_value=List( elts=[ - Name(id='int', ctx=Load()), - Name(id='str', ctx=Load())], - ctx=Load()))], + Name(id='int'), + Name(id='str')]))], value=Subscript( - value=Name(id='Callable', ctx=Load()), + value=Name(id='Callable'), slice=Tuple( elts=[ - Name(id='P', ctx=Load()), - Name(id='int', ctx=Load())], - ctx=Load()), - ctx=Load()))]) + Name(id='P'), + Name(id='int')])))]) .. versionadded:: 3.12 @@ -1848,18 +1931,13 @@ aliases. TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - TypeVarTuple( - name='Ts', - default_value=Tuple(ctx=Load()))], + TypeVarTuple(name='Ts', default_value=Tuple())], value=Subscript( - value=Name(id='tuple', ctx=Load()), + value=Name(id='tuple'), slice=Tuple( elts=[ Starred( - value=Name(id='Ts', ctx=Load()), - ctx=Load())], - ctx=Load()), - ctx=Load()))]) + value=Name(id='Ts'))])))]) .. versionadded:: 3.12 @@ -1964,8 +2042,8 @@ Function and class definitions body=[ Pass()], decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())], + Name(id='decorator1'), + Name(id='decorator2')], returns=Constant(value='return annotation'))]) @@ -1995,14 +2073,14 @@ Function and class definitions body=[ Expr( value=Yield( - value=Name(id='x', ctx=Load())))]) + value=Name(id='x')))]) >>> print(ast.dump(ast.parse('yield from x'), indent=4)) Module( body=[ Expr( value=YieldFrom( - value=Name(id='x', ctx=Load())))]) + value=Name(id='x')))]) .. class:: Global(names) @@ -2057,17 +2135,17 @@ Function and class definitions ClassDef( name='Foo', bases=[ - Name(id='base1', ctx=Load()), - Name(id='base2', ctx=Load())], + Name(id='base1'), + Name(id='base2')], keywords=[ keyword( arg='metaclass', - value=Name(id='meta', ctx=Load()))], + value=Name(id='meta'))], body=[ Pass()], decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())])]) + Name(id='decorator1'), + Name(id='decorator2')])]) .. versionchanged:: 3.12 Added ``type_params``. @@ -2104,7 +2182,7 @@ Async and await Expr( value=Await( value=Call( - func=Name(id='other_func', ctx=Load()))))])]) + func=Name(id='other_func'))))])]) .. class:: AsyncFor(target, iter, body, orelse, type_comment) @@ -2119,19 +2197,19 @@ Async and await of :class:`ast.operator`, :class:`ast.unaryop`, :class:`ast.cmpop`, :class:`ast.boolop` and :class:`ast.expr_context`) on the returned tree will be singletons. Changes to one will be reflected in all other - occurrences of the same value (e.g. :class:`ast.Add`). + occurrences of the same value (for example, :class:`ast.Add`). -:mod:`ast` Helpers +:mod:`ast` helpers ------------------ Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None) Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, flags=FLAGS_VALUE, optimize=optimize)``, + filename, mode, flags=FLAGS_VALUE, optimize=optimize, module=module)``, where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0`` and ``ast.PyCF_OPTIMIZED_AST`` otherwise. @@ -2184,6 +2262,9 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. + .. versionadded:: 3.15 + Added the *module* parameter. + .. function:: unparse(ast_obj) @@ -2339,12 +2420,12 @@ and classes for traversing abstract syntax trees: during traversal. For this a special visitor exists (:class:`NodeTransformer`) that allows modifications. - .. deprecated:: 3.8 + .. deprecated-removed:: 3.8 3.14 Methods :meth:`!visit_Num`, :meth:`!visit_Str`, :meth:`!visit_Bytes`, - :meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` are deprecated - now and will not be called in future Python versions. Add the - :meth:`visit_Constant` method to handle all constant nodes. + :meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` will not be called + in Python 3.14+. Add the :meth:`visit_Constant` method instead to handle + all constant nodes. .. class:: NodeTransformer() @@ -2365,7 +2446,7 @@ and classes for traversing abstract syntax trees: def visit_Name(self, node): return Subscript( - value=Name(id='data', ctx=Load()), + value=Name(id='data'), slice=Constant(value=node.id), ctx=node.ctx ) @@ -2408,8 +2489,26 @@ and classes for traversing abstract syntax trees: indents that many spaces per level. If *indent* is a string (such as ``"\t"``), that string is used to indent each level. - If *show_empty* is ``False`` (the default), empty lists and fields that are ``None`` - will be omitted from the output. + If *show_empty* is false (the default), optional empty lists and + ``Load()`` values will be omitted from the output. + Optional ``None`` values are always omitted. + + .. doctest:: + + >>> tree = ast.parse('print(None)', '?', 'eval') + >>> print(ast.dump(tree, indent=4)) + Expression( + body=Call( + func=Name(id='print'), + args=[ + Constant(value=None)])) + >>> print(ast.dump(tree, indent=4, show_empty=True)) + Expression( + body=Call( + func=Name(id='print', ctx=Load()), + args=[ + Constant(value=None)], + keywords=[])) .. versionchanged:: 3.9 Added the *indent* option. @@ -2417,37 +2516,13 @@ and classes for traversing abstract syntax trees: .. versionchanged:: 3.13 Added the *show_empty* option. - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... async def f(): - ... await other_func() - ... """), indent=4, show_empty=True)) - Module( - body=[ - AsyncFunctionDef( - name='f', - args=arguments( - posonlyargs=[], - args=[], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=[ - Expr( - value=Await( - value=Call( - func=Name(id='other_func', ctx=Load()), - args=[], - keywords=[])))], - decorator_list=[], - type_params=[])], - type_ignores=[]) + .. versionchanged:: 3.15 + Omit optional ``Load()`` values by default. .. _ast-compiler-flags: -Compiler Flags +Compiler flags -------------- The following flags may be passed to :func:`compile` in order to change @@ -2496,7 +2571,7 @@ effects on the compilation of a program: .. _ast-cli: -Command-Line Usage +Command-line usage ------------------ .. versionadded:: 3.9 @@ -2535,6 +2610,28 @@ The following options are accepted: Indentation of nodes in AST (number of spaces). +.. option:: --feature-version + + Python version in the format 3.x (for example, 3.10). Defaults to the + current version of the interpreter. + + .. versionadded:: 3.14 + +.. option:: -O + --optimize + + Optimization level for parser. Defaults to no optimization. + + .. versionadded:: 3.14 + +.. option:: --show-empty + + Show empty lists and fields that are ``None``. Defaults to not showing empty + objects. + + .. versionadded:: 3.14 + + If :file:`infile` is specified its contents are parsed to AST and dumped to stdout. Otherwise, the content is read from stdin. diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst new file mode 100644 index 00000000000000..5e5c3a99fe66f1 --- /dev/null +++ b/Doc/library/asynchat.rst @@ -0,0 +1,17 @@ +:mod:`!asynchat` --- Asynchronous socket command/response handler +================================================================= + +.. module:: asynchat + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.6 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.6. The removal was decided in :pep:`594`. + +Applications should use the :mod:`asyncio` module instead. + +The last version of Python that provided the :mod:`!asynchat` module was +`Python 3.11 `_. diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 44b507a9811116..7831b613bd4a60 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -46,10 +46,6 @@ In addition to enabling the debug mode, consider also: When the debug mode is enabled: -* asyncio checks for :ref:`coroutines that were not awaited - ` and logs them; this mitigates - the "forgotten await" pitfall. - * Many non-threadsafe asyncio APIs (such as :meth:`loop.call_soon` and :meth:`loop.call_at` methods) raise an exception if they are called from a wrong thread. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 14fd153f640f05..72f484fd1cbe77 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -59,9 +59,15 @@ an event loop: instead of using these lower level functions to manually create and close an event loop. - .. deprecated:: 3.12 - Deprecation warning is emitted if there is no current event loop. - In some future Python release this will become an error. + .. versionchanged:: 3.14 + Raises a :exc:`RuntimeError` if there is no current event loop. + + .. note:: + + The :mod:`!asyncio` policy system is deprecated and will be removed + in Python 3.16; from there on, this function will return the current + running event loop if present else it will return the + loop set by :func:`set_event_loop`. .. function:: set_event_loop(loop) @@ -162,7 +168,8 @@ Running and stopping the loop This method is idempotent and irreversible. No other methods should be called after the event loop is closed. -.. coroutinemethod:: loop.shutdown_asyncgens() +.. method:: loop.shutdown_asyncgens() + :async: Schedule all currently open :term:`asynchronous generator` objects to close with an :meth:`~agen.aclose` call. After calling this method, @@ -183,7 +190,8 @@ Running and stopping the loop .. versionadded:: 3.6 -.. coroutinemethod:: loop.shutdown_default_executor(timeout=None) +.. method:: loop.shutdown_default_executor(timeout=None) + :async: Schedule the closure of the default executor and wait for it to join all of the threads in the :class:`~concurrent.futures.ThreadPoolExecutor`. @@ -236,6 +244,9 @@ Scheduling callbacks another thread, this function *must* be used, since :meth:`call_soon` is not thread-safe. + This function is safe to be called from a reentrant context or signal handler, + however, it is not safe or fruitful to use the returned handle in such contexts. + Raises :exc:`RuntimeError` if called on a loop that's been closed. This can happen on a secondary thread when the main application is shutting down. @@ -293,6 +304,12 @@ clocks to track time. custom :class:`contextvars.Context` for the *callback* to run in. The current context is used when no *context* is provided. + .. note:: + + For performance, callbacks scheduled with :meth:`loop.call_later` + may run up to one clock-resolution early (see + ``time.get_clock_info('monotonic').resolution``). + .. versionchanged:: 3.7 The *context* keyword-only parameter was added. See :pep:`567` for more details. @@ -313,6 +330,12 @@ clocks to track time. An instance of :class:`asyncio.TimerHandle` is returned which can be used to cancel the callback. + .. note:: + + For performance, callbacks scheduled with :meth:`loop.call_at` + may run up to one clock-resolution early (see + ``time.get_clock_info('monotonic').resolution``). + .. versionchanged:: 3.7 The *context* keyword-only parameter was added. See :pep:`567` for more details. @@ -350,7 +373,7 @@ Creating Futures and Tasks .. versionadded:: 3.5.2 -.. method:: loop.create_task(coro, *, name=None, context=None) +.. method:: loop.create_task(coro, *, name=None, context=None, eager_start=None, **kwargs) Schedule the execution of :ref:`coroutine ` *coro*. Return a :class:`Task` object. @@ -359,6 +382,10 @@ Creating Futures and Tasks for interoperability. In this case, the result type is a subclass of :class:`Task`. + The full function signature is largely the same as that of the + :class:`Task` constructor (or factory) - all of the keyword arguments to + this function are passed through to that interface. + If the *name* argument is provided and not ``None``, it is set as the name of the task using :meth:`Task.set_name`. @@ -366,12 +393,27 @@ Creating Futures and Tasks custom :class:`contextvars.Context` for the *coro* to run in. The current context copy is created when no *context* is provided. + An optional keyword-only *eager_start* argument allows specifying + if the task should execute eagerly during the call to create_task, + or be scheduled later. If *eager_start* is not passed the mode set + by :meth:`loop.set_task_factory` will be used. + .. versionchanged:: 3.8 Added the *name* parameter. .. versionchanged:: 3.11 Added the *context* parameter. + .. versionchanged:: 3.13.3 + Added ``kwargs`` which passes on arbitrary extra parameters, including ``name`` and ``context``. + + .. versionchanged:: 3.13.4 + Rolled back the change that passes on *name* and *context* (if it is None), + while still passing on other arbitrary keyword arguments (to avoid breaking backwards compatibility with 3.13.3). + + .. versionchanged:: 3.14 + All *kwargs* are now passed on. The *eager_start* parameter works with eager task factories. + .. method:: loop.set_task_factory(factory) Set a task factory that will be used by @@ -379,9 +421,19 @@ Creating Futures and Tasks If *factory* is ``None`` the default task factory will be set. Otherwise, *factory* must be a *callable* with the signature matching - ``(loop, coro, context=None)``, where *loop* is a reference to the active + ``(loop, coro, **kwargs)``, where *loop* is a reference to the active event loop, and *coro* is a coroutine object. The callable - must return a :class:`asyncio.Future`-compatible object. + must pass on all *kwargs*, and return a :class:`asyncio.Task`-compatible object. + + .. versionchanged:: 3.13.3 + Required that all *kwargs* are passed on to :class:`asyncio.Task`. + + .. versionchanged:: 3.13.4 + *name* is no longer passed to task factories. *context* is no longer passed + to task factories if it is ``None``. + + .. versionchanged:: 3.14 + *name* and *context* are now unconditionally passed on to task factories again. .. method:: loop.get_task_factory() @@ -391,14 +443,15 @@ Creating Futures and Tasks Opening network connections ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. coroutinemethod:: loop.create_connection(protocol_factory, \ - host=None, port=None, *, ssl=None, \ - family=0, proto=0, flags=0, sock=None, \ - local_addr=None, server_hostname=None, \ - ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, \ - happy_eyeballs_delay=None, interleave=None, \ - all_errors=False) +.. method:: loop.create_connection(protocol_factory, \ + host=None, port=None, *, ssl=None, \ + family=0, proto=0, flags=0, sock=None, \ + local_addr=None, server_hostname=None, \ + ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, \ + happy_eyeballs_delay=None, interleave=None, \ + all_errors=False) + :async: Open a streaming transport connection to a given address specified by *host* and *port*. @@ -544,11 +597,12 @@ Opening network connections API. It returns a pair of (:class:`StreamReader`, :class:`StreamWriter`) that can be used directly in async/await code. -.. coroutinemethod:: loop.create_datagram_endpoint(protocol_factory, \ - local_addr=None, remote_addr=None, *, \ - family=0, proto=0, flags=0, \ - reuse_port=None, \ - allow_broadcast=None, sock=None) +.. method:: loop.create_datagram_endpoint(protocol_factory, \ + local_addr=None, remote_addr=None, *, \ + family=0, proto=0, flags=0, \ + reuse_port=None, \ + allow_broadcast=None, sock=None) + :async: Create a datagram connection. @@ -569,6 +623,12 @@ Opening network connections to bind the socket locally. The *local_host* and *local_port* are looked up using :meth:`getaddrinfo`. + .. note:: + + On Windows, when using the proactor event loop with ``local_addr=None``, + an :exc:`OSError` with :attr:`!errno.WSAEINVAL` will be raised + when running it. + * *remote_addr*, if given, is a ``(remote_host, remote_port)`` tuple used to connect the socket to a remote address. The *remote_host* and *remote_port* are looked up using :meth:`getaddrinfo`. @@ -629,10 +689,11 @@ Opening network connections The *reuse_address* parameter, disabled since Python 3.8.1, 3.7.6 and 3.6.10, has been entirely removed. -.. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ - path=None, *, ssl=None, sock=None, \ - server_hostname=None, ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None) +.. method:: loop.create_unix_connection(protocol_factory, \ + path=None, *, ssl=None, sock=None, \ + server_hostname=None, ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None) + :async: Create a Unix connection. @@ -665,16 +726,17 @@ Creating network servers .. _loop_create_server: -.. coroutinemethod:: loop.create_server(protocol_factory, \ - host=None, port=None, *, \ - family=socket.AF_UNSPEC, \ - flags=socket.AI_PASSIVE, \ - sock=None, backlog=100, ssl=None, \ - reuse_address=None, reuse_port=None, \ - keep_alive=None, \ - ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, \ - start_serving=True) +.. method:: loop.create_server(protocol_factory, \ + host=None, port=None, *, \ + family=socket.AF_UNSPEC, \ + flags=socket.AI_PASSIVE, \ + sock=None, backlog=100, ssl=None, \ + reuse_address=None, reuse_port=None, \ + keep_alive=None, \ + ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, \ + start_serving=True) + :async: Create a TCP server (socket type :const:`~socket.SOCK_STREAM`) listening on *port* of the *host* address. @@ -782,11 +844,12 @@ Creating network servers that can be used in an async/await code. -.. coroutinemethod:: loop.create_unix_server(protocol_factory, path=None, \ - *, sock=None, backlog=100, ssl=None, \ - ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, \ - start_serving=True, cleanup_socket=True) +.. method:: loop.create_unix_server(protocol_factory, path=None, \ + *, sock=None, backlog=100, ssl=None, \ + ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, \ + start_serving=True, cleanup_socket=True) + :async: Similar to :meth:`loop.create_server` but works with the :py:const:`~socket.AF_UNIX` socket family. @@ -819,9 +882,10 @@ Creating network servers Added the *cleanup_socket* parameter. -.. coroutinemethod:: loop.connect_accepted_socket(protocol_factory, \ - sock, *, ssl=None, ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None) +.. method:: loop.connect_accepted_socket(protocol_factory, \ + sock, *, ssl=None, ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None) + :async: Wrap an already accepted connection into a transport/protocol pair. @@ -869,8 +933,9 @@ Creating network servers Transferring files ^^^^^^^^^^^^^^^^^^ -.. coroutinemethod:: loop.sendfile(transport, file, \ - offset=0, count=None, *, fallback=True) +.. method:: loop.sendfile(transport, file, \ + offset=0, count=None, *, fallback=True) + :async: Send a *file* over a *transport*. Return the total number of bytes sent. @@ -899,10 +964,11 @@ Transferring files TLS Upgrade ^^^^^^^^^^^ -.. coroutinemethod:: loop.start_tls(transport, protocol, \ - sslcontext, *, server_side=False, \ - server_hostname=None, ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None) +.. method:: loop.start_tls(transport, protocol, \ + sslcontext, *, server_side=False, \ + server_hostname=None, ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None) + :async: Upgrade an existing transport-based connection to TLS. @@ -957,6 +1023,9 @@ Watching file descriptors invoke *callback* with the specified arguments once *fd* is available for reading. + Any preexisting callback registered for *fd* is cancelled and replaced by + *callback*. + .. method:: loop.remove_reader(fd) Stop monitoring the *fd* file descriptor for read availability. Returns @@ -968,6 +1037,9 @@ Watching file descriptors invoke *callback* with the specified arguments once *fd* is available for writing. + Any preexisting callback registered for *fd* is cancelled and replaced by + *callback*. + Use :func:`functools.partial` :ref:`to pass keyword arguments ` to *callback*. @@ -990,7 +1062,8 @@ However, there are some use cases when performance is not critical, and working with :class:`~socket.socket` objects directly is more convenient. -.. coroutinemethod:: loop.sock_recv(sock, nbytes) +.. method:: loop.sock_recv(sock, nbytes) + :async: Receive up to *nbytes* from *sock*. Asynchronous version of :meth:`socket.recv() `. @@ -1004,7 +1077,8 @@ convenient. method, releases before Python 3.7 returned a :class:`Future`. Since Python 3.7 this is an ``async def`` method. -.. coroutinemethod:: loop.sock_recv_into(sock, buf) +.. method:: loop.sock_recv_into(sock, buf) + :async: Receive data from *sock* into the *buf* buffer. Modeled after the blocking :meth:`socket.recv_into() ` method. @@ -1015,7 +1089,8 @@ convenient. .. versionadded:: 3.7 -.. coroutinemethod:: loop.sock_recvfrom(sock, bufsize) +.. method:: loop.sock_recvfrom(sock, bufsize) + :async: Receive a datagram of up to *bufsize* from *sock*. Asynchronous version of :meth:`socket.recvfrom() `. @@ -1026,7 +1101,8 @@ convenient. .. versionadded:: 3.11 -.. coroutinemethod:: loop.sock_recvfrom_into(sock, buf, nbytes=0) +.. method:: loop.sock_recvfrom_into(sock, buf, nbytes=0) + :async: Receive a datagram of up to *nbytes* from *sock* into *buf*. Asynchronous version of @@ -1038,7 +1114,8 @@ convenient. .. versionadded:: 3.11 -.. coroutinemethod:: loop.sock_sendall(sock, data) +.. method:: loop.sock_sendall(sock, data) + :async: Send *data* to the *sock* socket. Asynchronous version of :meth:`socket.sendall() `. @@ -1056,7 +1133,8 @@ convenient. method, before Python 3.7 it returned a :class:`Future`. Since Python 3.7, this is an ``async def`` method. -.. coroutinemethod:: loop.sock_sendto(sock, data, address) +.. method:: loop.sock_sendto(sock, data, address) + :async: Send a datagram from *sock* to *address*. Asynchronous version of @@ -1068,7 +1146,8 @@ convenient. .. versionadded:: 3.11 -.. coroutinemethod:: loop.sock_connect(sock, address) +.. method:: loop.sock_connect(sock, address) + :async: Connect *sock* to a remote socket at *address*. @@ -1089,7 +1168,8 @@ convenient. and :func:`asyncio.open_connection() `. -.. coroutinemethod:: loop.sock_accept(sock) +.. method:: loop.sock_accept(sock) + :async: Accept a connection. Modeled after the blocking :meth:`socket.accept() ` method. @@ -1111,8 +1191,9 @@ convenient. :meth:`loop.create_server` and :func:`start_server`. -.. coroutinemethod:: loop.sock_sendfile(sock, file, offset=0, count=None, \ - *, fallback=True) +.. method:: loop.sock_sendfile(sock, file, offset=0, count=None, \ + *, fallback=True) + :async: Send a file using high-performance :mod:`os.sendfile` if possible. Return the total number of bytes sent. @@ -1146,12 +1227,14 @@ convenient. DNS ^^^ -.. coroutinemethod:: loop.getaddrinfo(host, port, *, family=0, \ - type=0, proto=0, flags=0) +.. method:: loop.getaddrinfo(host, port, *, family=0, \ + type=0, proto=0, flags=0) + :async: Asynchronous version of :meth:`socket.getaddrinfo`. -.. coroutinemethod:: loop.getnameinfo(sockaddr, flags=0) +.. method:: loop.getnameinfo(sockaddr, flags=0) + :async: Asynchronous version of :meth:`socket.getnameinfo`. @@ -1173,7 +1256,8 @@ DNS Working with pipes ^^^^^^^^^^^^^^^^^^ -.. coroutinemethod:: loop.connect_read_pipe(protocol_factory, pipe) +.. method:: loop.connect_read_pipe(protocol_factory, pipe) + :async: Register the read end of *pipe* in the event loop. @@ -1189,7 +1273,8 @@ Working with pipes With :class:`SelectorEventLoop` event loop, the *pipe* is set to non-blocking mode. -.. coroutinemethod:: loop.connect_write_pipe(protocol_factory, pipe) +.. method:: loop.connect_write_pipe(protocol_factory, pipe) + :async: Register the write end of *pipe* in the event loop. @@ -1402,6 +1487,8 @@ Allows customizing how exceptions are handled in the event loop. * 'protocol' (optional): :ref:`Protocol ` instance; * 'transport' (optional): :ref:`Transport ` instance; * 'socket' (optional): :class:`socket.socket` instance; + * 'source_traceback' (optional): Traceback of the source; + * 'handle_traceback' (optional): Traceback of the handle; * 'asyncgen' (optional): Asynchronous generator that caused the exception. @@ -1461,9 +1548,10 @@ async/await code consider using the high-level .. _loop_subprocess_exec: -.. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \ - stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ - stderr=subprocess.PIPE, **kwargs) +.. method:: loop.subprocess_exec(protocol_factory, *args, \ + stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, **kwargs) + :async: Create a subprocess from one or more string arguments specified by *args*. @@ -1543,9 +1631,13 @@ async/await code consider using the high-level conforms to the :class:`asyncio.SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. -.. coroutinemethod:: loop.subprocess_shell(protocol_factory, cmd, *, \ - stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ - stderr=subprocess.PIPE, **kwargs) + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + +.. method:: loop.subprocess_shell(protocol_factory, cmd, *, \ + stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE, **kwargs) + :async: Create a subprocess from *cmd*, which can be a :class:`str` or a :class:`bytes` string encoded to the @@ -1565,6 +1657,9 @@ async/await code consider using the high-level conforms to the :class:`SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. note:: It is the application's responsibility to ensure that all whitespace and special characters are quoted appropriately to avoid `shell injection @@ -1690,7 +1785,8 @@ Do not instantiate the :class:`Server` class directly. .. versionadded:: 3.7 - .. coroutinemethod:: start_serving() + .. method:: start_serving() + :async: Start accepting connections. @@ -1706,7 +1802,8 @@ Do not instantiate the :class:`Server` class directly. .. versionadded:: 3.7 - .. coroutinemethod:: serve_forever() + .. method:: serve_forever() + :async: Start accepting connections until the coroutine is cancelled. Cancellation of ``serve_forever`` task causes the server @@ -1738,7 +1835,8 @@ Do not instantiate the :class:`Server` class directly. .. versionadded:: 3.7 - .. coroutinemethod:: wait_closed() + .. method:: wait_closed() + :async: Wait until the :meth:`close` method completes and all active connections have finished. @@ -1778,12 +1876,11 @@ By default asyncio is configured to use :class:`EventLoop`. import asyncio import selectors - class MyPolicy(asyncio.DefaultEventLoopPolicy): - def new_event_loop(self): - selector = selectors.SelectSelector() - return asyncio.SelectorEventLoop(selector) + async def main(): + ... - asyncio.set_event_loop_policy(MyPolicy()) + loop_factory = lambda: asyncio.SelectorEventLoop(selectors.SelectSelector()) + asyncio.run(main(), loop_factory=loop_factory) .. availability:: Unix, Windows. @@ -1798,7 +1895,7 @@ By default asyncio is configured to use :class:`EventLoop`. .. seealso:: `MSDN documentation on I/O Completion Ports - `_. + `_. .. class:: EventLoop diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 9dce0731411940..4b69e569523c58 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -51,12 +51,13 @@ Future Functions .. important:: - See also the :func:`create_task` function which is the - preferred way for creating new Tasks. - Save a reference to the result of this function, to avoid a task disappearing mid-execution. + See also the :func:`create_task` function which is the + preferred way for creating new tasks or use :class:`asyncio.TaskGroup` + which keeps reference to the task internally. + .. versionchanged:: 3.5.1 The function accepts any :term:`awaitable` object. @@ -74,6 +75,7 @@ Future Functions Deprecation warning is emitted if *future* is not a Future-like object and *loop* is not specified and there is no running event loop. +.. _asyncio-future-obj: Future Object ============= diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst new file mode 100644 index 00000000000000..5f642a32bf75c2 --- /dev/null +++ b/Doc/library/asyncio-graph.rst @@ -0,0 +1,145 @@ +.. currentmodule:: asyncio + + +.. _asyncio-graph: + +======================== +Call Graph Introspection +======================== + +**Source code:** :source:`Lib/asyncio/graph.py` + +------------------------------------- + +asyncio has powerful runtime call graph introspection utilities +to trace the entire call graph of a running *coroutine* or *task*, or +a suspended *future*. These utilities and the underlying machinery +can be used from within a Python program or by external profilers +and debuggers. + +.. versionadded:: 3.14 + + +.. function:: print_call_graph(future=None, /, *, file=None, depth=1, limit=None) + + Print the async call graph for the current task or the provided + :class:`Task` or :class:`Future`. + + This function prints entries starting from the top frame and going + down towards the invocation point. + + The function receives an optional *future* argument. + If not passed, the current running task will be used. + + If the function is called on *the current task*, the optional + keyword-only *depth* argument can be used to skip the specified + number of frames from top of the stack. + + If the optional keyword-only *limit* argument is provided, each call stack + in the resulting graph is truncated to include at most ``abs(limit)`` + entries. If *limit* is positive, the entries left are the closest to + the invocation point. If *limit* is negative, the topmost entries are + left. If *limit* is omitted or ``None``, all entries are present. + If *limit* is ``0``, the call stack is not printed at all, only + "awaited by" information is printed. + + If *file* is omitted or ``None``, the function will print + to :data:`sys.stdout`. + + **Example:** + + The following Python code: + + .. code-block:: python + + import asyncio + + async def test(): + asyncio.print_call_graph() + + async def main(): + async with asyncio.TaskGroup() as g: + g.create_task(test(), name='test') + + asyncio.run(main()) + + will print:: + + * Task(name='test', id=0x1039f0fe0) + + Call stack: + | File 't2.py', line 4, in async test() + + Awaited by: + * Task(name='Task-1', id=0x103a5e060) + + Call stack: + | File 'taskgroups.py', line 107, in async TaskGroup.__aexit__() + | File 't2.py', line 7, in async main() + +.. function:: format_call_graph(future=None, /, *, depth=1, limit=None) + + Like :func:`print_call_graph`, but returns a string. + If *future* is ``None`` and there's no current task, + the function returns an empty string. + + +.. function:: capture_call_graph(future=None, /, *, depth=1, limit=None) + + Capture the async call graph for the current task or the provided + :class:`Task` or :class:`Future`. + + The function receives an optional *future* argument. + If not passed, the current running task will be used. If there's no + current task, the function returns ``None``. + + If the function is called on *the current task*, the optional + keyword-only *depth* argument can be used to skip the specified + number of frames from top of the stack. + + Returns a ``FutureCallGraph`` data class object: + + * ``FutureCallGraph(future, call_stack, awaited_by)`` + + Where *future* is a reference to a :class:`Future` or + a :class:`Task` (or their subclasses.) + + ``call_stack`` is a tuple of ``FrameCallGraphEntry`` objects. + + ``awaited_by`` is a tuple of ``FutureCallGraph`` objects. + + * ``FrameCallGraphEntry(frame)`` + + Where *frame* is a frame object of a regular Python function + in the call stack. + + +Low level utility functions +=========================== + +To introspect an async call graph asyncio requires cooperation from +control flow structures, such as :func:`shield` or :class:`TaskGroup`. +Any time an intermediate :class:`Future` object with low-level APIs like +:meth:`Future.add_done_callback() ` is +involved, the following two functions should be used to inform asyncio +about how exactly such intermediate future objects are connected with +the tasks they wrap or control. + + +.. function:: future_add_to_awaited_by(future, waiter, /) + + Record that *future* is awaited on by *waiter*. + + Both *future* and *waiter* must be instances of + :class:`Future` or :class:`Task` or their subclasses, + otherwise the call would have no effect. + + A call to ``future_add_to_awaited_by()`` must be followed by an + eventual call to the :func:`future_discard_from_awaited_by` function + with the same arguments. + + +.. function:: future_discard_from_awaited_by(future, waiter, /) + + Record that *future* is no longer awaited on by *waiter*. + + Both *future* and *waiter* must be instances of + :class:`Future` or :class:`Task` or their subclasses, otherwise + the call would have no effect. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 837ccc6606786e..57f964912dd6ea 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -7,6 +7,14 @@ Policies ======== +.. warning:: + + Policies are deprecated and will be removed in Python 3.16. + Users are encouraged to use the :func:`asyncio.run` function + or the :class:`asyncio.Runner` with *loop_factory* to use + the desired loop implementation. + + An event loop policy is a global object used to get and set the current :ref:`event loop `, as well as create new event loops. @@ -40,12 +48,20 @@ for the current process: Return the current process-wide policy. + .. deprecated:: 3.14 + The :func:`get_event_loop_policy` function is deprecated and + will be removed in Python 3.16. + .. function:: set_event_loop_policy(policy) Set the current process-wide policy to *policy*. If *policy* is set to ``None``, the default policy is restored. + .. deprecated:: 3.14 + The :func:`set_event_loop_policy` function is deprecated and + will be removed in Python 3.16. + .. _asyncio-policy-objects: @@ -79,6 +95,10 @@ The abstract event loop policy base class is defined as follows: This method should never return ``None``. + .. deprecated:: 3.14 + The :class:`AbstractEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. + .. _asyncio-policy-builtin: @@ -97,11 +117,13 @@ asyncio ships with the following built-in policies: On Windows, :class:`ProactorEventLoop` is now used by default. - .. deprecated:: 3.12 - The :meth:`get_event_loop` method of the default asyncio policy now emits - a :exc:`DeprecationWarning` if there is no current event loop set and it - decides to create one. - In some future Python release this will become an error. + .. versionchanged:: 3.14 + The :meth:`get_event_loop` method of the default asyncio policy now + raises a :exc:`RuntimeError` if there is no set event loop. + + .. deprecated:: 3.14 + The :class:`DefaultEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. .. class:: WindowsSelectorEventLoopPolicy @@ -111,6 +133,10 @@ asyncio ships with the following built-in policies: .. availability:: Windows. + .. deprecated:: 3.14 + The :class:`WindowsSelectorEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. + .. class:: WindowsProactorEventLoopPolicy @@ -119,6 +145,10 @@ asyncio ships with the following built-in policies: .. availability:: Windows. + .. deprecated:: 3.14 + The :class:`WindowsProactorEventLoopPolicy` class is deprecated and + will be removed in Python 3.16. + .. _asyncio-custom-policies: diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 7c08d65f26bc27..5208f14c94a50f 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -390,11 +390,11 @@ Subprocess Transports Return the transport for the communication pipe corresponding to the integer file descriptor *fd*: - * ``0``: readable streaming transport of the standard input (*stdin*), + * ``0``: writable streaming transport of the standard input (*stdin*), or :const:`None` if the subprocess was not created with ``stdin=PIPE`` - * ``1``: writable streaming transport of the standard output (*stdout*), + * ``1``: readable streaming transport of the standard output (*stdout*), or :const:`None` if the subprocess was not created with ``stdout=PIPE`` - * ``2``: writable streaming transport of the standard error (*stderr*), + * ``2``: readable streaming transport of the standard error (*stderr*), or :const:`None` if the subprocess was not created with ``stderr=PIPE`` * other *fd*: :const:`None` diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index 61991bf2f4ed1d..a9735ae80652df 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -57,7 +57,8 @@ Queue If the queue was initialized with ``maxsize=0`` (the default), then :meth:`full` never returns ``True``. - .. coroutinemethod:: get() + .. method:: get() + :async: Remove and return an item from the queue. If queue is empty, wait until an item is available. @@ -70,7 +71,8 @@ Queue Return an item if one is immediately available, else raise :exc:`QueueEmpty`. - .. coroutinemethod:: join() + .. method:: join() + :async: Block until all items in the queue have been received and processed. @@ -80,7 +82,8 @@ Queue work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. - .. coroutinemethod:: put(item) + .. method:: put(item) + :async: Put an item into the queue. If the queue is full, wait until a free slot is available before adding the item. @@ -99,36 +102,50 @@ Queue .. method:: shutdown(immediate=False) - Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` + Put a :class:`Queue` instance into a shutdown mode. + + The queue can no longer grow. + Future calls to :meth:`~Queue.put` raise :exc:`QueueShutDown`. + Currently blocked callers of :meth:`~Queue.put` will be unblocked + and will raise :exc:`QueueShutDown` in the formerly awaiting task. + + If *immediate* is false (the default), the queue can be wound + down normally with :meth:`~Queue.get` calls to extract tasks + that have already been loaded. + + And if :meth:`~Queue.task_done` is called for each remaining task, a + pending :meth:`~Queue.join` will be unblocked normally. + + Once the queue is empty, future calls to :meth:`~Queue.get` will raise :exc:`QueueShutDown`. - By default, :meth:`~Queue.get` on a shut down queue will only - raise once the queue is empty. Set *immediate* to true to make - :meth:`~Queue.get` raise immediately instead. + If *immediate* is true, the queue is terminated immediately. + The queue is drained to be completely empty and the count + of unfinished tasks is reduced by the number of tasks drained. + If unfinished tasks is zero, callers of :meth:`~Queue.join` + are unblocked. Also, blocked callers of :meth:`~Queue.get` + are unblocked and will raise :exc:`QueueShutDown` because the + queue is empty. - All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` - will be unblocked. If *immediate* is true, a task will be marked - as done for each remaining item in the queue, which may unblock - callers of :meth:`~Queue.join`. + Use caution when using :meth:`~Queue.join` with *immediate* set + to true. This unblocks the join even when no work has been done + on the tasks, violating the usual invariant for joining a queue. .. versionadded:: 3.13 .. method:: task_done() - Indicate that a formerly enqueued task is complete. + Indicate that a formerly enqueued work item is complete. Used by queue consumers. For each :meth:`~Queue.get` used to - fetch a task, a subsequent call to :meth:`task_done` tells the - queue that the processing on the task is complete. + fetch a work item, a subsequent call to :meth:`task_done` tells the + queue that the processing on the work item is complete. If a :meth:`join` is currently blocking, it will resume when all items have been processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`~Queue.put` into the queue). - ``shutdown(immediate=True)`` calls :meth:`task_done` for each - remaining item in the queue. - Raises :exc:`ValueError` if called more times than there were items placed in the queue. diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index 28d5aaf3692baa..48d78099fd3ce7 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -76,6 +76,12 @@ Running an asyncio Program *coro* can be any awaitable object. + .. note:: + + The :mod:`!asyncio` policy system is deprecated and will be removed + in Python 3.16; from there on, an explicit *loop_factory* is needed + to configure the event loop. + Runner context manager ====================== diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 3fdc79b3c6896c..05445219510ca5 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -48,12 +48,13 @@ The following top-level asyncio functions can be used to create and work with streams: -.. coroutinefunction:: open_connection(host=None, port=None, *, \ - limit=None, ssl=None, family=0, proto=0, \ - flags=0, sock=None, local_addr=None, \ - server_hostname=None, ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, \ - happy_eyeballs_delay=None, interleave=None) +.. function:: open_connection(host=None, port=None, *, \ + limit=None, ssl=None, family=0, proto=0, \ + flags=0, sock=None, local_addr=None, \ + server_hostname=None, ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, \ + happy_eyeballs_delay=None, interleave=None) + :async: Establish a network connection and return a pair of ``(reader, writer)`` objects. @@ -87,13 +88,15 @@ and work with streams: Added the *ssl_shutdown_timeout* parameter. -.. coroutinefunction:: start_server(client_connected_cb, host=None, \ - port=None, *, limit=None, \ - family=socket.AF_UNSPEC, \ - flags=socket.AI_PASSIVE, sock=None, \ - backlog=100, ssl=None, reuse_address=None, \ - reuse_port=None, ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, start_serving=True) +.. function:: start_server(client_connected_cb, host=None, \ + port=None, *, limit=None, \ + family=socket.AF_UNSPEC, \ + flags=socket.AI_PASSIVE, sock=None, \ + backlog=100, ssl=None, reuse_address=None, \ + reuse_port=None, keep_alive=None, \ + ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, start_serving=True) + :async: Start a socket server. @@ -128,12 +131,16 @@ and work with streams: .. versionchanged:: 3.11 Added the *ssl_shutdown_timeout* parameter. + .. versionchanged:: 3.13 + Added the *keep_alive* parameter. + .. rubric:: Unix Sockets -.. coroutinefunction:: open_unix_connection(path=None, *, limit=None, \ - ssl=None, sock=None, server_hostname=None, \ - ssl_handshake_timeout=None, ssl_shutdown_timeout=None) +.. function:: open_unix_connection(path=None, *, limit=None, \ + ssl=None, sock=None, server_hostname=None, \ + ssl_handshake_timeout=None, ssl_shutdown_timeout=None) + :async: Establish a Unix socket connection and return a pair of ``(reader, writer)``. @@ -161,15 +168,20 @@ and work with streams: Added the *ssl_shutdown_timeout* parameter. -.. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ - *, limit=None, sock=None, backlog=100, ssl=None, \ - ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, start_serving=True) +.. function:: start_unix_server(client_connected_cb, path=None, \ + *, limit=None, sock=None, backlog=100, ssl=None, \ + ssl_handshake_timeout=None, \ + ssl_shutdown_timeout=None, start_serving=True, cleanup_socket=True) + :async: Start a Unix socket server. Similar to :func:`start_server` but works with Unix sockets. + If *cleanup_socket* is true then the Unix socket will automatically + be removed from the filesystem when the server is closed, unless the + socket has been replaced after the server has been created. + See also the documentation of :meth:`loop.create_unix_server`. .. note:: @@ -190,6 +202,9 @@ and work with streams: .. versionchanged:: 3.11 Added the *ssl_shutdown_timeout* parameter. + .. versionchanged:: 3.13 + Added the *cleanup_socket* parameter. + StreamReader ============ @@ -208,7 +223,8 @@ StreamReader Acknowledge the EOF. - .. coroutinemethod:: read(n=-1) + .. method:: read(n=-1) + :async: Read up to *n* bytes from the stream. @@ -224,7 +240,8 @@ StreamReader If EOF is received before any byte is read, return an empty ``bytes`` object. - .. coroutinemethod:: readline() + .. method:: readline() + :async: Read one line, where "line" is a sequence of bytes ending with ``\n``. @@ -235,7 +252,8 @@ StreamReader If EOF is received and the internal buffer is empty, return an empty ``bytes`` object. - .. coroutinemethod:: readexactly(n) + .. method:: readexactly(n) + :async: Read exactly *n* bytes. @@ -243,7 +261,8 @@ StreamReader can be read. Use the :attr:`IncompleteReadError.partial` attribute to get the partially read data. - .. coroutinemethod:: readuntil(separator=b'\n') + .. method:: readuntil(separator=b'\n') + :async: Read data from the stream until *separator* is found. @@ -297,11 +316,15 @@ StreamWriter If that fails, the data is queued in an internal write buffer until it can be sent. + The *data* buffer should be a bytes, bytearray, or C-contiguous one-dimensional + memoryview object. + The method should be used along with the ``drain()`` method:: stream.write(data) await stream.drain() + .. method:: writelines(data) The method writes a list (or any iterable) of bytes to the underlying socket @@ -343,7 +366,8 @@ StreamWriter Access optional transport information; see :meth:`BaseTransport.get_extra_info` for details. - .. coroutinemethod:: drain() + .. method:: drain() + :async: Wait until it is appropriate to resume writing to the stream. Example:: @@ -358,8 +382,9 @@ StreamWriter be resumed. When there is nothing to wait for, the :meth:`drain` returns immediately. - .. coroutinemethod:: start_tls(sslcontext, *, server_hostname=None, \ - ssl_handshake_timeout=None, ssl_shutdown_timeout=None) + .. method:: start_tls(sslcontext, *, server_hostname=None, \ + ssl_handshake_timeout=None, ssl_shutdown_timeout=None) + :async: Upgrade an existing stream-based connection to TLS. @@ -391,7 +416,8 @@ StreamWriter .. versionadded:: 3.7 - .. coroutinemethod:: wait_closed() + .. method:: wait_closed() + :async: Wait until the stream is closed. diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index b477a7fa2d37ef..9416c758e51d95 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -61,13 +61,14 @@ See also the `Examples`_ subsection. Creating Subprocesses ===================== -.. coroutinefunction:: create_subprocess_exec(program, *args, stdin=None, \ - stdout=None, stderr=None, limit=None, **kwds) +.. function:: create_subprocess_exec(program, *args, stdin=None, \ + stdout=None, stderr=None, limit=None, **kwds) + :async: Create a subprocess. The *limit* argument sets the buffer limit for :class:`StreamReader` - wrappers for :attr:`Process.stdout` and :attr:`Process.stderr` + wrappers for :attr:`~asyncio.subprocess.Process.stdout` and :attr:`~asyncio.subprocess.Process.stderr` (if :const:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments). Return a :class:`~asyncio.subprocess.Process` instance. @@ -75,17 +76,21 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_exec` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. versionchanged:: 3.10 Removed the *loop* parameter. -.. coroutinefunction:: create_subprocess_shell(cmd, stdin=None, \ - stdout=None, stderr=None, limit=None, **kwds) +.. function:: create_subprocess_shell(cmd, stdin=None, \ + stdout=None, stderr=None, limit=None, **kwds) + :async: Run the *cmd* shell command. The *limit* argument sets the buffer limit for :class:`StreamReader` - wrappers for :attr:`Process.stdout` and :attr:`Process.stderr` + wrappers for :attr:`~asyncio.subprocess.Process.stdout` and :attr:`~asyncio.subprocess.Process.stderr` (if :const:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments). Return a :class:`~asyncio.subprocess.Process` instance. @@ -93,6 +98,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_shell` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. important:: It is the application's responsibility to ensure that all whitespace and @@ -130,12 +138,12 @@ Constants If *PIPE* is passed to *stdin* argument, the :attr:`Process.stdin ` attribute - will point to a :class:`StreamWriter` instance. + will point to a :class:`~asyncio.StreamWriter` instance. If *PIPE* is passed to *stdout* or *stderr* arguments, the :attr:`Process.stdout ` and :attr:`Process.stderr ` - attributes will point to :class:`StreamReader` instances. + attributes will point to :class:`~asyncio.StreamReader` instances. .. data:: asyncio.subprocess.STDOUT :module: @@ -163,7 +171,7 @@ their completion. :module: An object that wraps OS processes created by the - :func:`create_subprocess_exec` and :func:`create_subprocess_shell` + :func:`~asyncio.create_subprocess_exec` and :func:`~asyncio.create_subprocess_shell` functions. This class is designed to have a similar API to the @@ -188,7 +196,8 @@ their completion. See also the :ref:`Subprocess and Threads ` section. - .. coroutinemethod:: wait() + .. method:: wait() + :async: Wait for the child process to terminate. @@ -202,7 +211,8 @@ their completion. more data. Use the :meth:`communicate` method when using pipes to avoid this condition. - .. coroutinemethod:: communicate(input=None) + .. method:: communicate(input=None) + :async: Interact with process: @@ -232,7 +242,7 @@ their completion. .. versionchanged:: 3.12 - *stdin* gets closed when `input=None` too. + *stdin* gets closed when ``input=None`` too. .. method:: send_signal(signal) @@ -259,24 +269,24 @@ their completion. Kill the child process. - On POSIX systems this method sends :py:data:`SIGKILL` to the child + On POSIX systems this method sends :py:data:`~signal.SIGKILL` to the child process. On Windows this method is an alias for :meth:`terminate`. .. attribute:: stdin - Standard input stream (:class:`StreamWriter`) or ``None`` + Standard input stream (:class:`~asyncio.StreamWriter`) or ``None`` if the process was created with ``stdin=None``. .. attribute:: stdout - Standard output stream (:class:`StreamReader`) or ``None`` + Standard output stream (:class:`~asyncio.StreamReader`) or ``None`` if the process was created with ``stdout=None``. .. attribute:: stderr - Standard error stream (:class:`StreamReader`) or ``None`` + Standard error stream (:class:`~asyncio.StreamReader`) or ``None`` if the process was created with ``stderr=None``. .. warning:: @@ -292,7 +302,7 @@ their completion. Process identification number (PID). - Note that for processes created by the :func:`create_subprocess_shell` + Note that for processes created by the :func:`~asyncio.create_subprocess_shell` function, this attribute is the PID of the spawned shell. .. attribute:: returncode diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 3cf8e2737e85dc..f9e98e05cab7ac 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -67,7 +67,8 @@ Lock .. versionchanged:: 3.10 Removed the *loop* parameter. - .. coroutinemethod:: acquire() + .. method:: acquire() + :async: Acquire the lock. @@ -137,7 +138,8 @@ Event asyncio.run(main()) - .. coroutinemethod:: wait() + .. method:: wait() + :async: Wait until the event is set. @@ -155,7 +157,7 @@ Event Clear (unset) the event. - Tasks awaiting on :meth:`~Event.wait` will now block until the + Subsequent tasks awaiting on :meth:`~Event.wait` will now block until the :meth:`~Event.set` method is called again. .. method:: is_set() @@ -207,7 +209,8 @@ Condition finally: cond.release() - .. coroutinemethod:: acquire() + .. method:: acquire() + :async: Acquire the underlying lock. @@ -245,7 +248,8 @@ Condition When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. - .. coroutinemethod:: wait() + .. method:: wait() + :async: Wait until notified. @@ -259,16 +263,17 @@ Condition Note that a task *may* return from this call spuriously, which is why the caller should always re-check the state - and be prepared to :meth:`wait` again. For this reason, you may - prefer to use :meth:`wait_for` instead. + and be prepared to :meth:`~Condition.wait` again. For this reason, you may + prefer to use :meth:`~Condition.wait_for` instead. - .. coroutinemethod:: wait_for(predicate) + .. method:: wait_for(predicate) + :async: Wait until a predicate becomes *true*. The predicate must be a callable which result will be interpreted as a boolean value. The method will repeatedly - :meth:`wait` until the predicate evaluates to *true*. The final value is the + :meth:`~Condition.wait` until the predicate evaluates to *true*. The final value is the return value. @@ -312,7 +317,8 @@ Semaphore finally: sem.release() - .. coroutinemethod:: acquire() + .. method:: acquire() + :async: Acquire a semaphore. @@ -400,7 +406,8 @@ Barrier .. versionadded:: 3.11 - .. coroutinemethod:: wait() + .. method:: wait() + :async: Pass the barrier. When all the tasks party to the barrier have called this function, they are all unblocked simultaneously. @@ -424,17 +431,19 @@ Barrier barrier is broken or reset while a task is waiting. It could raise a :exc:`CancelledError` if a task is cancelled. - .. coroutinemethod:: reset() + .. method:: reset() + :async: Return the barrier to the default, empty state. Any tasks waiting on it will receive the :class:`BrokenBarrierError` exception. If a barrier is broken it may be better to just leave it and create a new one. - .. coroutinemethod:: abort() + .. method:: abort() + :async: Put the barrier into a broken state. This causes any active or future - calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. + calls to :meth:`~Barrier.wait` to fail with the :class:`BrokenBarrierError`. Use this for example if one of the tasks needs to abort, to avoid infinite waiting tasks. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index f27e858cf420f4..863b3e336572aa 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -238,18 +238,24 @@ Creating Tasks ----------------------------------------------- -.. function:: create_task(coro, *, name=None, context=None) +.. function:: create_task(coro, *, name=None, context=None, eager_start=None, **kwargs) Wrap the *coro* :ref:`coroutine ` into a :class:`Task` and schedule its execution. Return the Task object. - If *name* is not ``None``, it is set as the name of the task using - :meth:`Task.set_name`. + The full function signature is largely the same as that of the + :class:`Task` constructor (or factory) - all of the keyword arguments to + this function are passed through to that interface. An optional keyword-only *context* argument allows specifying a custom :class:`contextvars.Context` for the *coro* to run in. The current context copy is created when no *context* is provided. + An optional keyword-only *eager_start* argument allows specifying + if the task should execute eagerly during the call to create_task, + or be scheduled later. If *eager_start* is not passed the mode set + by :meth:`loop.set_task_factory` will be used. + The task is executed in the loop returned by :func:`get_running_loop`, :exc:`RuntimeError` is raised if there is no running loop in current thread. @@ -290,6 +296,9 @@ Creating Tasks .. versionchanged:: 3.11 Added the *context* parameter. + .. versionchanged:: 3.14 + Added the *eager_start* parameter by passing on all *kwargs*. + Task Cancellation ================= @@ -330,7 +339,7 @@ and reliable way to wait for all tasks in the group to finish. .. versionadded:: 3.11 - .. method:: create_task(coro, *, name=None, context=None) + .. method:: create_task(coro, *, name=None, context=None, eager_start=None, **kwargs) Create a task in this task group. The signature matches that of :func:`asyncio.create_task`. @@ -342,6 +351,10 @@ and reliable way to wait for all tasks in the group to finish. Close the given coroutine if the task group is not active. + .. versionchanged:: 3.14 + + Passes on all *kwargs* to :meth:`loop.create_task` + Example:: async def main(): @@ -464,7 +477,8 @@ Expected output: Sleeping ======== -.. coroutinefunction:: sleep(delay, result=None) +.. function:: sleep(delay, result=None) + :async: Block for *delay* seconds. @@ -819,7 +833,8 @@ Timeouts .. versionadded:: 3.11 -.. coroutinefunction:: wait_for(aw, timeout) +.. function:: wait_for(aw, timeout) + :async: Wait for the *aw* :ref:`awaitable ` to complete with a timeout. @@ -879,7 +894,8 @@ Timeouts Waiting Primitives ================== -.. coroutinefunction:: wait(aws, *, timeout=None, return_when=ALL_COMPLETED) +.. function:: wait(aws, *, timeout=None, return_when=ALL_COMPLETED) + :async: Run :class:`~asyncio.Future` and :class:`~asyncio.Task` instances in the *aws* iterable concurrently and block until the condition specified @@ -998,7 +1014,8 @@ Waiting Primitives Running in Threads ================== -.. coroutinefunction:: to_thread(func, /, *args, **kwargs) +.. function:: to_thread(func, /, *args, **kwargs) + :async: Asynchronously run function *func* in a separate thread. @@ -1067,14 +1084,59 @@ Scheduling From Other Threads This function is meant to be called from a different OS thread than the one where the event loop is running. Example:: - # Create a coroutine - coro = asyncio.sleep(1, result=3) + def in_thread(loop: asyncio.AbstractEventLoop) -> None: + # Run some blocking IO + pathlib.Path("example.txt").write_text("hello world", encoding="utf8") + + # Create a coroutine + coro = asyncio.sleep(1, result=3) + + # Submit the coroutine to a given loop + future = asyncio.run_coroutine_threadsafe(coro, loop) + + # Wait for the result with an optional timeout argument + assert future.result(timeout=2) == 3 + + async def amain() -> None: + # Get the running loop + loop = asyncio.get_running_loop() - # Submit the coroutine to a given loop - future = asyncio.run_coroutine_threadsafe(coro, loop) + # Run something in a thread + await asyncio.to_thread(in_thread, loop) - # Wait for the result with an optional timeout argument - assert future.result(timeout) == 3 + It's also possible to run the other way around. Example:: + + @contextlib.contextmanager + def loop_in_thread() -> Generator[asyncio.AbstractEventLoop]: + loop_fut = concurrent.futures.Future[asyncio.AbstractEventLoop]() + stop_event = asyncio.Event() + + async def main() -> None: + loop_fut.set_result(asyncio.get_running_loop()) + await stop_event.wait() + + with concurrent.futures.ThreadPoolExecutor(1) as tpe: + complete_fut = tpe.submit(asyncio.run, main()) + for fut in concurrent.futures.as_completed((loop_fut, complete_fut)): + if fut is loop_fut: + loop = loop_fut.result() + try: + yield loop + finally: + loop.call_soon_threadsafe(stop_event.set) + else: + fut.result() + + # Create a loop in another thread + with loop_in_thread() as loop: + # Create a coroutine + coro = asyncio.sleep(1, result=3) + + # Submit the coroutine to a given loop + future = asyncio.run_coroutine_threadsafe(coro, loop) + + # Wait for the result with an optional timeout argument + assert future.result(timeout=2) == 3 If an exception is raised in the coroutine, the returned Future will be notified. It can also be used to cancel the task in @@ -1131,6 +1193,7 @@ Introspection .. versionadded:: 3.4 +.. _asyncio-task-obj: Task Object =========== @@ -1158,8 +1221,8 @@ Task Object To cancel a running Task use the :meth:`cancel` method. Calling it will cause the Task to throw a :exc:`CancelledError` exception into - the wrapped coroutine. If a coroutine is awaiting on a Future - object during cancellation, the Future object will be cancelled. + the wrapped coroutine. If a coroutine is awaiting on a future-like + object during cancellation, the awaited object will be cancelled. :meth:`cancelled` can be used to check if the Task was cancelled. The method returns ``True`` if the wrapped coroutine did not @@ -1332,7 +1395,10 @@ Task Object Request the Task to be cancelled. - This arranges for a :exc:`CancelledError` exception to be thrown + If the Task is already *done* or *cancelled*, return ``False``, + otherwise, return ``True``. + + The method arranges for a :exc:`CancelledError` exception to be thrown into the wrapped coroutine on the next cycle of the event loop. The coroutine then has a chance to clean up or even deny the @@ -1345,6 +1411,10 @@ Task Object the cancellation, it needs to call :meth:`Task.uncancel` in addition to catching the exception. + If the Task being cancelled is currently awaiting on a future-like + object, that awaited object will also be cancelled. This cancellation + propagates down the entire chain of awaited objects. + .. versionchanged:: 3.9 Added the *msg* parameter. diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 5f83b3a2658da4..0f72e31dee5f1d 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -29,6 +29,11 @@ database connection libraries, distributed task queues, etc. asyncio is often a perfect fit for IO-bound and high-level **structured** network code. +.. seealso:: + + :ref:`a-conceptual-overview-of-asyncio` + Explanation of the fundamentals of asyncio. + asyncio provides a set of **high-level** APIs to: * :ref:`run Python coroutines ` concurrently and @@ -74,6 +79,10 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: >>> await asyncio.sleep(10, result='hello') 'hello' +This REPL provides limited compatibility with :envvar:`PYTHON_BASIC_REPL`. +It is recommended that the default REPL is used +for full functionality and the latest features. + .. audit-event:: cpython.run_stdin "" "" .. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) @@ -99,6 +108,7 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: asyncio-subprocess.rst asyncio-queue.rst asyncio-exceptions.rst + asyncio-graph.rst .. toctree:: :caption: Low-level APIs diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst new file mode 100644 index 00000000000000..22c9881c3cca36 --- /dev/null +++ b/Doc/library/asyncore.rst @@ -0,0 +1,17 @@ +:mod:`!asyncore` --- Asynchronous socket handler +================================================ + +.. module:: asyncore + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.6 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.6. The removal was decided in :pep:`594`. + +Applications should use the :mod:`asyncio` module instead. + +The last version of Python that provided the :mod:`!asyncore` module was +`Python 3.11 `_. diff --git a/Doc/library/audioop.rst b/Doc/library/audioop.rst new file mode 100644 index 00000000000000..3bc580b0bd3433 --- /dev/null +++ b/Doc/library/audioop.rst @@ -0,0 +1,15 @@ +:mod:`!audioop` --- Manipulate raw audio data +============================================= + +.. module:: audioop + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!audioop` module was +`Python 3.12 `_. diff --git a/Doc/library/audit_events.rst b/Doc/library/audit_events.rst index a2a90a00d0cfca..73a58092024631 100644 --- a/Doc/library/audit_events.rst +++ b/Doc/library/audit_events.rst @@ -23,25 +23,30 @@ information on handling these events. The following events are raised internally and do not correspond to any public API of CPython: -+--------------------------+-------------------------------------------+ -| Audit event | Arguments | -+==========================+===========================================+ -| _winapi.CreateFile | ``file_name``, ``desired_access``, | -| | ``share_mode``, ``creation_disposition``, | -| | ``flags_and_attributes`` | -+--------------------------+-------------------------------------------+ -| _winapi.CreateJunction | ``src_path``, ``dst_path`` | -+--------------------------+-------------------------------------------+ -| _winapi.CreateNamedPipe | ``name``, ``open_mode``, ``pipe_mode`` | -+--------------------------+-------------------------------------------+ -| _winapi.CreatePipe | | -+--------------------------+-------------------------------------------+ -| _winapi.CreateProcess | ``application_name``, ``command_line``, | -| | ``current_directory`` | -+--------------------------+-------------------------------------------+ -| _winapi.OpenProcess | ``process_id``, ``desired_access`` | -+--------------------------+-------------------------------------------+ -| _winapi.TerminateProcess | ``handle``, ``exit_code`` | -+--------------------------+-------------------------------------------+ -| ctypes.PyObj_FromPtr | ``obj`` | -+--------------------------+-------------------------------------------+ ++----------------------------+-------------------------------------------+ +| Audit event | Arguments | ++============================+===========================================+ +| _winapi.CreateFile | ``file_name``, ``desired_access``, | +| | ``share_mode``, ``creation_disposition``, | +| | ``flags_and_attributes`` | ++----------------------------+-------------------------------------------+ +| _winapi.CreateJunction | ``src_path``, ``dst_path`` | ++----------------------------+-------------------------------------------+ +| _winapi.CreateNamedPipe | ``name``, ``open_mode``, ``pipe_mode`` | ++----------------------------+-------------------------------------------+ +| _winapi.CreatePipe | | ++----------------------------+-------------------------------------------+ +| _winapi.CreateProcess | ``application_name``, ``command_line``, | +| | ``current_directory`` | ++----------------------------+-------------------------------------------+ +| _winapi.OpenProcess | ``process_id``, ``desired_access`` | ++----------------------------+-------------------------------------------+ +| _winapi.TerminateProcess | ``handle``, ``exit_code`` | ++----------------------------+-------------------------------------------+ +| _posixsubprocess.fork_exec | ``exec_list``, ``args``, ``env`` | ++----------------------------+-------------------------------------------+ +| ctypes.PyObj_FromPtr | ``obj`` | ++----------------------------+-------------------------------------------+ + +.. versionadded:: 3.14 + The ``_posixsubprocess.fork_exec`` internal audit event. diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 834ab2536e6c14..65b8aeaef8e939 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -15,14 +15,9 @@ This module provides functions for encoding binary data to printable ASCII characters and decoding such encodings back to binary data. -It provides encoding and decoding functions for the encodings specified in -:rfc:`4648`, which defines the Base16, Base32, and Base64 algorithms, -and for the de-facto standard Ascii85 and Base85 encodings. - -The :rfc:`4648` encodings are suitable for encoding binary data so that it can be -safely sent by email, used as parts of URLs, or included as part of an HTTP -POST request. The encoding algorithm is not the same as the -:program:`uuencode` program. +This includes the :ref:`encodings specified in ` +:rfc:`4648` (Base64, Base32 and Base16) +and the non-standard :ref:`Base85 encodings `. There are two interfaces provided by this module. The modern interface supports encoding :term:`bytes-like objects ` to ASCII @@ -30,7 +25,7 @@ supports encoding :term:`bytes-like objects ` to ASCII strings containing ASCII to :class:`bytes`. Both base-64 alphabets defined in :rfc:`4648` (normal, and URL- and filesystem-safe) are supported. -The legacy interface does not support decoding from strings, but it does +The :ref:`legacy interface ` does not support decoding from strings, but it does provide functions for encoding and decoding to and from :term:`file objects `. It only supports the Base64 standard alphabet, and it adds newlines every 76 characters as per :rfc:`2045`. Note that if you are looking @@ -46,9 +41,17 @@ package instead. Any :term:`bytes-like objects ` are now accepted by all encoding and decoding functions in this module. Ascii85/Base85 support added. -The modern interface provides: -.. function:: b64encode(s, altchars=None) +.. _base64-rfc-4648: + +RFC 4648 Encodings +------------------ + +The :rfc:`4648` encodings are suitable for encoding binary data so that it can be +safely sent by email, used as parts of URLs, or included as part of an HTTP +POST request. + +.. function:: b64encode(s, altchars=None, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base64 and return the encoded :class:`bytes`. @@ -58,11 +61,19 @@ The modern interface provides: This allows an application to e.g. generate URL or filesystem safe Base64 strings. The default is ``None``, for which the standard Base64 alphabet is used. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. Raises a :exc:`TypeError` if *altchars* is not a :term:`bytes-like object`. + .. versionchanged:: 3.15 + Added the *wrapcol* parameter. + .. function:: b64decode(s, altchars=None, validate=False) + b64decode(s, altchars=None, validate=True, *, ignorechars) Decode the Base64 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -74,15 +85,30 @@ The modern interface provides: A :exc:`binascii.Error` exception is raised if *s* is incorrectly padded. - If *validate* is ``False`` (the default), characters that are neither + If *ignorechars* is specified, it should be a :term:`bytes-like object` + containing characters to ignore from the input when *validate* is true. + The default value of *validate* is ``True`` if *ignorechars* is specified, + ``False`` otherwise. + + If *validate* is false, characters that are neither in the normal base-64 alphabet nor the alternative alphabet are - discarded prior to the padding check. If *validate* is ``True``, - these non-alphabet characters in the input result in a - :exc:`binascii.Error`. + discarded prior to the padding check, but the ``+`` and ``/`` characters + keep their meaning if they are not in *altchars* (they will be discarded + in future Python versions). + + If *validate* is true, these non-alphabet characters in the input + result in a :exc:`binascii.Error`. For more information about the strict base64 check, see :func:`binascii.a2b_base64` - May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. + .. deprecated:: next + Accepting the ``+`` and ``/`` characters with an alternative alphabet + is now deprecated. + + + .. versionchanged:: next + Added the *ignorechars* parameter. + .. function:: standard_b64encode(s) @@ -113,6 +139,9 @@ The modern interface provides: ``/`` in the standard Base64 alphabet, and return the decoded :class:`bytes`. + .. deprecated:: next + Accepting the ``+`` and ``/`` characters is now deprecated. + .. function:: b32encode(s) @@ -181,6 +210,26 @@ The modern interface provides: incorrectly padded or if there are non-alphabet characters present in the input. +.. _base64-base-85: + +Base85 Encodings +----------------- + +Base85 encoding is not formally specified but rather a de facto standard, +thus different systems perform the encoding differently. + +The :func:`a85encode` and :func:`b85encode` functions in this module are two implementations of +the de facto standard. You should call the function with the Base85 +implementation used by the software you intend to work with. + +The two functions present in this module differ in how they handle the following: + +* Whether to include enclosing ``<~`` and ``~>`` markers +* Whether to include newline characters +* The set of ASCII characters used for encoding +* Handling of null bytes + +Refer to the documentation of the individual functions for more information. .. function:: a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False) @@ -191,9 +240,9 @@ The modern interface provides: instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This feature is not supported by the "standard" Ascii85 encoding. - *wrapcol* controls whether the output should have newline (``b'\n'``) - characters added to it. If this is non-zero, each output line will be - at most this many characters long, excluding the trailing newline. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. *pad* controls whether the input is padded to a multiple of 4 before encoding. Note that the ``btoa`` implementation always pads. @@ -216,8 +265,7 @@ The modern interface provides: *adobe* controls whether the input sequence is in Adobe Ascii85 format (i.e. is framed with <~ and ~>). - *ignorechars* should be a :term:`bytes-like object` or ASCII string - containing characters to ignore + *ignorechars* should be a byte string containing characters to ignore from the input. This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. @@ -244,14 +292,20 @@ The modern interface provides: .. versionadded:: 3.4 -.. function:: z85encode(s) +.. function:: z85encode(s, pad=False) Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) and return the encoded :class:`bytes`. See `Z85 specification `_ for more information. + If *pad* is true, the input is padded with ``b'\0'`` so its length is a + multiple of 4 bytes before encoding. + .. versionadded:: 3.13 + .. versionchanged:: 3.15 + The *pad* parameter was added. + .. function:: z85decode(s) @@ -262,7 +316,10 @@ The modern interface provides: .. versionadded:: 3.13 -The legacy interface: +.. _base64-legacy: + +Legacy Interface +---------------- .. function:: decode(input, output) diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 85df7914a9a014..a3c6da7a6d686b 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -118,7 +118,7 @@ The :mod:`bdb` module also defines two classes: Count of the number of times a :class:`Breakpoint` has been hit. -.. class:: Bdb(skip=None) +.. class:: Bdb(skip=None, backend='settrace') The :class:`Bdb` class acts as a generic Python debugger base class. @@ -132,9 +132,22 @@ The :mod:`bdb` module also defines two classes: frame is considered to originate in a certain module is determined by the ``__name__`` in the frame globals. + The *backend* argument specifies the backend to use for :class:`Bdb`. It + can be either ``'settrace'`` or ``'monitoring'``. ``'settrace'`` uses + :func:`sys.settrace` which has the best backward compatibility. The + ``'monitoring'`` backend uses the new :mod:`sys.monitoring` that was + introduced in Python 3.12, which can be much more efficient because it + can disable unused events. We are trying to keep the exact interfaces + for both backends, but there are some differences. The debugger developers + are encouraged to use the ``'monitoring'`` backend to achieve better + performance. + .. versionchanged:: 3.1 Added the *skip* parameter. + .. versionchanged:: 3.14 + Added the *backend* parameter. + The following methods of :class:`Bdb` normally don't need to be overridden. .. method:: canonic(filename) @@ -146,6 +159,20 @@ The :mod:`bdb` module also defines two classes: `. A *filename* with angle brackets, such as ``""`` generated in interactive mode, is returned unchanged. + .. method:: start_trace(self) + + Start tracing. For ``'settrace'`` backend, this method is equivalent to + ``sys.settrace(self.trace_dispatch)`` + + .. versionadded:: 3.14 + + .. method:: stop_trace(self) + + Stop tracing. For ``'settrace'`` backend, this method is equivalent to + ``sys.settrace(None)`` + + .. versionadded:: 3.14 + .. method:: reset() Set the :attr:`!botframe`, :attr:`!stopframe`, :attr:`!returnframe` and @@ -165,12 +192,8 @@ The :mod:`bdb` module also defines two classes: entered. * ``"return"``: A function or other code block is about to return. * ``"exception"``: An exception has occurred. - * ``"c_call"``: A C function is about to be called. - * ``"c_return"``: A C function has returned. - * ``"c_exception"``: A C function has raised an exception. - For the Python events, specialized functions (see below) are called. For - the C events, no action is taken. + For all the events, specialized functions (see below) are called. The *arg* parameter depends on the previous event. @@ -213,7 +236,7 @@ The :mod:`bdb` module also defines two classes: Normally derived classes don't override the following methods, but they may if they want to redefine the definition of stopping and breakpoints. - .. method:: is_skipped_line(module_name) + .. method:: is_skipped_module(module_name) Return ``True`` if *module_name* matches any skip pattern. @@ -364,6 +387,28 @@ The :mod:`bdb` module also defines two classes: Return all breakpoints that are set. + Derived classes and clients can call the following methods to disable and + restart events to achieve better performance. These methods only work + when using the ``'monitoring'`` backend. + + .. method:: disable_current_event() + + Disable the current event until the next time :func:`restart_events` is + called. This is helpful when the debugger is not interested in the current + line. + + .. versionadded:: 3.14 + + .. method:: restart_events() + + Restart all the disabled events. This function is automatically called in + ``dispatch_*`` methods after ``user_*`` methods are called. If the + ``dispatch_*`` methods are not overridden, the disabled events will be + restarted after each user interaction. + + .. versionadded:: 3.14 + + Derived classes and clients can call the following methods to get a data structure representing a stack trace. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 1bab785684bbab..d9f0baedec85f2 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -49,16 +49,22 @@ The :mod:`binascii` module defines the following functions: .. function:: a2b_base64(string, /, *, strict_mode=False) + a2b_base64(string, /, *, strict_mode=True, ignorechars) Convert a block of base64 data back to binary and return the binary data. More than one line may be passed at a time. + If *ignorechars* is specified, it should be a :term:`bytes-like object` + containing characters to ignore from the input when *strict_mode* is true. + The default value of *strict_mode* is ``True`` if *ignorechars* is specified, + ``False`` otherwise. + If *strict_mode* is true, only valid base64 data will be converted. Invalid base64 data will raise :exc:`binascii.Error`. Valid base64: - * Conforms to :rfc:`3548`. + * Conforms to :rfc:`4648`. * Contains only characters from the base64 alphabet. * Contains no excess data after padding (including excess padding, newlines, etc.). * Does not start with a padding. @@ -66,16 +72,28 @@ The :mod:`binascii` module defines the following functions: .. versionchanged:: 3.11 Added the *strict_mode* parameter. + .. versionchanged:: next + Added the *ignorechars* parameter. + + +.. function:: b2a_base64(data, *, wrapcol=0, newline=True) -.. function:: b2a_base64(data, *, newline=True) + Convert binary data to a line(s) of ASCII characters in base64 coding, + as specified in :rfc:`4648`. - Convert binary data to a line of ASCII characters in base64 coding. The return - value is the converted line, including a newline char if *newline* is - true. The output of this function conforms to :rfc:`3548`. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + If *newline* is true (default), a newline character will be added + at the end of the output. .. versionchanged:: 3.6 Added the *newline* parameter. + .. versionchanged:: 3.15 + Added the *wrapcol* parameter. + .. function:: a2b_qp(data, header=False) diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 78da563397b625..d5ec4212c1f9f4 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -24,6 +24,16 @@ method to determine whether a value has been found. Instead, the functions only call the :meth:`~object.__lt__` method and will return an insertion point between values in an array. +.. note:: + + The functions in this module are not thread-safe. If multiple threads + concurrently use :mod:`bisect` functions on the same sequence, this + may result in undefined behaviour. Likewise, if the provided sequence + is mutated by a different thread while a :mod:`bisect` function + is operating on it, the result is undefined. For example, using + :py:func:`~bisect.insort_left` on the same list from multiple threads + may result in the list becoming unsorted. + .. _bisect functions: The following functions are provided: @@ -73,7 +83,7 @@ The following functions are provided: Insert *x* in *a* in sorted order. This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point. - Next, it runs the :meth:`!insert` method on *a* to insert *x* at the + Next, it runs the :meth:`~sequence.insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is @@ -93,7 +103,7 @@ The following functions are provided: entries of *x*. This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point. - Next, it runs the :meth:`!insert` method on *a* to insert *x* at the + Next, it runs the :meth:`~sequence.insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index ebe2e43febaefa..12650861c0fb5d 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -25,6 +25,8 @@ The :mod:`bz2` module contains: * The :func:`compress` and :func:`decompress` functions for one-shot (de)compression. +.. include:: ../includes/optional-module.rst + (De)compression of files ------------------------ diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index eafc038d6cb722..822e627af8db95 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -38,13 +38,33 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is itself. This is the job of subclasses. - :class:`Calendar` instances have the following methods: + :class:`Calendar` instances have the following methods and attributes: + + .. attribute:: firstweekday + + The first weekday as an integer (0--6). + + This property can also be set and read using + :meth:`~Calendar.setfirstweekday` and + :meth:`~Calendar.getfirstweekday` respectively. + + .. method:: getfirstweekday() + + Return an :class:`int` for the current first weekday (0--6). + + Identical to reading the :attr:`~Calendar.firstweekday` property. + + .. method:: setfirstweekday(firstweekday) + + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + + Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() Return an iterator for the week day numbers that will be used for one week. The first value from the iterator will be the same as the value of - the :attr:`firstweekday` property. + the :attr:`~Calendar.firstweekday` property. .. method:: itermonthdates(year, month) @@ -138,6 +158,41 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :class:`TextCalendar` instances have the following methods: + .. method:: prweek(theweek, width) + + Print a week's calendar as returned by :meth:`formatweek` and without a + final newline. + + + .. method:: formatday(theday, weekday, width) + + Return a string representing a single day formatted with the given *width*. + If *theday* is ``0``, return a string of spaces of + the specified width, representing an empty day. The *weekday* parameter + is unused. + + .. method:: formatweek(theweek, w=0) + + Return a single week in a string with no newline. If *w* is provided, it + specifies the width of the date columns, which are centered. Depends + on the first weekday as specified in the constructor or set by the + :meth:`setfirstweekday` method. + + + .. method:: formatweekday(weekday, width) + + Return a string representing the name of a single weekday formatted to + the specified *width*. The *weekday* parameter is an integer representing + the day of the week, where ``0`` is Monday and ``6`` is Sunday. + + + .. method:: formatweekheader(width) + + Return a string containing the header row of weekday names, formatted + with the given *width* for each column. The names depend on the locale + settings and are padded to the specified width. + + .. method:: formatmonth(theyear, themonth, w=0, l=0) Return a month's calendar in a multi-line string. If *w* is provided, it @@ -147,6 +202,14 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :meth:`setfirstweekday` method. + .. method:: formatmonthname(theyear, themonth, width=0, withyear=True) + + Return a string representing the month's name centered within the + specified *width*. If *withyear* is ``True``, include the year in the + output. The *theyear* and *themonth* parameters specify the year + and month for the name to be formatted respectively. + + .. method:: prmonth(theyear, themonth, w=0, l=0) Print a month's calendar as returned by :meth:`formatmonth`. @@ -193,7 +256,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is 3) specifies the number of months per row. *css* is the name for the cascading style sheet to be used. :const:`None` can be passed if no style sheet should be used. *encoding* specifies the encoding to be used for the - output (defaulting to the system default encoding). + output (defaulting to ``'utf-8'``). .. method:: formatmonthname(theyear, themonth, withyear=True) @@ -437,12 +500,20 @@ The :mod:`calendar` module exports the following data attributes: A sequence that represents the months of the year in the current locale. This follows normal convention of January being month number 1, so it has a length of - 13 and ``month_name[0]`` is the empty string. + 13 and ``month_name[0]`` is the empty string. >>> import calendar >>> list(calendar.month_name) ['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'] + .. caution:: + + In locales with alternative month names forms, the :data:`!month_name` sequence + may not be suitable when a month name stands by itself and not as part of a date. + For instance, in Greek and in many Slavic and Baltic languages, :data:`!month_name` + will produce the month in genitive case. Use :data:`standalone_month_name` for a form + suitable for standalone use. + .. data:: month_abbr @@ -454,6 +525,31 @@ The :mod:`calendar` module exports the following data attributes: >>> list(calendar.month_abbr) ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + .. caution:: + + In locales with alternative month names forms, the :data:`!month_abbr` sequence + may not be suitable when a month name stands by itself and not as part of a date. + Use :data:`standalone_month_abbr` for a form suitable for standalone use. + + +.. data:: standalone_month_name + + A sequence that represents the months of the year in the current locale + in the standalone form if the locale provides one. Else it is equivalent + to :data:`month_name`. + + .. versionadded:: 3.15 + + +.. data:: standalone_month_abbr + + A sequence that represents the abbreviated months of the year in the current + locale in the standalone form if the locale provides one. Else it is + equivalent to :data:`month_abbr`. + + .. versionadded:: 3.15 + + .. data:: JANUARY FEBRUARY MARCH @@ -516,7 +612,7 @@ The :mod:`calendar` module defines the following exceptions: .. _calendar-cli: -Command-Line Usage +Command-line usage ------------------ .. versionadded:: 2.5 @@ -619,8 +715,7 @@ The following options are accepted: .. option:: month The month of the specified :option:`year` to print the calendar for. - Must be a number between 1 and 12, - and may only be used in text mode. + Must be a number between 1 and 12. Defaults to printing a calendar for the full year. @@ -654,6 +749,9 @@ The following options are accepted: The number of months printed per row. Defaults to 3. +.. versionchanged:: 3.14 + By default, today's date is highlighted in color and can be + :ref:`controlled using environment variables `. *HTML-mode options:* diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst new file mode 100644 index 00000000000000..f9108fa954a906 --- /dev/null +++ b/Doc/library/cgi.rst @@ -0,0 +1,19 @@ +:mod:`!cgi` --- Common Gateway Interface support +================================================ + +.. module:: cgi + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +A fork of the module on PyPI can be used instead: :pypi:`legacy-cgi`. +This is a copy of the cgi module, no longer maintained or supported by the core +Python team. + +The last version of Python that provided the :mod:`!cgi` module was +`Python 3.12 `_. diff --git a/Doc/library/cgitb.rst b/Doc/library/cgitb.rst new file mode 100644 index 00000000000000..fc646aa4c48acd --- /dev/null +++ b/Doc/library/cgitb.rst @@ -0,0 +1,19 @@ +:mod:`!cgitb` --- Traceback manager for CGI scripts +=================================================== + +.. module:: cgitb + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +A fork of the module on PyPI can now be used instead: :pypi:`legacy-cgi`. +This is a copy of the cgi module, no longer maintained or supported by the core +Python team. + +The last version of Python that provided the :mod:`!cgitb` module was +`Python 3.12 `_. diff --git a/Doc/library/chunk.rst b/Doc/library/chunk.rst new file mode 100644 index 00000000000000..9950a0ea70649a --- /dev/null +++ b/Doc/library/chunk.rst @@ -0,0 +1,15 @@ +:mod:`!chunk` --- Read IFF chunked data +======================================= + +.. module:: chunk + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +The last version of Python that provided the :mod:`!chunk` module was +`Python 3.12 `_. diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 381a8332f4b187..b6d5dbee21dcd5 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -24,20 +24,71 @@ the function is then applied to the result of the conversion. imaginary axis we look at the sign of the real part. For example, the :func:`cmath.sqrt` function has a branch cut along the - negative real axis. An argument of ``complex(-2.0, -0.0)`` is treated as + negative real axis. An argument of ``-2-0j`` is treated as though it lies *below* the branch cut, and so gives a result on the negative imaginary axis:: - >>> cmath.sqrt(complex(-2.0, -0.0)) + >>> cmath.sqrt(-2-0j) -1.4142135623730951j - But an argument of ``complex(-2.0, 0.0)`` is treated as though it lies above + But an argument of ``-2+0j`` is treated as though it lies above the branch cut:: - >>> cmath.sqrt(complex(-2.0, 0.0)) + >>> cmath.sqrt(-2+0j) 1.4142135623730951j +==================================================== ============================================ +**Conversions to and from polar coordinates** +-------------------------------------------------------------------------------------------------- +:func:`phase(z) ` Return the phase of *z* +:func:`polar(z) ` Return the representation of *z* in polar coordinates +:func:`rect(r, phi) ` Return the complex number *z* with polar coordinates *r* and *phi* + +**Power and logarithmic functions** +-------------------------------------------------------------------------------------------------- +:func:`exp(z) ` Return *e* raised to the power *z* +:func:`log(z[, base]) ` Return the logarithm of *z* to the given *base* (*e* by default) +:func:`log10(z) ` Return the base-10 logarithm of *z* +:func:`sqrt(z) ` Return the square root of *z* + +**Trigonometric functions** +-------------------------------------------------------------------------------------------------- +:func:`acos(z) ` Return the arc cosine of *z* +:func:`asin(z) ` Return the arc sine of *z* +:func:`atan(z) ` Return the arc tangent of *z* +:func:`cos(z) ` Return the cosine of *z* +:func:`sin(z) ` Return the sine of *z* +:func:`tan(z) ` Return the tangent of *z* + +**Hyperbolic functions** +-------------------------------------------------------------------------------------------------- +:func:`acosh(z) ` Return the inverse hyperbolic cosine of *z* +:func:`asinh(z) ` Return the inverse hyperbolic sine of *z* +:func:`atanh(z) ` Return the inverse hyperbolic tangent of *z* +:func:`cosh(z) ` Return the hyperbolic cosine of *z* +:func:`sinh(z) ` Return the hyperbolic sine of *z* +:func:`tanh(z) ` Return the hyperbolic tangent of *z* + +**Classification functions** +-------------------------------------------------------------------------------------------------- +:func:`isfinite(z) ` Check if all components of *z* are finite +:func:`isinf(z) ` Check if any component of *z* is infinite +:func:`isnan(z) ` Check if any component of *z* is a NaN +:func:`isclose(a, b, *, rel_tol, abs_tol) ` Check if the values *a* and *b* are close to each other + +**Constants** +-------------------------------------------------------------------------------------------------- +:data:`pi` *π* = 3.141592... +:data:`e` *e* = 2.718281... +:data:`tau` *τ* = 2\ *π* = 6.283185... +:data:`inf` Positive infinity +:data:`infj` Pure imaginary infinity +:data:`nan` "Not a number" (NaN) +:data:`nanj` Pure imaginary NaN +==================================================== ============================================ + + Conversions to and from polar coordinates ----------------------------------------- @@ -55,163 +106,163 @@ segment that joins the origin to *z*. The following functions can be used to convert from the native rectangular coordinates to polar coordinates and back. -.. function:: phase(x) +.. function:: phase(z) - Return the phase of *x* (also known as the *argument* of *x*), as a float. - ``phase(x)`` is equivalent to ``math.atan2(x.imag, x.real)``. The result + Return the phase of *z* (also known as the *argument* of *z*), as a float. + ``phase(z)`` is equivalent to ``math.atan2(z.imag, z.real)``. The result lies in the range [-\ *π*, *π*], and the branch cut for this operation lies along the negative real axis. The sign of the result is the same as the - sign of ``x.imag``, even when ``x.imag`` is zero:: + sign of ``z.imag``, even when ``z.imag`` is zero:: - >>> phase(complex(-1.0, 0.0)) + >>> phase(-1+0j) 3.141592653589793 - >>> phase(complex(-1.0, -0.0)) + >>> phase(-1-0j) -3.141592653589793 .. note:: - The modulus (absolute value) of a complex number *x* can be + The modulus (absolute value) of a complex number *z* can be computed using the built-in :func:`abs` function. There is no separate :mod:`cmath` module function for this operation. -.. function:: polar(x) +.. function:: polar(z) - Return the representation of *x* in polar coordinates. Returns a - pair ``(r, phi)`` where *r* is the modulus of *x* and phi is the - phase of *x*. ``polar(x)`` is equivalent to ``(abs(x), - phase(x))``. + Return the representation of *z* in polar coordinates. Returns a + pair ``(r, phi)`` where *r* is the modulus of *z* and *phi* is the + phase of *z*. ``polar(z)`` is equivalent to ``(abs(z), + phase(z))``. .. function:: rect(r, phi) - Return the complex number *x* with polar coordinates *r* and *phi*. + Return the complex number *z* with polar coordinates *r* and *phi*. Equivalent to ``complex(r * math.cos(phi), r * math.sin(phi))``. Power and logarithmic functions ------------------------------- -.. function:: exp(x) +.. function:: exp(z) - Return *e* raised to the power *x*, where *e* is the base of natural + Return *e* raised to the power *z*, where *e* is the base of natural logarithms. -.. function:: log(x[, base]) +.. function:: log(z[, base]) - Returns the logarithm of *x* to the given *base*. If the *base* is not - specified, returns the natural logarithm of *x*. There is one branch cut, + Return the logarithm of *z* to the given *base*. If the *base* is not + specified, returns the natural logarithm of *z*. There is one branch cut, from 0 along the negative real axis to -∞. -.. function:: log10(x) +.. function:: log10(z) - Return the base-10 logarithm of *x*. This has the same branch cut as + Return the base-10 logarithm of *z*. This has the same branch cut as :func:`log`. -.. function:: sqrt(x) +.. function:: sqrt(z) - Return the square root of *x*. This has the same branch cut as :func:`log`. + Return the square root of *z*. This has the same branch cut as :func:`log`. Trigonometric functions ----------------------- -.. function:: acos(x) +.. function:: acos(z) - Return the arc cosine of *x*. There are two branch cuts: One extends right + Return the arc cosine of *z*. There are two branch cuts: One extends right from 1 along the real axis to ∞. The other extends left from -1 along the real axis to -∞. -.. function:: asin(x) +.. function:: asin(z) - Return the arc sine of *x*. This has the same branch cuts as :func:`acos`. + Return the arc sine of *z*. This has the same branch cuts as :func:`acos`. -.. function:: atan(x) +.. function:: atan(z) - Return the arc tangent of *x*. There are two branch cuts: One extends from + Return the arc tangent of *z*. There are two branch cuts: One extends from ``1j`` along the imaginary axis to ``∞j``. The other extends from ``-1j`` along the imaginary axis to ``-∞j``. -.. function:: cos(x) +.. function:: cos(z) - Return the cosine of *x*. + Return the cosine of *z*. -.. function:: sin(x) +.. function:: sin(z) - Return the sine of *x*. + Return the sine of *z*. -.. function:: tan(x) +.. function:: tan(z) - Return the tangent of *x*. + Return the tangent of *z*. Hyperbolic functions -------------------- -.. function:: acosh(x) +.. function:: acosh(z) - Return the inverse hyperbolic cosine of *x*. There is one branch cut, + Return the inverse hyperbolic cosine of *z*. There is one branch cut, extending left from 1 along the real axis to -∞. -.. function:: asinh(x) +.. function:: asinh(z) - Return the inverse hyperbolic sine of *x*. There are two branch cuts: + Return the inverse hyperbolic sine of *z*. There are two branch cuts: One extends from ``1j`` along the imaginary axis to ``∞j``. The other extends from ``-1j`` along the imaginary axis to ``-∞j``. -.. function:: atanh(x) +.. function:: atanh(z) - Return the inverse hyperbolic tangent of *x*. There are two branch cuts: One + Return the inverse hyperbolic tangent of *z*. There are two branch cuts: One extends from ``1`` along the real axis to ``∞``. The other extends from ``-1`` along the real axis to ``-∞``. -.. function:: cosh(x) +.. function:: cosh(z) - Return the hyperbolic cosine of *x*. + Return the hyperbolic cosine of *z*. -.. function:: sinh(x) +.. function:: sinh(z) - Return the hyperbolic sine of *x*. + Return the hyperbolic sine of *z*. -.. function:: tanh(x) +.. function:: tanh(z) - Return the hyperbolic tangent of *x*. + Return the hyperbolic tangent of *z*. Classification functions ------------------------ -.. function:: isfinite(x) +.. function:: isfinite(z) - Return ``True`` if both the real and imaginary parts of *x* are finite, and + Return ``True`` if both the real and imaginary parts of *z* are finite, and ``False`` otherwise. .. versionadded:: 3.2 -.. function:: isinf(x) +.. function:: isinf(z) - Return ``True`` if either the real or the imaginary part of *x* is an + Return ``True`` if either the real or the imaginary part of *z* is an infinity, and ``False`` otherwise. -.. function:: isnan(x) +.. function:: isnan(z) - Return ``True`` if either the real or the imaginary part of *x* is a NaN, + Return ``True`` if either the real or the imaginary part of *z* is a NaN, and ``False`` otherwise. @@ -221,19 +272,21 @@ Classification functions ``False`` otherwise. Whether or not two values are considered close is determined according to - given absolute and relative tolerances. + given absolute and relative tolerances. If no errors occur, the result will + be: ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. *rel_tol* is the relative tolerance -- it is the maximum allowed difference between *a* and *b*, relative to the larger absolute value of *a* or *b*. For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default tolerance is ``1e-09``, which assures that the two values are the same - within about 9 decimal digits. *rel_tol* must be greater than zero. - - *abs_tol* is the minimum absolute tolerance -- useful for comparisons near - zero. *abs_tol* must be at least zero. + within about 9 decimal digits. *rel_tol* must be nonnegative and less + than ``1.0``. - If no errors occur, the result will be: - ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. + *abs_tol* is the absolute tolerance; it defaults to ``0.0`` and it must be + nonnegative. When comparing ``x`` to ``0.0``, ``isclose(x, 0)`` is computed + as ``abs(x) <= rel_tol * abs(x)``, which is ``False`` for any ``x`` and + rel_tol less than ``1.0``. So add an appropriate positive abs_tol argument + to the call. The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be handled according to IEEE rules. Specifically, ``NaN`` is not considered @@ -285,7 +338,7 @@ Constants .. data:: nan A floating-point "not a number" (NaN) value. Equivalent to - ``float('nan')``. + ``float('nan')``. See also :data:`math.nan`. .. versionadded:: 3.6 diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index 78fe95a014ff7c..6418706269f1ed 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -1,3 +1,5 @@ +.. _library-cmdline: + ++++++++++++++++++++++++++++++++++++ Modules command-line interface (CLI) ++++++++++++++++++++++++++++++++++++ @@ -10,28 +12,28 @@ The following modules have a command-line interface. * :ref:`calendar ` * :mod:`code` * :ref:`compileall ` -* :mod:`cProfile`: see :ref:`profile ` -* :ref:`difflib ` +* ``cProfile``: see :ref:`profiling.tracing ` * :ref:`dis ` -* :mod:`doctest` +* :ref:`doctest ` * :mod:`!encodings.rot_13` -* :mod:`ensurepip` +* :ref:`ensurepip ` * :mod:`filecmp` * :mod:`fileinput` * :mod:`ftplib` * :ref:`gzip ` * :ref:`http.server ` -* :mod:`!idlelib` +* :ref:`idlelib ` * :ref:`inspect ` * :ref:`json ` -* :mod:`mimetypes` -* :mod:`pdb` -* :mod:`pickle` +* :ref:`mimetypes ` +* :ref:`pdb ` +* :ref:`pickle ` * :ref:`pickletools ` -* :mod:`platform` +* :ref:`platform ` * :mod:`poplib` -* :ref:`profile ` -* :mod:`pstats` +* :ref:`profiling.sampling ` +* :ref:`profiling.tracing ` +* :ref:`pstats ` * :ref:`py_compile ` * :mod:`pyclbr` * :mod:`pydoc` @@ -51,8 +53,8 @@ The following modules have a command-line interface. * :mod:`turtledemo` * :ref:`unittest ` * :ref:`uuid ` -* :mod:`venv` -* :mod:`webbrowser` +* :ref:`venv ` +* :ref:`webbrowser ` * :ref:`zipapp ` * :ref:`zipfile ` diff --git a/Doc/library/cmdlinelibs.rst b/Doc/library/cmdlinelibs.rst new file mode 100644 index 00000000000000..32f8c2c9f4ae32 --- /dev/null +++ b/Doc/library/cmdlinelibs.rst @@ -0,0 +1,22 @@ +.. _cmdlinelibs: + +******************************** +Command-line interface libraries +******************************** + +The modules described in this chapter assist with implementing +command line and terminal interfaces for applications. + +Here's an overview: + +.. toctree:: + :maxdepth: 1 + + argparse.rst + optparse.rst + getpass.rst + fileinput.rst + curses.rst + curses.ascii.rst + curses.panel.rst + cmd.rst diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 8f7692df9fb22d..59c016d21501b0 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -22,8 +22,14 @@ build applications which provide an interactive interpreter prompt. it defaults to a newly created dictionary with key ``'__name__'`` set to ``'__console__'`` and key ``'__doc__'`` set to ``None``. + Note that functions and classes objects created under an + :class:`!InteractiveInterpreter` instance will belong to the namespace + specified by *locals*. + They are only pickleable if *locals* is the namespace of an existing + module. -.. class:: InteractiveConsole(locals=None, filename="", local_exit=False) + +.. class:: InteractiveConsole(locals=None, filename="", *, local_exit=False) Closely emulate the behavior of the interactive Python interpreter. This class builds on :class:`InteractiveInterpreter` and adds prompting using the familiar diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 2cfd8a1eaee806..305e5d07a3529e 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -53,18 +53,36 @@ any codec: :exc:`UnicodeDecodeError`). Refer to :ref:`codec-base-classes` for more information on codec error handling. +.. function:: charmap_build(string) + + Return a mapping suitable for encoding with a custom single-byte encoding. + Given a :class:`str` *string* of up to 256 characters representing a + decoding table, returns either a compact internal mapping object + ``EncodingMap`` or a :class:`dictionary ` mapping character ordinals + to byte values. Raises a :exc:`TypeError` on invalid input. + The full details for each codec can also be looked up directly: -.. function:: lookup(encoding) +.. function:: lookup(encoding, /) Looks up the codec info in the Python codec registry and returns a :class:`CodecInfo` object as defined below. - Encodings are first looked up in the registry's cache. If not found, the list of + This function first normalizes the *encoding*: all ASCII letters are + converted to lower case, spaces are replaced with hyphens. + Then encoding is looked up in the registry's cache. If not found, the list of registered search functions is scanned. If no :class:`CodecInfo` object is found, a :exc:`LookupError` is raised. Otherwise, the :class:`CodecInfo` object is stored in the cache and returned to the caller. + .. versionchanged:: 3.9 + Any characters except ASCII letters and digits and a dot are converted to underscore. + + .. versionchanged:: 3.15 + No characters are converted to underscore anymore. + Spaces are converted to hyphens. + + .. class:: CodecInfo(encode, decode, streamreader=None, streamwriter=None, incrementalencoder=None, incrementaldecoder=None, name=None) Codec details when looking up the codec registry. The constructor @@ -156,19 +174,16 @@ these additional functions which use :func:`lookup` for the codec lookup: Custom codecs are made available by registering a suitable codec search function: -.. function:: register(search_function) +.. function:: register(search_function, /) Register a codec search function. Search functions are expected to take one - argument, being the encoding name in all lower case letters with hyphens - and spaces converted to underscores, and return a :class:`CodecInfo` object. + argument, being the encoding name in all lower case letters with spaces + converted to hyphens, and return a :class:`CodecInfo` object. In case a search function cannot find a given encoding, it should return ``None``. - .. versionchanged:: 3.9 - Hyphens and spaces are converted to underscore. - -.. function:: unregister(search_function) +.. function:: unregister(search_function, /) Unregister a codec search function and clear the registry's cache. If the search function is not registered, do nothing. @@ -208,6 +223,10 @@ wider range of codecs when working with binary files: .. versionchanged:: 3.11 The ``'U'`` mode has been removed. + .. deprecated:: 3.14 + + :func:`codecs.open` has been superseded by :func:`open`. + .. function:: EncodedFile(file, data_encoding, file_encoding=None, errors='strict') @@ -231,8 +250,8 @@ wider range of codecs when working with binary files: .. function:: iterencode(iterator, encoding, errors='strict', **kwargs) Uses an incremental encoder to iteratively encode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any + *iterator*. *iterator* must yield :class:`str` objects. + This function is a :term:`generator`. The *errors* argument (as well as any other keyword argument) is passed through to the incremental encoder. This function requires that the codec accept text :class:`str` objects @@ -243,8 +262,8 @@ wider range of codecs when working with binary files: .. function:: iterdecode(iterator, encoding, errors='strict', **kwargs) Uses an incremental decoder to iteratively decode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any + *iterator*. *iterator* must yield :class:`bytes` objects. + This function is a :term:`generator`. The *errors* argument (as well as any other keyword argument) is passed through to the incremental decoder. This function requires that the codec accept :class:`bytes` objects @@ -253,6 +272,20 @@ wider range of codecs when working with binary files: :func:`iterencode`. +.. function:: readbuffer_encode(buffer, errors=None, /) + + Return a :class:`tuple` containing the raw bytes of *buffer*, a + :ref:`buffer-compatible object ` or :class:`str` + (encoded to UTF-8 before processing), and their length in bytes. + + The *errors* argument is ignored. + + .. code-block:: pycon + + >>> codecs.readbuffer_encode(b"Zito") + (b'Zito', 4) + + The module also provides the following constants which are useful for reading and writing to platform dependent files: @@ -416,7 +449,7 @@ In addition, the following error handler is specific to the given codecs: The set of allowed values can be extended by registering a new named error handler: -.. function:: register_error(name, error_handler) +.. function:: register_error(name, error_handler, /) Register the error handling function *error_handler* under the name *name*. The *error_handler* argument will be called during encoding and decoding @@ -442,7 +475,7 @@ handler: Previously registered error handlers (including the standard error handlers) can be looked up by name: -.. function:: lookup_error(name) +.. function:: lookup_error(name, /) Return the error handler previously registered under the name *name*. @@ -956,17 +989,22 @@ defined in Unicode. A simple and straightforward way that can store each Unicode code point, is to store each code point as four consecutive bytes. There are two possibilities: store the bytes in big endian or in little endian order. These two encodings are called ``UTF-32-BE`` and ``UTF-32-LE`` respectively. Their -disadvantage is that if e.g. you use ``UTF-32-BE`` on a little endian machine you -will always have to swap bytes on encoding and decoding. ``UTF-32`` avoids this -problem: bytes will always be in natural endianness. When these bytes are read -by a CPU with a different endianness, then bytes have to be swapped though. To -be able to detect the endianness of a ``UTF-16`` or ``UTF-32`` byte sequence, -there's the so called BOM ("Byte Order Mark"). This is the Unicode character -``U+FEFF``. This character can be prepended to every ``UTF-16`` or ``UTF-32`` -byte sequence. The byte swapped version of this character (``0xFFFE``) is an -illegal character that may not appear in a Unicode text. So when the -first character in a ``UTF-16`` or ``UTF-32`` byte sequence -appears to be a ``U+FFFE`` the bytes have to be swapped on decoding. +disadvantage is that if, for example, you use ``UTF-32-BE`` on a little endian +machine you will always have to swap bytes on encoding and decoding. +Python's ``UTF-16`` and ``UTF-32`` codecs avoid this problem by using the +platform's native byte order when no BOM is present. +Python follows prevailing platform +practice, so native-endian data round-trips without redundant byte swapping, +even though the Unicode Standard defaults to big-endian when the byte order is +unspecified. When these bytes are read by a CPU with a different endianness, +the bytes have to be swapped. To be able to detect the endianness of a +``UTF-16`` or ``UTF-32`` byte sequence, a BOM ("Byte Order Mark") is used. +This is the Unicode character ``U+FEFF``. This character can be prepended to every +``UTF-16`` or ``UTF-32`` byte sequence. The byte swapped version of this character +(``0xFFFE``) is an illegal character that may not appear in a Unicode text. +When the first character of a ``UTF-16`` or ``UTF-32`` byte sequence is +``U+FFFE``, the bytes have to be swapped on decoding. + Unfortunately the character ``U+FEFF`` had a second purpose as a ``ZERO WIDTH NO-BREAK SPACE``: a character that has no width and doesn't allow a word to be split. It can e.g. be used to give hints to a ligature algorithm. @@ -1039,8 +1077,19 @@ or with dictionaries as mapping tables. The following table lists the codecs by name, together with a few common aliases, and the languages for which the encoding is likely used. Neither the list of aliases nor the list of languages is meant to be exhaustive. Notice that spelling alternatives that only differ in -case or use a hyphen instead of an underscore are also valid aliases; therefore, -e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. +case or use a space or a hyphen instead of an underscore are also valid aliases +because they are equivalent when normalized by +:func:`~encodings.normalize_encoding`. For example, ``'utf-8'`` is a valid +alias for the ``'utf_8'`` codec. + +.. note:: + + The below table lists the most common aliases, for a complete list + refer to the source :source:`aliases.py ` file. + +On Windows, ``cpXXX`` codecs are available for all code pages. +But only codecs listed in the following table are guaranteed to exist on +other platforms. .. impl-detail:: @@ -1103,7 +1152,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp852 | 852, IBM852 | Central and Eastern Europe | +-----------------+--------------------------------+--------------------------------+ -| cp855 | 855, IBM855 | Bulgarian, Byelorussian, | +| cp855 | 855, IBM855 | Belarusian, Bulgarian, | | | | Macedonian, Russian, Serbian | +-----------------+--------------------------------+--------------------------------+ | cp856 | | Hebrew | @@ -1151,7 +1200,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp1250 | windows-1250 | Central and Eastern Europe | +-----------------+--------------------------------+--------------------------------+ -| cp1251 | windows-1251 | Bulgarian, Byelorussian, | +| cp1251 | windows-1251 | Belarusian, Bulgarian, | | | | Macedonian, Russian, Serbian | +-----------------+--------------------------------+--------------------------------+ | cp1252 | windows-1252 | Western Europe | @@ -1214,9 +1263,9 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | iso8859_3 | iso-8859-3, latin3, L3 | Esperanto, Maltese | +-----------------+--------------------------------+--------------------------------+ -| iso8859_4 | iso-8859-4, latin4, L4 | Baltic languages | +| iso8859_4 | iso-8859-4, latin4, L4 | Northern Europe | +-----------------+--------------------------------+--------------------------------+ -| iso8859_5 | iso-8859-5, cyrillic | Bulgarian, Byelorussian, | +| iso8859_5 | iso-8859-5, cyrillic | Belarusian, Bulgarian, | | | | Macedonian, Russian, Serbian | +-----------------+--------------------------------+--------------------------------+ | iso8859_6 | iso-8859-6, arabic | Arabic | @@ -1253,7 +1302,7 @@ particular, the following variants typically exist: | | | | | | | .. versionadded:: 3.5 | +-----------------+--------------------------------+--------------------------------+ -| mac_cyrillic | maccyrillic | Bulgarian, Byelorussian, | +| mac_cyrillic | maccyrillic | Belarusian, Bulgarian, | | | | Macedonian, Russian, Serbian | +-----------------+--------------------------------+--------------------------------+ | mac_greek | macgreek | Greek | @@ -1295,7 +1344,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | utf_8 | U8, UTF, utf8, cp65001 | all languages | +-----------------+--------------------------------+--------------------------------+ -| utf_8_sig | | all languages | +| utf_8_sig | utf8-sig | all languages | +-----------------+--------------------------------+--------------------------------+ .. versionchanged:: 3.4 @@ -1307,6 +1356,9 @@ particular, the following variants typically exist: .. versionchanged:: 3.8 ``cp65001`` is now an alias to ``utf_8``. +.. versionchanged:: 3.14 + On Windows, ``cpXXX`` codecs are now available for all code pages. + Python Specific Encodings ------------------------- @@ -1362,7 +1414,11 @@ encodings. | | | It is used in the Python | | | | pickle protocol. | +--------------------+---------+---------------------------+ -| undefined | | Raise an exception for | +| undefined | | This Codec should only | +| | | be used for testing | +| | | purposes. | +| | | | +| | | Raise an exception for | | | | all conversions, even | | | | empty strings. The error | | | | handler is ignored. | @@ -1439,6 +1495,36 @@ to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` Restoration of the aliases for the binary transforms. +.. _standalone-codec-functions: + +Standalone Codec Functions +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following functions provide encoding and decoding functionality similar to codecs, +but are not available as named codecs through :func:`codecs.encode` or :func:`codecs.decode`. +They are used internally (for example, by :mod:`pickle`) and behave similarly to the +``string_escape`` codec that was removed in Python 3. + +.. function:: codecs.escape_encode(input, errors=None) + + Encode *input* using escape sequences. Similar to how :func:`repr` on bytes + produces escaped byte values. + + *input* must be a :class:`bytes` object. + + Returns a tuple ``(output, length)`` where *output* is a :class:`bytes` + object and *length* is the number of bytes consumed. + +.. function:: codecs.escape_decode(input, errors=None) + + Decode *input* from escape sequences back to the original bytes. + + *input* must be a :term:`bytes-like object`. + + Returns a tuple ``(output, length)`` where *output* is a :class:`bytes` + object and *length* is the number of bytes consumed. + + .. _text-transforms: Text Transforms @@ -1465,6 +1551,66 @@ mapping. It is not supported by :meth:`str.encode` (which only produces Restoration of the ``rot13`` alias. +:mod:`encodings` --- Encodings package +-------------------------------------- + +.. module:: encodings + :synopsis: Encodings package + +This module implements the following functions: + +.. function:: normalize_encoding(encoding) + + Normalize encoding name *encoding*. + + Normalization works as follows: all non-alphanumeric characters except the + dot used for Python package names are collapsed and replaced with a single + underscore, leading and trailing underscores are removed. + For example, ``' -;#'`` becomes ``'_'``. + + Note that *encoding* should be ASCII only. + + +.. note:: + The following functions should not be used directly, except for testing + purposes; :func:`codecs.lookup` should be used instead. + + +.. function:: search_function(encoding) + + Search for the codec module corresponding to the given encoding name + *encoding*. + + This function first normalizes the *encoding* using + :func:`normalize_encoding`, then looks for a corresponding alias. + It attempts to import a codec module from the encodings package using either + the alias or the normalized name. If the module is found and defines a valid + ``getregentry()`` function that returns a :class:`codecs.CodecInfo` object, + the codec is cached and returned. + + If the codec module defines a ``getaliases()`` function any returned aliases + are registered for future use. + + +.. function:: win32_code_page_search_function(encoding) + + Search for a Windows code page encoding *encoding* of the form ``cpXXXX``. + + If the code page is valid and supported, return a :class:`codecs.CodecInfo` + object for it. + + .. availability:: Windows. + + .. versionadded:: 3.14 + + +This module implements the following exception: + +.. exception:: CodecRegistryError + + Raised when a codec is invalid or incompatible. + + :mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------ diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index b77a36393b2769..e6daccb91f2b4e 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -28,81 +28,80 @@ An :func:`issubclass` or :func:`isinstance` test for an interface works in one of three ways. 1) A newly written class can inherit directly from one of the -abstract base classes. The class must supply the required abstract -methods. The remaining mixin methods come from inheritance and can be -overridden if desired. Other methods may be added as needed: + abstract base classes. The class must supply the required abstract + methods. The remaining mixin methods come from inheritance and can be + overridden if desired. Other methods may be added as needed: -.. testcode:: + .. testcode:: - class C(Sequence): # Direct inheritance - def __init__(self): ... # Extra method not required by the ABC - def __getitem__(self, index): ... # Required abstract method - def __len__(self): ... # Required abstract method - def count(self, value): ... # Optionally override a mixin method + class C(Sequence): # Direct inheritance + def __init__(self): ... # Extra method not required by the ABC + def __getitem__(self, index): ... # Required abstract method + def __len__(self): ... # Required abstract method + def count(self, value): ... # Optionally override a mixin method -.. doctest:: + .. doctest:: - >>> issubclass(C, Sequence) - True - >>> isinstance(C(), Sequence) - True + >>> issubclass(C, Sequence) + True + >>> isinstance(C(), Sequence) + True 2) Existing classes and built-in classes can be registered as "virtual -subclasses" of the ABCs. Those classes should define the full API -including all of the abstract methods and all of the mixin methods. -This lets users rely on :func:`issubclass` or :func:`isinstance` tests -to determine whether the full interface is supported. The exception to -this rule is for methods that are automatically inferred from the rest -of the API: + subclasses" of the ABCs. Those classes should define the full API + including all of the abstract methods and all of the mixin methods. + This lets users rely on :func:`issubclass` or :func:`isinstance` tests + to determine whether the full interface is supported. The exception to + this rule is for methods that are automatically inferred from the rest + of the API: -.. testcode:: + .. testcode:: - class D: # No inheritance - def __init__(self): ... # Extra method not required by the ABC - def __getitem__(self, index): ... # Abstract method - def __len__(self): ... # Abstract method - def count(self, value): ... # Mixin method - def index(self, value): ... # Mixin method + class D: # No inheritance + def __init__(self): ... # Extra method not required by the ABC + def __getitem__(self, index): ... # Abstract method + def __len__(self): ... # Abstract method + def count(self, value): ... # Mixin method + def index(self, value): ... # Mixin method - Sequence.register(D) # Register instead of inherit + Sequence.register(D) # Register instead of inherit -.. doctest:: + .. doctest:: - >>> issubclass(D, Sequence) - True - >>> isinstance(D(), Sequence) - True + >>> issubclass(D, Sequence) + True + >>> isinstance(D(), Sequence) + True -In this example, class :class:`!D` does not need to define -``__contains__``, ``__iter__``, and ``__reversed__`` because the -:ref:`in-operator `, the :term:`iteration ` -logic, and the :func:`reversed` function automatically fall back to -using ``__getitem__`` and ``__len__``. + In this example, class :class:`!D` does not need to define + ``__contains__``, ``__iter__``, and ``__reversed__`` because the + :ref:`in-operator `, the :term:`iteration ` + logic, and the :func:`reversed` function automatically fall back to + using ``__getitem__`` and ``__len__``. 3) Some simple interfaces are directly recognizable by the presence of -the required methods (unless those methods have been set to -:const:`None`): + the required methods (unless those methods have been set to :const:`None`): -.. testcode:: + .. testcode:: - class E: - def __iter__(self): ... - def __next__(self): ... + class E: + def __iter__(self): ... + def __next__(self): ... -.. doctest:: + .. doctest:: - >>> issubclass(E, Iterable) - True - >>> isinstance(E(), Iterable) - True + >>> issubclass(E, Iterable) + True + >>> isinstance(E(), Iterable) + True -Complex interfaces do not support this last technique because an -interface is more than just the presence of method names. Interfaces -specify semantics and relationships between methods that cannot be -inferred solely from the presence of specific method names. For -example, knowing that a class supplies ``__getitem__``, ``__len__``, and -``__iter__`` is insufficient for distinguishing a :class:`Sequence` from -a :class:`Mapping`. + Complex interfaces do not support this last technique because an + interface is more than just the presence of method names. Interfaces + specify semantics and relationships between methods that cannot be + inferred solely from the presence of specific method names. For + example, knowing that a class supplies ``__getitem__``, ``__len__``, and + ``__iter__`` is insufficient for distinguishing a :class:`Sequence` from + a :class:`Mapping`. .. versionadded:: 3.9 These abstract classes now support ``[]``. See :ref:`types-genericalias` @@ -141,9 +140,13 @@ ABC Inherits from Abstract Methods Mi ``__len__``, ``insert`` +:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods + ``__len__`` + :class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, - ``__len__`` ``__sub__``, ``__xor__``, and ``isdisjoint`` + ``__len__`` ``__sub__``, ``__rsub__``, ``__xor__``, ``__rxor__`` + and ``isdisjoint`` :class:`MutableSet` :class:`Set` ``__contains__``, Inherited :class:`Set` methods and ``__iter__``, ``clear``, ``pop``, ``remove``, ``__ior__``, @@ -162,7 +165,7 @@ ABC Inherits from Abstract Methods Mi ``__len__`` -:class:`MappingView` :class:`Sized` ``__len__`` +:class:`MappingView` :class:`Sized` ``__init__``, ``__len__`` and ``__repr__`` :class:`ItemsView` :class:`MappingView`, ``__contains__``, :class:`Set` ``__iter__`` :class:`KeysView` :class:`MappingView`, ``__contains__``, @@ -260,21 +263,50 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Sequence MutableSequence + ByteString ABCs for read-only and mutable :term:`sequences `. Implementation note: Some of the mixin methods, such as - :meth:`~container.__iter__`, :meth:`~object.__reversed__` and :meth:`index`, make - repeated calls to the underlying :meth:`~object.__getitem__` method. + :meth:`~container.__iter__`, :meth:`~object.__reversed__`, + and :meth:`~sequence.index` make repeated calls to the underlying + :meth:`~object.__getitem__` method. Consequently, if :meth:`~object.__getitem__` is implemented with constant access speed, the mixin methods will have linear performance; however, if the underlying method is linear (as it would be with a linked list), the mixins will have quadratic performance and will likely need to be overridden. - .. versionchanged:: 3.5 - The index() method added support for *stop* and *start* - arguments. + .. method:: index(value, start=0, stop=None) + + Return first index of *value*. + + Raises :exc:`ValueError` if the value is not present. + + Supporting the *start* and *stop* arguments is optional, but recommended. + + .. versionchanged:: 3.5 + The :meth:`~sequence.index` method gained support for + the *stop* and *start* arguments. + + .. deprecated-removed:: 3.12 3.17 + The :class:`ByteString` ABC has been deprecated. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`Buffer` or a union that + explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was + an instance of :class:`!ByteString` never actually told you anything + useful about the object. Other common buffer types such as + :class:`memoryview` were also never understood as subtypes of + :class:`!ByteString` (either at runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. .. class:: Set MutableSet @@ -304,7 +336,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. note:: In CPython, generator-based coroutines (:term:`generators ` - decorated with :func:`@types.coroutine `) are + decorated with :deco:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -322,7 +354,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. note:: In CPython, generator-based coroutines (:term:`generators ` - decorated with :func:`@types.coroutine `) are + decorated with :deco:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 0cc9063f153aba..4e0db485e068a8 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -367,9 +367,11 @@ Several mathematical operations are provided for combining :class:`Counter` objects to produce multisets (counters that have counts greater than zero). Addition and subtraction combine counters by adding or subtracting the counts of corresponding elements. Intersection and union return the minimum and -maximum of corresponding counts. Equality and inclusion compare -corresponding counts. Each operation can accept inputs with signed -counts, but the output will exclude results with counts of zero or less. +maximum of corresponding counts. Symmetric difference returns the difference +between the maximum and minimum of the corresponding counts. Equality and +inclusion compare corresponding counts. Each operation can accept inputs +with signed counts, but the output will exclude results with counts of zero +or below. .. doctest:: @@ -383,6 +385,8 @@ counts, but the output will exclude results with counts of zero or less. Counter({'a': 1, 'b': 1}) >>> c | d # union: max(c[x], d[x]) Counter({'a': 3, 'b': 2}) + >>> c ^ d # max(c[x], d[x]) - min(c[x], d[x]) + Counter({'a': 2, 'b': 1}) >>> c == d # equality: c[x] == d[x] False >>> c <= d # inclusion: c[x] <= d[x] @@ -400,6 +404,9 @@ or subtracting from an empty counter. .. versionadded:: 3.3 Added support for unary plus, unary minus, and in-place multiset operations. +.. versionadded:: 3.15 + Added support for the symmetric difference multiset operation, ``c ^ d``. + .. note:: Counters were primarily designed to work with positive integers to represent @@ -748,8 +755,8 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, returns or raises is then returned or raised by :meth:`~object.__getitem__`. Note that :meth:`__missing__` is *not* called for any operations besides - :meth:`~object.__getitem__`. This means that :meth:`get` will, like normal - dictionaries, return ``None`` as a default rather than using + :meth:`~object.__getitem__`. This means that :meth:`~dict.get` will, like + normal dictionaries, return ``None`` as a default rather than using :attr:`default_factory`. @@ -758,9 +765,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, .. attribute:: default_factory - This attribute is used by the :meth:`__missing__` method; it is - initialized from the first argument to the constructor, if present, or to - ``None``, if absent. + This attribute is used by the :meth:`~defaultdict.__missing__` method; + it is initialized from the first argument to the constructor, if present, + or to ``None``, if absent. .. versionchanged:: 3.9 Added merge (``|``) and update (``|=``) operators, specified in @@ -849,8 +856,9 @@ they add the ability to access fields by name instead of position index. Returns a new tuple subclass named *typename*. The new subclass is used to create tuple-like objects that have fields accessible by attribute lookup as well as being indexable and iterable. Instances of the subclass also have a - helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` - method which lists the tuple contents in a ``name=value`` format. + helpful docstring (with *typename* and *field_names*) and a helpful + :meth:`~object.__repr__` method which lists the tuple contents in a ``name=value`` + format. The *field_names* are a sequence of strings such as ``['x', 'y']``. Alternatively, *field_names* can be a single string with each fieldname @@ -894,10 +902,10 @@ they add the ability to access fields by name instead of position index. Added the *module* parameter. .. versionchanged:: 3.7 - Removed the *verbose* parameter and the :attr:`_source` attribute. + Removed the *verbose* parameter and the :attr:`!_source` attribute. .. versionchanged:: 3.7 - Added the *defaults* parameter and the :attr:`_field_defaults` + Added the *defaults* parameter and the :attr:`~somenamedtuple._field_defaults` attribute. .. doctest:: @@ -1109,7 +1117,7 @@ Some differences from :class:`dict` still remain: A regular :class:`dict` can emulate the order sensitive equality test with ``p == q and all(k1 == k2 for k1, k2 in zip(p, q))``. -* The :meth:`popitem` method of :class:`OrderedDict` has a different +* The :meth:`~OrderedDict.popitem` method of :class:`OrderedDict` has a different signature. It accepts an optional argument to specify which item is popped. A regular :class:`dict` can emulate OrderedDict's ``od.popitem(last=True)`` @@ -1119,7 +1127,7 @@ Some differences from :class:`dict` still remain: with ``(k := next(iter(d)), d.pop(k))`` which will return and remove the leftmost (first) item if it exists. -* :class:`OrderedDict` has a :meth:`move_to_end` method to efficiently +* :class:`OrderedDict` has a :meth:`~OrderedDict.move_to_end` method to efficiently reposition an element to an endpoint. A regular :class:`dict` can emulate OrderedDict's ``od.move_to_end(k, @@ -1130,7 +1138,7 @@ Some differences from :class:`dict` still remain: OrderedDict's ``od.move_to_end(k, last=False)`` which moves the key and its associated value to the leftmost (first) position. -* Until Python 3.8, :class:`dict` lacked a :meth:`__reversed__` method. +* Until Python 3.8, :class:`dict` lacked a :meth:`~object.__reversed__` method. .. class:: OrderedDict([items]) @@ -1185,7 +1193,7 @@ anywhere a regular dictionary is used. .. versionchanged:: 3.6 With the acceptance of :pep:`468`, order is retained for keyword arguments - passed to the :class:`OrderedDict` constructor and its :meth:`update` + passed to the :class:`OrderedDict` constructor and its :meth:`~dict.update` method. .. versionchanged:: 3.9 @@ -1201,7 +1209,7 @@ If a new entry overwrites an existing entry, the original insertion position is changed and moved to the end:: class LastUpdatedOrderedDict(OrderedDict): - 'Store items in the order the keys were last added' + 'Store items in the order that the keys were last updated.' def __setitem__(self, key, value): super().__setitem__(key, value) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index c42288419c4d2d..ebbbf857e717a4 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -56,11 +56,18 @@ compile Python sources. executed. .. option:: -s strip_prefix + + Remove the given prefix from paths recorded in the ``.pyc`` files. + Paths are made relative to the prefix. + + This option can be used with ``-p`` but not with ``-d``. + .. option:: -p prepend_prefix - Remove (``-s``) or append (``-p``) the given prefix of paths - recorded in the ``.pyc`` files. - Cannot be combined with ``-d``. + Prepend the given prefix to paths recorded in the ``.pyc`` files. + Use ``-p /`` to make the paths absolute. + + This option can be used with ``-s`` but not with ``-d``. .. option:: -x regex diff --git a/Doc/library/compression.rst b/Doc/library/compression.rst new file mode 100644 index 00000000000000..98719be9992acc --- /dev/null +++ b/Doc/library/compression.rst @@ -0,0 +1,20 @@ +The :mod:`!compression` package +=============================== + +.. module:: compression + +.. versionadded:: 3.14 + +The :mod:`!compression` package contains the canonical compression modules +containing interfaces to several different compression algorithms. Some of +these modules have historically been available as separate modules; those will +continue to be available under their original names for compatibility reasons, +and will not be removed without a deprecation cycle. The use of modules in +:mod:`!compression` is encouraged where practical. + +* :mod:`!compression.bz2` -- Re-exports :mod:`bz2` +* :mod:`!compression.gzip` -- Re-exports :mod:`gzip` +* :mod:`!compression.lzma` -- Re-exports :mod:`lzma` +* :mod:`!compression.zlib` -- Re-exports :mod:`zlib` +* :mod:`compression.zstd` -- Wrapper for the Zstandard compression library + diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst new file mode 100644 index 00000000000000..7ca843f27f5e9a --- /dev/null +++ b/Doc/library/compression.zstd.rst @@ -0,0 +1,899 @@ +:mod:`!compression.zstd` --- Compression compatible with the Zstandard format +============================================================================= + +.. module:: compression.zstd + :synopsis: Low-level interface to compression and decompression routines in + the zstd library. + +.. versionadded:: 3.14 + +**Source code:** :source:`Lib/compression/zstd/__init__.py` + +-------------- + +This module provides classes and functions for compressing and decompressing +data using the Zstandard (or *zstd*) compression algorithm. The +`zstd manual `__ +describes Zstandard as "a fast lossless compression algorithm, targeting +real-time compression scenarios at zlib-level and better compression ratios." +Also included is a file interface that supports reading and writing the +contents of ``.zst`` files created by the :program:`zstd` utility, as well as +raw zstd compressed streams. + +The :mod:`!compression.zstd` module contains: + +* The :func:`.open` function and :class:`ZstdFile` class for reading and + writing compressed files. +* The :class:`ZstdCompressor` and :class:`ZstdDecompressor` classes for + incremental (de)compression. +* The :func:`compress` and :func:`decompress` functions for one-shot + (de)compression. +* The :func:`train_dict` and :func:`finalize_dict` functions and the + :class:`ZstdDict` class to train and manage Zstandard dictionaries. +* The :class:`CompressionParameter`, :class:`DecompressionParameter`, and + :class:`Strategy` classes for setting advanced (de)compression parameters. + +.. include:: ../includes/optional-module.rst + + +Exceptions +---------- + +.. exception:: ZstdError + + This exception is raised when an error occurs during compression or + decompression, or while initializing the (de)compressor state. + + +Reading and writing compressed files +------------------------------------ + +.. function:: open(file, /, mode='rb', *, level=None, options=None, \ + zstd_dict=None, encoding=None, errors=None, newline=None) + + Open a Zstandard-compressed file in binary or text mode, returning a + :term:`file object`. + + The *file* argument can be either a file name (given as a + :class:`str`, :class:`bytes` or :term:`path-like ` + object), in which case the named file is opened, or it can be an existing + file object to read from or write to. + + The mode argument can be either ``'rb'`` for reading (default), ``'wb'`` for + overwriting, ``'ab'`` for appending, or ``'xb'`` for exclusive creation. + These can equivalently be given as ``'r'``, ``'w'``, ``'a'``, and ``'x'`` + respectively. You may also open in text mode with ``'rt'``, ``'wt'``, + ``'at'``, and ``'xt'`` respectively. + + When reading, the *options* argument can be a dictionary providing advanced + decompression parameters; see :class:`DecompressionParameter` for detailed + information about supported + parameters. The *zstd_dict* argument is a :class:`ZstdDict` instance to be + used during decompression. When reading, if the *level* + argument is not None, a :exc:`!TypeError` will be raised. + + When writing, the *options* argument can be a dictionary + providing advanced compression parameters; see + :class:`CompressionParameter` for detailed information about supported + parameters. The *level* argument is the compression level to use when + writing compressed data. Only one of *level* or *options* may be non-None. + The *zstd_dict* argument is a :class:`ZstdDict` instance to be used during + compression. + + In binary mode, this function is equivalent to the :class:`ZstdFile` + constructor: ``ZstdFile(file, mode, ...)``. In this case, the + *encoding*, *errors*, and *newline* parameters must not be provided. + + In text mode, a :class:`ZstdFile` object is created, and wrapped in an + :class:`io.TextIOWrapper` instance with the specified encoding, error + handling behavior, and line endings. + + +.. class:: ZstdFile(file, /, mode='rb', *, level=None, options=None, \ + zstd_dict=None) + + Open a Zstandard-compressed file in binary mode. + + A :class:`ZstdFile` can wrap an already-open :term:`file object`, or operate + directly on a named file. The *file* argument specifies either the file + object to wrap, or the name of the file to open (as a :class:`str`, + :class:`bytes` or :term:`path-like ` object). If + wrapping an existing file object, the wrapped file will not be closed when + the :class:`ZstdFile` is closed. + + The *mode* argument can be either ``'rb'`` for reading (default), ``'wb'`` + for overwriting, ``'xb'`` for exclusive creation, or ``'ab'`` for appending. + These can equivalently be given as ``'r'``, ``'w'``, ``'x'`` and ``'a'`` + respectively. + + If *file* is a file object (rather than an actual file name), a mode of + ``'w'`` does not truncate the file, and is instead equivalent to ``'a'``. + + When reading, the *options* argument can be a dictionary + providing advanced decompression parameters; see + :class:`DecompressionParameter` for detailed information about supported + parameters. The *zstd_dict* argument is a :class:`ZstdDict` instance to be + used during decompression. When reading, if the *level* + argument is not None, a :exc:`!TypeError` will be raised. + + When writing, the *options* argument can be a dictionary + providing advanced compression parameters; see + :class:`CompressionParameter` for detailed information about supported + parameters. The *level* argument is the compression level to use when + writing compressed data. Only one of *level* or *options* may be passed. The + *zstd_dict* argument is a :class:`ZstdDict` instance to be used during + compression. + + :class:`!ZstdFile` supports all the members specified by + :class:`io.BufferedIOBase`, except for :meth:`~io.BufferedIOBase.detach` + and :meth:`~io.IOBase.truncate`. + Iteration and the :keyword:`with` statement are supported. + + The following method and attributes are also provided: + + .. method:: peek(size=-1) + + Return buffered data without advancing the file position. At least one + byte of data will be returned, unless EOF has been reached. The exact + number of bytes returned is unspecified (the *size* argument is ignored). + + .. note:: While calling :meth:`peek` does not change the file position of + the :class:`ZstdFile`, it may change the position of the underlying + file object (for example, if the :class:`ZstdFile` was constructed by + passing a file object for *file*). + + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. attribute:: name + + The name of the Zstandard file. Equivalent to the :attr:`~io.FileIO.name` + attribute of the underlying :term:`file object`. + + +Compressing and decompressing data in memory +-------------------------------------------- + +.. function:: compress(data, level=None, options=None, zstd_dict=None) + + Compress *data* (a :term:`bytes-like object`), returning the compressed + data as a :class:`bytes` object. + + The *level* argument is an integer controlling the level of + compression. *level* is an alternative to setting + :attr:`CompressionParameter.compression_level` in *options*. Use + :meth:`~CompressionParameter.bounds` on + :attr:`~CompressionParameter.compression_level` to get the values that can + be passed for *level*. If advanced compression options are needed, the + *level* argument must be omitted and in the *options* dictionary the + :attr:`!CompressionParameter.compression_level` parameter should be set. + + The *options* argument is a Python dictionary containing advanced + compression parameters. The valid keys and values for compression parameters + are documented as part of the :class:`CompressionParameter` documentation. + + The *zstd_dict* argument is an instance of :class:`ZstdDict` + containing trained data to improve compression efficiency. The + function :func:`train_dict` can be used to generate a Zstandard dictionary. + + +.. function:: decompress(data, zstd_dict=None, options=None) + + Decompress *data* (a :term:`bytes-like object`), returning the uncompressed + data as a :class:`bytes` object. + + The *options* argument is a Python dictionary containing advanced + decompression parameters. The valid keys and values for compression + parameters are documented as part of the :class:`DecompressionParameter` + documentation. + + The *zstd_dict* argument is an instance of :class:`ZstdDict` + containing trained data used during compression. This must be + the same Zstandard dictionary used during compression. + + If *data* is the concatenation of multiple distinct compressed frames, + decompress all of these frames, and return the concatenation of the results. + + +.. class:: ZstdCompressor(level=None, options=None, zstd_dict=None) + + Create a compressor object, which can be used to compress data + incrementally. + + For a more convenient way of compressing a single chunk of data, see the + module-level function :func:`compress`. + + The *level* argument is an integer controlling the level of + compression. *level* is an alternative to setting + :attr:`CompressionParameter.compression_level` in *options*. Use + :meth:`~CompressionParameter.bounds` on + :attr:`~CompressionParameter.compression_level` to get the values that can + be passed for *level*. If advanced compression options are needed, the + *level* argument must be omitted and in the *options* dictionary the + :attr:`!CompressionParameter.compression_level` parameter should be set. + + The *options* argument is a Python dictionary containing advanced + compression parameters. The valid keys and values for compression parameters + are documented as part of the :class:`CompressionParameter` documentation. + + The *zstd_dict* argument is an optional instance of :class:`ZstdDict` + containing trained data to improve compression efficiency. The + function :func:`train_dict` can be used to generate a Zstandard dictionary. + + + .. method:: compress(data, mode=ZstdCompressor.CONTINUE) + + Compress *data* (a :term:`bytes-like object`), returning a :class:`bytes` + object with compressed data if possible, or otherwise an empty + :class:`!bytes` object. Some of *data* may be buffered internally, for + use in later calls to :meth:`!compress` and :meth:`~.flush`. The returned + data should be concatenated with the output of any previous calls to + :meth:`~.compress`. + + The *mode* argument is a :class:`ZstdCompressor` attribute, either + :attr:`~.CONTINUE`, :attr:`~.FLUSH_BLOCK`, + or :attr:`~.FLUSH_FRAME`. + + When all data has been provided to the compressor, call the + :meth:`~.flush` method to finish the compression process. If + :meth:`~.compress` is called with *mode* set to :attr:`~.FLUSH_FRAME`, + :meth:`~.flush` should not be called, as it would write out a new empty + frame. + + .. method:: flush(mode=ZstdCompressor.FLUSH_FRAME) + + Finish the compression process, returning a :class:`bytes` object + containing any data stored in the compressor's internal buffers. + + The *mode* argument is a :class:`ZstdCompressor` attribute, either + :attr:`~.FLUSH_BLOCK`, or :attr:`~.FLUSH_FRAME`. + + .. method:: set_pledged_input_size(size) + + Specify the amount of uncompressed data *size* that will be provided for + the next frame. *size* will be written into the frame header of the next + frame unless :attr:`CompressionParameter.content_size_flag` is ``False`` + or ``0``. A size of ``0`` means that the frame is empty. If *size* is + ``None``, the frame header will omit the frame size. Frames that include + the uncompressed data size require less memory to decompress, especially + at higher compression levels. + + If :attr:`last_mode` is not :attr:`FLUSH_FRAME`, a + :exc:`ValueError` is raised as the compressor is not at the start of + a frame. If the pledged size does not match the actual size of data + provided to :meth:`.compress`, future calls to :meth:`!compress` or + :meth:`flush` may raise :exc:`ZstdError` and the last chunk of data may + be lost. + + After :meth:`flush` or :meth:`.compress` are called with mode + :attr:`FLUSH_FRAME`, the next frame will not include the frame size into + the header unless :meth:`!set_pledged_input_size` is called again. + + .. attribute:: CONTINUE + + Collect more data for compression, which may or may not generate output + immediately. This mode optimizes the compression ratio by maximizing the + amount of data per block and frame. + + .. attribute:: FLUSH_BLOCK + + Complete and write a block to the data stream. The data returned so far + can be immediately decompressed. Past data can still be referenced in + future blocks generated by calls to :meth:`~.compress`, + improving compression. + + .. attribute:: FLUSH_FRAME + + Complete and write out a frame. Future data provided to + :meth:`~.compress` will be written into a new frame and + *cannot* reference past data. + + .. attribute:: last_mode + + The last mode passed to either :meth:`~.compress` or :meth:`~.flush`. + The value can be one of :attr:`~.CONTINUE`, :attr:`~.FLUSH_BLOCK`, or + :attr:`~.FLUSH_FRAME`. The initial value is :attr:`~.FLUSH_FRAME`, + signifying that the compressor is at the start of a new frame. + + +.. class:: ZstdDecompressor(zstd_dict=None, options=None) + + Create a decompressor object, which can be used to decompress data + incrementally. + + For a more convenient way of decompressing an entire compressed stream at + once, see the module-level function :func:`decompress`. + + The *options* argument is a Python dictionary containing advanced + decompression parameters. The valid keys and values for compression + parameters are documented as part of the :class:`DecompressionParameter` + documentation. + + The *zstd_dict* argument is an instance of :class:`ZstdDict` + containing trained data used during compression. This must be + the same Zstandard dictionary used during compression. + + .. note:: + This class does not transparently handle inputs containing multiple + compressed frames, unlike the :func:`decompress` function and + :class:`ZstdFile` class. To decompress a multi-frame input, you should + use :func:`decompress`, :class:`ZstdFile` if working with a + :term:`file object`, or multiple :class:`!ZstdDecompressor` instances. + + .. method:: decompress(data, max_length=-1) + + Decompress *data* (a :term:`bytes-like object`), returning + uncompressed data as bytes. Some of *data* may be buffered + internally, for use in later calls to :meth:`!decompress`. + The returned data should be concatenated with the output of any previous + calls to :meth:`!decompress`. + + If *max_length* is non-negative, the method returns at most *max_length* + bytes of decompressed data. If this limit is reached and further + output can be produced, the :attr:`~.needs_input` attribute will + be set to ``False``. In this case, the next call to + :meth:`~.decompress` may provide *data* as ``b''`` to obtain + more of the output. + + If all of the input data was decompressed and returned (either + because this was less than *max_length* bytes, or because + *max_length* was negative), the :attr:`~.needs_input` attribute + will be set to ``True``. + + Attempting to decompress data after the end of a frame will raise a + :exc:`ZstdError`. Any data found after the end of the frame is ignored + and saved in the :attr:`~.unused_data` attribute. + + .. attribute:: eof + + ``True`` if the end-of-stream marker has been reached. + + .. attribute:: unused_data + + Data found after the end of the compressed stream. + + Before the end of the stream is reached, this will be ``b''``. + + .. attribute:: needs_input + + ``False`` if the :meth:`.decompress` method can provide more + decompressed data before requiring new compressed input. + + +Zstandard dictionaries +---------------------- + + +.. function:: train_dict(samples, dict_size) + + Train a Zstandard dictionary, returning a :class:`ZstdDict` instance. + Zstandard dictionaries enable more efficient compression of smaller sizes + of data, which is traditionally difficult to compress due to less + repetition. If you are compressing multiple similar groups of data (such as + similar files), Zstandard dictionaries can improve compression ratios and + speed significantly. + + The *samples* argument (an iterable of :class:`bytes` objects), is the + population of samples used to train the Zstandard dictionary. + + The *dict_size* argument, an integer, is the maximum size (in bytes) the + Zstandard dictionary should be. The Zstandard documentation suggests an + absolute maximum of no more than 100 KB, but the maximum can often be smaller + depending on the data. Larger dictionaries generally slow down compression, + but improve compression ratios. Smaller dictionaries lead to faster + compression, but reduce the compression ratio. + + +.. function:: finalize_dict(zstd_dict, /, samples, dict_size, level) + + An advanced function for converting a "raw content" Zstandard dictionary into + a regular Zstandard dictionary. "Raw content" dictionaries are a sequence of + bytes that do not need to follow the structure of a normal Zstandard + dictionary. + + The *zstd_dict* argument is a :class:`ZstdDict` instance with + the :attr:`~ZstdDict.dict_content` containing the raw dictionary contents. + + The *samples* argument (an iterable of :class:`bytes` objects), contains + sample data for generating the Zstandard dictionary. + + The *dict_size* argument, an integer, is the maximum size (in bytes) the + Zstandard dictionary should be. See :func:`train_dict` for + suggestions on the maximum dictionary size. + + The *level* argument (an integer) is the compression level expected to be + passed to the compressors using this dictionary. The dictionary information + varies for each compression level, so tuning for the proper compression + level can make compression more efficient. + + +.. class:: ZstdDict(dict_content, /, *, is_raw=False) + + A wrapper around Zstandard dictionaries. Dictionaries can be used to improve + the compression of many small chunks of data. Use :func:`train_dict` if you + need to train a new dictionary from sample data. + + The *dict_content* argument (a :term:`bytes-like object`), is the already + trained dictionary information. + + The *is_raw* argument, a boolean, is an advanced parameter controlling the + meaning of *dict_content*. ``True`` means *dict_content* is a "raw content" + dictionary, without any format restrictions. ``False`` means *dict_content* + is an ordinary Zstandard dictionary, created from Zstandard functions, + for example, :func:`train_dict` or the external :program:`zstd` CLI. + + When passing a :class:`!ZstdDict` to a function, the + :attr:`!as_digested_dict` and :attr:`!as_undigested_dict` attributes can + control how the dictionary is loaded by passing them as the ``zstd_dict`` + argument, for example, ``compress(data, zstd_dict=zd.as_digested_dict)``. + Digesting a dictionary is a costly operation that occurs when loading a + Zstandard dictionary. When making multiple calls to compression or + decompression, passing a digested dictionary will reduce the overhead of + loading the dictionary. + + .. list-table:: Difference for compression + :widths: 10 14 10 + :header-rows: 1 + + * - + - Digested dictionary + - Undigested dictionary + * - Advanced parameters of the compressor which may be overridden by + the dictionary's parameters + - ``window_log``, ``hash_log``, ``chain_log``, ``search_log``, + ``min_match``, ``target_length``, ``strategy``, + ``enable_long_distance_matching``, ``ldm_hash_log``, + ``ldm_min_match``, ``ldm_bucket_size_log``, ``ldm_hash_rate_log``, + and some non-public parameters. + - None + * - :class:`!ZstdDict` internally caches the dictionary + - Yes. It's faster when loading a digested dictionary again with the + same compression level. + - No. If you wish to load an undigested dictionary multiple times, + consider reusing a compressor object. + + If passing a :class:`!ZstdDict` without any attribute, an undigested + dictionary is passed by default when compressing and a digested dictionary + is generated if necessary and passed by default when decompressing. + + .. attribute:: dict_content + + The content of the Zstandard dictionary, a ``bytes`` object. It's the + same as the *dict_content* argument in the ``__init__`` method. It can + be used with other programs, such as the ``zstd`` CLI program. + + .. attribute:: dict_id + + Identifier of the Zstandard dictionary, a non-negative int value. + + Non-zero means the dictionary is ordinary, created by Zstandard + functions and following the Zstandard format. + + ``0`` means a "raw content" dictionary, free of any format restriction, + used for advanced users. + + .. note:: + + The meaning of ``0`` for :attr:`!ZstdDict.dict_id` is different + from the ``dictionary_id`` attribute to the :func:`get_frame_info` + function. + + .. attribute:: as_digested_dict + + Load as a digested dictionary. + + .. attribute:: as_undigested_dict + + Load as an undigested dictionary. + + +Advanced parameter control +-------------------------- + +.. class:: CompressionParameter() + + An :class:`~enum.IntEnum` containing the advanced compression parameter + keys that can be used when compressing data. + + The :meth:`~.bounds` method can be used on any attribute to get the valid + values for that parameter. + + Parameters are optional; any omitted parameter will have it's value selected + automatically. + + Example getting the lower and upper bound of :attr:`~.compression_level`:: + + lower, upper = CompressionParameter.compression_level.bounds() + + Example setting the :attr:`~.window_log` to the maximum size:: + + _lower, upper = CompressionParameter.window_log.bounds() + options = {CompressionParameter.window_log: upper} + compress(b'venezuelan beaver cheese', options=options) + + .. method:: bounds() + + Return the tuple of int bounds, ``(lower, upper)``, of a compression + parameter. This method should be called on the attribute you wish to + retrieve the bounds of. For example, to get the valid values for + :attr:`~.compression_level`, one may check the result of + ``CompressionParameter.compression_level.bounds()``. + + Both the lower and upper bounds are inclusive. + + .. attribute:: compression_level + + A high-level means of setting other compression parameters that affect + the speed and ratio of compressing data. + + Regular compression levels are greater than ``0``. Values greater than + ``20`` are considered "ultra" compression and require more memory than + other levels. Negative values can be used to trade off faster compression + for worse compression ratios. + + Setting the level to zero uses :attr:`COMPRESSION_LEVEL_DEFAULT`. + + .. attribute:: window_log + + Maximum allowed back-reference distance the compressor can use when + compressing data, expressed as power of two, ``1 << window_log`` bytes. + This parameter greatly influences the memory usage of compression. Higher + values require more memory but gain better compression values. + + A value of zero causes the value to be selected automatically. + + .. attribute:: hash_log + + Size of the initial probe table, as a power of two. The resulting memory + usage is ``1 << (hash_log+2)`` bytes. Larger tables improve compression + ratio of strategies <= :attr:`~Strategy.dfast`, and improve compression + speed of strategies > :attr:`~Strategy.dfast`. + + A value of zero causes the value to be selected automatically. + + .. attribute:: chain_log + + Size of the multi-probe search table, as a power of two. The resulting + memory usage is ``1 << (chain_log+2)`` bytes. Larger tables result in + better and slower compression. This parameter has no effect for the + :attr:`~Strategy.fast` strategy. It's still useful when using + :attr:`~Strategy.dfast` strategy, in which case it defines a secondary + probe table. + + A value of zero causes the value to be selected automatically. + + .. attribute:: search_log + + Number of search attempts, as a power of two. More attempts result in + better and slower compression. This parameter is useless for + :attr:`~Strategy.fast` and :attr:`~Strategy.dfast` strategies. + + A value of zero causes the value to be selected automatically. + + .. attribute:: min_match + + Minimum size of searched matches. Larger values increase compression and + decompression speed, but decrease ratio. Note that Zstandard can still + find matches of smaller size, it just tweaks its search algorithm to look + for this size and larger. For all strategies < :attr:`~Strategy.btopt`, + the effective minimum is ``4``; for all strategies + > :attr:`~Strategy.fast`, the effective maximum is ``6``. + + A value of zero causes the value to be selected automatically. + + .. attribute:: target_length + + The impact of this field depends on the selected :class:`Strategy`. + + For strategies :attr:`~Strategy.btopt`, :attr:`~Strategy.btultra` and + :attr:`~Strategy.btultra2`, the value is the length of a match + considered "good enough" to stop searching. Larger values make + compression ratios better, but compresses slower. + + For strategy :attr:`~Strategy.fast`, it is the distance between match + sampling. Larger values make compression faster, but with a worse + compression ratio. + + A value of zero causes the value to be selected automatically. + + .. attribute:: strategy + + The higher the value of selected strategy, the more complex the + compression technique used by zstd, resulting in higher compression + ratios but slower compression. + + .. seealso:: :class:`Strategy` + + .. attribute:: enable_long_distance_matching + + Long distance matching can be used to improve compression for large + inputs by finding large matches at greater distances. It increases memory + usage and window size. + + ``True`` or ``1`` enable long distance matching while ``False`` or ``0`` + disable it. + + Enabling this parameter increases default + :attr:`~CompressionParameter.window_log` to 128 MiB except when expressly + set to a different value. This setting is enabled by default if + :attr:`!window_log` >= 128 MiB and the compression + strategy >= :attr:`~Strategy.btopt` (compression level 16+). + + .. attribute:: ldm_hash_log + + Size of the table for long distance matching, as a power of two. Larger + values increase memory usage and compression ratio, but decrease + compression speed. + + A value of zero causes the value to be selected automatically. + + .. attribute:: ldm_min_match + + Minimum match size for long distance matcher. Larger or too small values + can often decrease the compression ratio. + + A value of zero causes the value to be selected automatically. + + .. attribute:: ldm_bucket_size_log + + Log size of each bucket in the long distance matcher hash table for + collision resolution. Larger values improve collision resolution but + decrease compression speed. + + A value of zero causes the value to be selected automatically. + + .. attribute:: ldm_hash_rate_log + + Frequency of inserting/looking up entries into the long distance matcher + hash table. Larger values improve compression speed. Deviating far from + the default value will likely result in a compression ratio decrease. + + A value of zero causes the value to be selected automatically. + + .. attribute:: content_size_flag + + Write the size of the data to be compressed into the Zstandard frame + header when known prior to compressing. + + This flag only takes effect under the following scenarios: + + * Calling :func:`compress` for one-shot compression + * Providing all of the data to be compressed in the frame in a single + :meth:`ZstdCompressor.compress` call, with the + :attr:`ZstdCompressor.FLUSH_FRAME` mode. + * Calling :meth:`ZstdCompressor.set_pledged_input_size` with the exact + amount of data that will be provided to the compressor prior to any + calls to :meth:`ZstdCompressor.compress` for the current frame. + :meth:`!ZstdCompressor.set_pledged_input_size` must be called for each + new frame. + + All other compression calls may not write the size information into the + frame header. + + ``True`` or ``1`` enable the content size flag while ``False`` or ``0`` + disable it. + + .. attribute:: checksum_flag + + A four-byte checksum using XXHash64 of the uncompressed content is + written at the end of each frame. Zstandard's decompression code verifies + the checksum. If there is a mismatch a :class:`ZstdError` exception is + raised. + + ``True`` or ``1`` enable checksum generation while ``False`` or ``0`` + disable it. + + .. attribute:: dict_id_flag + + When compressing with a :class:`ZstdDict`, the dictionary's ID is written + into the frame header. + + ``True`` or ``1`` enable storing the dictionary ID while ``False`` or + ``0`` disable it. + + .. attribute:: nb_workers + + Select how many threads will be spawned to compress in parallel. When + :attr:`!nb_workers` > 0, enables multi-threaded compression, a value of + ``1`` means "one-thread multi-threaded mode". More workers improve speed, + but also increase memory usage and slightly reduce compression ratio. + + A value of zero disables multi-threading. + + .. attribute:: job_size + + Size of a compression job, in bytes. This value is enforced only when + :attr:`~CompressionParameter.nb_workers` >= 1. Each compression job is + completed in parallel, so this value can indirectly impact the number of + active threads. + + A value of zero causes the value to be selected automatically. + + .. attribute:: overlap_log + + Sets how much data is reloaded from previous jobs (threads) for new jobs + to be used by the look behind window during compression. This value is + only used when :attr:`~CompressionParameter.nb_workers` >= 1. Acceptable + values vary from 0 to 9. + + * 0 means dynamically set the overlap amount + * 1 means no overlap + * 9 means use a full window size from the previous job + + Each increment halves/doubles the overlap size. "8" means an overlap of + ``window_size/2``, "7" means an overlap of ``window_size/4``, etc. + +.. class:: DecompressionParameter() + + An :class:`~enum.IntEnum` containing the advanced decompression parameter + keys that can be used when decompressing data. Parameters are optional; any + omitted parameter will have it's value selected automatically. + + The :meth:`~.bounds` method can be used on any attribute to get the valid + values for that parameter. + + Example setting the :attr:`~.window_log_max` to the maximum size:: + + data = compress(b'Some very long buffer of bytes...') + + _lower, upper = DecompressionParameter.window_log_max.bounds() + + options = {DecompressionParameter.window_log_max: upper} + decompress(data, options=options) + + .. method:: bounds() + + Return the tuple of int bounds, ``(lower, upper)``, of a decompression + parameter. This method should be called on the attribute you wish to + retrieve the bounds of. + + Both the lower and upper bounds are inclusive. + + .. attribute:: window_log_max + + The base-two logarithm of the maximum size of the window used during + decompression. This can be useful to limit the amount of memory used when + decompressing data. A larger maximum window size leads to faster + decompression. + + A value of zero causes the value to be selected automatically. + + +.. class:: Strategy() + + An :class:`~enum.IntEnum` containing strategies for compression. + Higher-numbered strategies correspond to more complex and slower + compression. + + .. note:: + + The values of attributes of :class:`!Strategy` are not necessarily stable + across zstd versions. Only the ordering of the attributes may be relied + upon. The attributes are listed below in order. + + The following strategies are available: + + .. attribute:: fast + + .. attribute:: dfast + + .. attribute:: greedy + + .. attribute:: lazy + + .. attribute:: lazy2 + + .. attribute:: btlazy2 + + .. attribute:: btopt + + .. attribute:: btultra + + .. attribute:: btultra2 + + +Miscellaneous +------------- + +.. function:: get_frame_info(frame_buffer) + + Retrieve a :class:`FrameInfo` object containing metadata about a Zstandard + frame. Frames contain metadata related to the compressed data they hold. + + +.. class:: FrameInfo + + Metadata related to a Zstandard frame. + + .. attribute:: decompressed_size + + The size of the decompressed contents of the frame. + + .. attribute:: dictionary_id + + An integer representing the Zstandard dictionary ID needed for + decompressing the frame. ``0`` means the dictionary ID was not + recorded in the frame header. This may mean that a Zstandard dictionary + is not needed, or that the ID of a required dictionary was not recorded. + + +.. attribute:: COMPRESSION_LEVEL_DEFAULT + + The default compression level for Zstandard: ``3``. + + +.. attribute:: zstd_version_info + + Version number of the runtime zstd library as a tuple of integers + (major, minor, release). + + +Examples +-------- + +Reading in a compressed file: + +.. code-block:: python + + from compression import zstd + + with zstd.open("file.zst") as f: + file_content = f.read() + +Creating a compressed file: + +.. code-block:: python + + from compression import zstd + + data = b"Insert Data Here" + with zstd.open("file.zst", "w") as f: + f.write(data) + +Compressing data in memory: + +.. code-block:: python + + from compression import zstd + + data_in = b"Insert Data Here" + data_out = zstd.compress(data_in) + +Incremental compression: + +.. code-block:: python + + from compression import zstd + + comp = zstd.ZstdCompressor() + out1 = comp.compress(b"Some data\n") + out2 = comp.compress(b"Another piece of data\n") + out3 = comp.compress(b"Even more data\n") + out4 = comp.flush() + # Concatenate all the partial results: + result = b"".join([out1, out2, out3, out4]) + +Writing compressed data to an already-open file: + +.. code-block:: python + + from compression import zstd + + with open("myfile", "wb") as f: + f.write(b"This data will not be compressed\n") + with zstd.open(f, "w") as zstf: + zstf.write(b"This *will* be compressed\n") + f.write(b"Not compressed\n") + +Creating a compressed file using compression parameters: + +.. code-block:: python + + from compression import zstd + + options = { + zstd.CompressionParameter.checksum_flag: 1 + } + with zstd.open("file.zst", "w", options=options) as f: + f.write(b"Mind if I squeeze in?") diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 5be1a1106b09a0..18f9443cbfea20 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -18,6 +18,7 @@ multitasking). Here's an overview: multiprocessing.shared_memory.rst concurrent.rst concurrent.futures.rst + concurrent.interpreters.rst subprocess.rst sched.rst queue.rst diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 48e027152a9851..e4b505c3f9761e 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -6,8 +6,9 @@ .. versionadded:: 3.2 -**Source code:** :source:`Lib/concurrent/futures/thread.py` -and :source:`Lib/concurrent/futures/process.py` +**Source code:** :source:`Lib/concurrent/futures/thread.py`, +:source:`Lib/concurrent/futures/process.py`, +and :source:`Lib/concurrent/futures/interpreter.py` -------------- @@ -20,6 +21,11 @@ or separate processes, using :class:`ProcessPoolExecutor`. Each implements the same interface, which is defined by the abstract :class:`Executor` class. +:class:`concurrent.futures.Future` must not be confused with +:class:`asyncio.Future`, which is designed for use with :mod:`asyncio` +tasks and coroutines. See the :doc:`asyncio's Future ` +documentation for a detailed comparison of the two. + .. include:: ../includes/wasm-notavail.rst Executor Objects @@ -40,11 +46,14 @@ Executor Objects future = executor.submit(pow, 323, 1235) print(future.result()) - .. method:: map(fn, *iterables, timeout=None, chunksize=1) + .. method:: map(fn, *iterables, timeout=None, chunksize=1, buffersize=None) Similar to :func:`map(fn, *iterables) ` except: - * the *iterables* are collected immediately rather than lazily; + * The *iterables* are collected immediately rather than lazily, unless a + *buffersize* is specified to limit the number of submitted tasks whose + results have not yet been yielded. If the buffer is full, iteration over + the *iterables* pauses until a result is yielded from the buffer. * *fn* is executed asynchronously and several calls to *fn* may be made concurrently. @@ -68,7 +77,10 @@ Executor Objects *chunksize* has no effect. .. versionchanged:: 3.5 - Added the *chunksize* argument. + Added the *chunksize* parameter. + + .. versionchanged:: 3.14 + Added the *buffersize* parameter. .. method:: shutdown(wait=True, *, cancel_futures=False) @@ -94,10 +106,10 @@ Executor Objects executor has started running will be completed prior to this method returning. The remaining futures are cancelled. - You can avoid having to call this method explicitly if you use the - :keyword:`with` statement, which will shutdown the :class:`Executor` - (waiting as if :meth:`Executor.shutdown` were called with *wait* set to - ``True``):: + You can avoid having to call this method explicitly if you use the executor + as a :term:`context manager` via the :keyword:`with` statement, which + will shutdown the :class:`Executor` (waiting as if :meth:`Executor.shutdown` + were called with *wait* set to ``True``):: import shutil with ThreadPoolExecutor(max_workers=4) as e: @@ -232,6 +244,8 @@ ThreadPoolExecutor Example InterpreterPoolExecutor ----------------------- +.. versionadded:: 3.14 + The :class:`InterpreterPoolExecutor` class uses a pool of interpreters to execute calls asynchronously. It is a :class:`ThreadPoolExecutor` subclass, which means each worker is running in its own thread. @@ -252,13 +266,13 @@ This results in several benefits that help balance the extra effort, including true multi-core parallelism, For example, code written this way can make it easier to reason about concurrency. Another major benefit is that you don't have to deal with several of the -big pain points of using threads, like nrace conditions. +big pain points of using threads, like race conditions. Each worker's interpreter is isolated from all the other interpreters. "Isolated" means each interpreter has its own runtime state and operates completely independently. For example, if you redirect :data:`sys.stdout` in one interpreter, it will not be automatically -redirected any other interpreter. If you import a module in one +redirected to any other interpreter. If you import a module in one interpreter, it is not automatically imported in any other. You would need to import the module separately in interpreter where you need it. In fact, each module imported in an interpreter is @@ -280,7 +294,7 @@ efficient alternative is to serialize with :mod:`pickle` and then send the bytes over a shared :mod:`socket ` or :func:`pipe `. -.. class:: InterpreterPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=(), shared=None) +.. class:: InterpreterPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=()) A :class:`ThreadPoolExecutor` subclass that executes calls asynchronously using a pool of at most *max_workers* threads. Each thread runs @@ -292,25 +306,14 @@ the bytes over a shared :mod:`socket ` or The optional *initializer* and *initargs* arguments have the same meaning as for :class:`!ThreadPoolExecutor`: the initializer is run - when each worker is created, though in this case it is run.in + when each worker is created, though in this case it is run in the worker's interpreter. The executor serializes the *initializer* and *initargs* using :mod:`pickle` when sending them to the worker's interpreter. - .. note:: - Functions defined in the ``__main__`` module cannot be pickled - and thus cannot be used. - .. note:: The executor may replace uncaught exceptions from *initializer* - with :class:`~concurrent.futures.interpreter.ExecutionFailed`. - - The optional *shared* argument is a :class:`dict` of objects that all - interpreters in the pool share. The *shared* items are added to each - interpreter's ``__main__`` module. Not all objects are shareable. - Shareable objects include the builtin singletons, :class:`str` - and :class:`bytes`, and :class:`memoryview`. See :pep:`734` - for more info. + with :class:`~concurrent.interpreters.ExecutionFailed`. Other caveats from parent :class:`ThreadPoolExecutor` apply here. @@ -319,18 +322,14 @@ except the worker serializes the callable and arguments using :mod:`pickle` when sending them to its interpreter. The worker likewise serializes the return value when sending it back. -.. note:: - Functions defined in the ``__main__`` module cannot be pickled - and thus cannot be used. - When a worker's current task raises an uncaught exception, the worker always tries to preserve the exception as-is. If that is successful then it also sets the ``__cause__`` to a corresponding -:class:`~concurrent.futures.interpreter.ExecutionFailed` +:class:`~concurrent.interpreters.ExecutionFailed` instance, which contains a summary of the original exception. In the uncommon case that the worker is not able to preserve the original as-is then it directly preserves the corresponding -:class:`~concurrent.futures.interpreter.ExecutionFailed` +:class:`~concurrent.interpreters.ExecutionFailed` instance instead. @@ -350,6 +349,11 @@ that :class:`ProcessPoolExecutor` will not work in the interactive interpreter. Calling :class:`Executor` or :class:`Future` methods from a callable submitted to a :class:`ProcessPoolExecutor` will result in deadlock. +Note that the restrictions on functions and arguments needing to picklable as +per :class:`multiprocessing.Process` apply when using :meth:`~Executor.submit` +and :meth:`~Executor.map` on a :class:`ProcessPoolExecutor`. A function defined +in a REPL or a lambda should not be expected to work. + .. class:: ProcessPoolExecutor(max_workers=None, mp_context=None, initializer=None, initargs=(), max_tasks_per_child=None) An :class:`Executor` subclass that executes calls asynchronously using a pool @@ -380,6 +384,11 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. default in absence of a *mp_context* parameter. This feature is incompatible with the "fork" start method. + .. note:: + Bugs have been reported when using the *max_tasks_per_child* feature that + can result in the :class:`ProcessPoolExecutor` hanging in some + circumstances. Follow its eventual resolution in :gh:`115634`. + .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a :exc:`~concurrent.futures.process.BrokenProcessPool` error is now raised. @@ -415,6 +424,30 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. require the *fork* start method for :class:`ProcessPoolExecutor` you must explicitly pass ``mp_context=multiprocessing.get_context("fork")``. + .. method:: terminate_workers() + + Attempt to terminate all living worker processes immediately by calling + :meth:`Process.terminate ` on each of them. + Internally, it will also call :meth:`Executor.shutdown` to ensure that all + other resources associated with the executor are freed. + + After calling this method the caller should no longer submit tasks to the + executor. + + .. versionadded:: 3.14 + + .. method:: kill_workers() + + Attempt to kill all living worker processes immediately by calling + :meth:`Process.kill ` on each of them. + Internally, it will also call :meth:`Executor.shutdown` to ensure that all + other resources associated with the executor are freed. + + After calling this method the caller should no longer submit tasks to the + executor. + + .. versionadded:: 3.14 + .. _processpoolexecutor-example: ProcessPoolExecutor Example @@ -690,16 +723,7 @@ Exception classes of a :class:`~concurrent.futures.InterpreterPoolExecutor` has failed initializing. - .. versionadded:: next - -.. exception:: ExecutionFailed - - Raised from :class:`~concurrent.futures.InterpreterPoolExecutor` when - the given initializer fails or from - :meth:`~concurrent.futures.Executor.submit` when there's an uncaught - exception from the submitted task. - - .. versionadded:: next + .. versionadded:: 3.14 .. currentmodule:: concurrent.futures.process diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst new file mode 100644 index 00000000000000..55036090e8d5b8 --- /dev/null +++ b/Doc/library/concurrent.interpreters.rst @@ -0,0 +1,387 @@ +:mod:`!concurrent.interpreters` --- Multiple interpreters in the same process +============================================================================= + +.. module:: concurrent.interpreters + :synopsis: Multiple interpreters in the same process + +.. moduleauthor:: Eric Snow +.. sectionauthor:: Eric Snow + +.. versionadded:: 3.14 + +**Source code:** :source:`Lib/concurrent/interpreters` + +-------------- + +The :mod:`!concurrent.interpreters` module constructs higher-level +interfaces on top of the lower level :mod:`!_interpreters` module. + +The module is primarily meant to provide a basic API for managing +interpreters (AKA "subinterpreters") and running things in them. +Running mostly involves switching to an interpreter (in the current +thread) and calling a function in that execution context. + +For concurrency, interpreters themselves (and this module) don't +provide much more than isolation, which on its own isn't useful. +Actual concurrency is available separately through +:mod:`threads ` See `below `_ + +.. seealso:: + + :class:`~concurrent.futures.InterpreterPoolExecutor` + Combines threads with interpreters in a familiar interface. + + .. XXX Add references to the upcoming HOWTO docs in the seealso block. + + :ref:`isolating-extensions-howto` + How to update an extension module to support multiple interpreters. + + :pep:`554` + + :pep:`734` + + :pep:`684` + +.. XXX Why do we disallow multiple interpreters on WASM? + +.. include:: ../includes/wasm-notavail.rst + + +Key details +----------- + +Before we dive in further, there are a small number of details +to keep in mind about using multiple interpreters: + +* `isolated `_, by default +* no implicit threads +* not all PyPI packages support use in multiple interpreters yet + +.. XXX Are there other relevant details to list? + + +.. _interpreters-intro: + +Introduction +------------ + +An "interpreter" is effectively the execution context of the Python +runtime. It contains all of the state the runtime needs to execute +a program. This includes things like the import state and builtins. +(Each thread, even if there's only the main thread, has some extra +runtime state, in addition to the current interpreter, related to +the current exception and the bytecode eval loop.) + +The concept and functionality of the interpreter have been a part of +Python since version 2.2, but the feature was only available through +the C-API and not well known, and the `isolation `_ +was relatively incomplete until version 3.12. + +.. _interp-isolation: + +Multiple Interpreters and Isolation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A Python implementation may support using multiple interpreters in the +same process. CPython has this support. Each interpreter is +effectively isolated from the others (with a limited number of +carefully managed process-global exceptions to the rule). + +That isolation is primarily useful as a strong separation between +distinct logical components of a program, where you want to have +careful control of how those components interact. + +.. note:: + + Interpreters in the same process can technically never be strictly + isolated from one another since there are few restrictions on memory + access within the same process. The Python runtime makes a best + effort at isolation but extension modules may easily violate that. + Therefore, do not use multiple interpreters in security-sensitive + situations, where they shouldn't have access to each other's data. + +Running in an Interpreter +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Running in a different interpreter involves switching to it in the +current thread and then calling some function. The runtime will +execute the function using the current interpreter's state. The +:mod:`!concurrent.interpreters` module provides a basic API for +creating and managing interpreters, as well as the switch-and-call +operation. + +No other threads are automatically started for the operation. +There is `a helper `_ for that though. +There is another dedicated helper for calling the builtin +:func:`exec` in an interpreter. + +When :func:`exec` (or :func:`eval`) are called in an interpreter, +they run using the interpreter's :mod:`!__main__` module as the +"globals" namespace. The same is true for functions that aren't +associated with any module. This is the same as how scripts invoked +from the command-line run in the :mod:`!__main__` module. + + +.. _interp-concurrency: + +Concurrency and Parallelism +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As noted earlier, interpreters do not provide any concurrency +on their own. They strictly represent the isolated execution +context the runtime will use *in the current thread*. That isolation +makes them similar to processes, but they still enjoy in-process +efficiency, like threads. + +All that said, interpreters do naturally support certain flavors of +concurrency. +There's a powerful side effect of that isolation. It enables a +different approach to concurrency than you can take with async or +threads. It's a similar concurrency model to CSP or the actor model, +a model which is relatively easy to reason about. + +You can take advantage of that concurrency model in a single thread, +switching back and forth between interpreters, Stackless-style. +However, this model is more useful when you combine interpreters +with multiple threads. This mostly involves starting a new thread, +where you switch to another interpreter and run what you want there. + +Each actual thread in Python, even if you're only running in the main +thread, has its own *current* execution context. Multiple threads can +use the same interpreter or different ones. + +At a high level, you can think of the combination of threads and +interpreters as threads with opt-in sharing. + +As a significant bonus, interpreters are sufficiently isolated that +they do not share the :term:`GIL`, which means combining threads with +multiple interpreters enables full multi-core parallelism. +(This has been the case since Python 3.12.) + +Communication Between Interpreters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In practice, multiple interpreters are useful only if we have a way +to communicate between them. This usually involves some form of +message passing, but can even mean sharing data in some carefully +managed way. + +With this in mind, the :mod:`!concurrent.interpreters` module provides +a :class:`queue.Queue` implementation, available through +:func:`create_queue`. + +.. _interp-object-sharing: + +"Sharing" Objects +^^^^^^^^^^^^^^^^^ + +Any data actually shared between interpreters loses the thread-safety +provided by the :term:`GIL`. There are various options for dealing with +this in extension modules. However, from Python code the lack of +thread-safety means objects can't actually be shared, with a few +exceptions. Instead, a copy must be created, which means mutable +objects won't stay in sync. + +By default, most objects are copied with :mod:`pickle` when they are +passed to another interpreter. Nearly all of the immutable builtin +objects are either directly shared or copied efficiently. For example: + +* :const:`None` +* :class:`bool` (:const:`True` and :const:`False`) +* :class:`bytes` +* :class:`str` +* :class:`int` +* :class:`float` +* :class:`tuple` (of similarly supported objects) + +There is a small number of Python types that actually share mutable +data between interpreters: + +* :class:`memoryview` +* :class:`Queue` + + +Reference +--------- + +This module defines the following functions: + +.. function:: list_all() + + Return a :class:`list` of :class:`Interpreter` objects, + one for each existing interpreter. + +.. function:: get_current() + + Return an :class:`Interpreter` object for the currently running + interpreter. + +.. function:: get_main() + + Return an :class:`Interpreter` object for the main interpreter. + This is the interpreter the runtime created to run the :term:`REPL` + or the script given at the command-line. It is usually the only one. + +.. function:: create() + + Initialize a new (idle) Python interpreter + and return a :class:`Interpreter` object for it. + +.. function:: create_queue() + + Initialize a new cross-interpreter queue and return a :class:`Queue` + object for it. + + +Interpreter objects +^^^^^^^^^^^^^^^^^^^ + +.. class:: Interpreter(id) + + A single interpreter in the current process. + + Generally, :class:`Interpreter` shouldn't be called directly. + Instead, use :func:`create` or one of the other module functions. + + .. attribute:: id + + (read-only) + + The underlying interpreter's ID. + + .. attribute:: whence + + (read-only) + + A string describing where the interpreter came from. + + .. method:: is_running() + + Return ``True`` if the interpreter is currently executing code + in its :mod:`!__main__` module and ``False`` otherwise. + + .. method:: close() + + Finalize and destroy the interpreter. + + .. method:: prepare_main(ns=None, **kwargs) + + Bind objects in the interpreter's :mod:`!__main__` module. + + Some objects are actually shared and some are copied efficiently, + but most are copied via :mod:`pickle`. See :ref:`interp-object-sharing`. + + .. method:: exec(code, /, dedent=True) + + Run the given source code in the interpreter (in the current thread). + + .. method:: call(callable, /, *args, **kwargs) + + Return the result of calling running the given function in the + interpreter (in the current thread). + + .. _interp-call-in-thread: + + .. method:: call_in_thread(callable, /, *args, **kwargs) + + Run the given function in the interpreter (in a new thread). + +Exceptions +^^^^^^^^^^ + +.. exception:: InterpreterError + + This exception, a subclass of :exc:`Exception`, is raised when + an interpreter-related error happens. + +.. exception:: InterpreterNotFoundError + + This exception, a subclass of :exc:`InterpreterError`, is raised when + the targeted interpreter no longer exists. + +.. exception:: ExecutionFailed + + This exception, a subclass of :exc:`InterpreterError`, is raised when + the running code raised an uncaught exception. + + .. attribute:: excinfo + + A basic snapshot of the exception raised in the other interpreter. + +.. XXX Document the excinfoattrs? + +.. exception:: NotShareableError + + This exception, a subclass of :exc:`TypeError`, is raised when + an object cannot be sent to another interpreter. + + +Communicating Between Interpreters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: Queue(id) + + A wrapper around a low-level, cross-interpreter queue, which + implements the :class:`queue.Queue` interface. The underlying queue + can only be created through :func:`create_queue`. + + Some objects are actually shared and some are copied efficiently, + but most are copied via :mod:`pickle`. See :ref:`interp-object-sharing`. + + .. attribute:: id + + (read-only) + + The queue's ID. + + +.. exception:: QueueEmptyError + + This exception, a subclass of :exc:`queue.Empty`, is raised from + :meth:`!Queue.get` and :meth:`!Queue.get_nowait` when the queue + is empty. + +.. exception:: QueueFullError + + This exception, a subclass of :exc:`queue.Full`, is raised from + :meth:`!Queue.put` and :meth:`!Queue.put_nowait` when the queue + is full. + + +Basic usage +----------- + +Creating an interpreter and running code in it:: + + from concurrent import interpreters + + interp = interpreters.create() + + # Run in the current OS thread. + + interp.exec('print("spam!")') + + interp.exec("""if True: + print('spam!') + """) + + from textwrap import dedent + interp.exec(dedent(""" + print('spam!') + """)) + + def run(arg): + return arg + + res = interp.call(run, 'spam!') + print(res) + + def run(): + print('spam!') + + interp.call(run) + + # Run in new OS thread. + + t = interp.call_in_thread(run) + t.join() diff --git a/Doc/library/concurrent.rst b/Doc/library/concurrent.rst index 8caea78bbb57e8..748c72c733bba2 100644 --- a/Doc/library/concurrent.rst +++ b/Doc/library/concurrent.rst @@ -1,6 +1,7 @@ The :mod:`!concurrent` package ============================== -Currently, there is only one module in this package: +This package contains the following modules: * :mod:`concurrent.futures` -- Launching parallel tasks +* :mod:`concurrent.interpreters` -- Multiple interpreters in the same process diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 3aad6f7b5d2d20..bb109a9b742cb7 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -942,7 +942,13 @@ interpolation if an option used is not defined elsewhere. :: ConfigParser Objects -------------------- -.. class:: ConfigParser(defaults=None, dict_type=dict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation(), converters={}) +.. class:: ConfigParser(defaults=None, dict_type=dict, allow_no_value=False, *, \ + delimiters=('=', ':'), comment_prefixes=('#', ';'), \ + inline_comment_prefixes=None, strict=True, \ + empty_lines_in_values=True, \ + default_section=configparser.DEFAULTSECT, \ + interpolation=BasicInterpolation(), converters={}, \ + allow_unnamed_section=False) The main configuration parser. When *defaults* is given, it is initialized into the dictionary of intrinsic defaults. When *dict_type* is given, it @@ -990,6 +996,10 @@ ConfigParser Objects converter gets its own corresponding :meth:`!get*` method on the parser object and section proxies. + When *allow_unnamed_section* is ``True`` (default: ``False``), + the first section name can be omitted. See the + `"Unnamed Sections" section <#unnamed-sections>`_. + It is possible to read several configurations into a single :class:`ConfigParser`, where the most recently added configuration has the highest priority. Any conflicting keys are taken from the more recent @@ -1039,6 +1049,9 @@ ConfigParser Objects Raise a :exc:`MultilineContinuationError` when *allow_no_value* is ``True``, and a key without a value is continued with an indented line. + .. versionchanged:: 3.13 + The *allow_unnamed_section* argument was added. + .. method:: defaults() Return a dictionary containing the instance-wide defaults. @@ -1231,6 +1244,10 @@ ConfigParser Objects *space_around_delimiters* is true, delimiters between keys and values are surrounded by spaces. + .. versionchanged:: 3.14 + Raises InvalidWriteError if this would write a representation which cannot + be accurately parsed by a future :meth:`read` call from this parser. + .. note:: Comments in the original configuration file are not preserved when @@ -1295,18 +1312,30 @@ RawConfigParser Objects comment_prefixes=('#', ';'), \ inline_comment_prefixes=None, strict=True, \ empty_lines_in_values=True, \ - default_section=configparser.DEFAULTSECT[, \ - interpolation]) + default_section=configparser.DEFAULTSECT, \ + interpolation=BasicInterpolation(), converters={}, \ + allow_unnamed_section=False) Legacy variant of the :class:`ConfigParser`. It has interpolation disabled by default and allows for non-string section names, option names, and values via its unsafe ``add_section`` and ``set`` methods, as well as the legacy ``defaults=`` keyword argument handling. + .. versionchanged:: 3.2 + *allow_no_value*, *delimiters*, *comment_prefixes*, *strict*, + *empty_lines_in_values*, *default_section* and *interpolation* were + added. + + .. versionchanged:: 3.5 + The *converters* argument was added. + .. versionchanged:: 3.8 The default *dict_type* is :class:`dict`, since it now preserves insertion order. + .. versionchanged:: 3.13 + The *allow_unnamed_section* argument was added. + .. note:: Consider using :class:`ConfigParser` instead which checks types of the values to be stored internally. If you don't want interpolation, you @@ -1434,6 +1463,17 @@ Exceptions .. versionadded:: 3.14 +.. exception:: InvalidWriteError + + Exception raised when an attempted :meth:`ConfigParser.write` would not be parsed + accurately with a future :meth:`ConfigParser.read` call. + + Ex: Writing a key beginning with the :attr:`ConfigParser.SECTCRE` pattern + would parse as a section header when read. Attempting to write this will raise + this exception. + + .. versionadded:: 3.14 + .. rubric:: Footnotes .. [1] Config parsers allow for heavy customization. If you are interested in diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 04bb8c51a3b197..d058ba206c6cd6 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -46,11 +46,12 @@ A small number of constants live in the built-in namespace. They are: See :ref:`implementing-the-arithmetic-operations` for examples. - .. note:: + .. caution:: - ``NotImplementedError`` and :data:`!NotImplemented` are not interchangeable, - even though they have similar names and purposes. - See :exc:`NotImplementedError` for details on when to use it. + :data:`!NotImplemented` and :exc:`!NotImplementedError` are not + interchangeable. This constant should only be used as described + above; see :exc:`NotImplementedError` for details on correct usage + of the exception. .. versionchanged:: 3.9 Evaluating :data:`!NotImplemented` in a boolean context was deprecated. @@ -64,8 +65,9 @@ A small number of constants live in the built-in namespace. They are: .. index:: single: ...; ellipsis literal .. data:: Ellipsis - The same as the ellipsis literal "``...``". Special value used mostly in conjunction - with extended slicing syntax for user-defined container data types. + The same as the ellipsis literal "``...``", an object frequently used to + indicate that something is omitted. Assignment to ``Ellipsis`` is possible, but + assignment to ``...`` raises a :exc:`SyntaxError`. ``Ellipsis`` is the sole instance of the :data:`types.EllipsisType` type. @@ -96,15 +98,17 @@ should not be used in programs. exit(code=None) Objects that when printed, print a message like "Use quit() or Ctrl-D - (i.e. EOF) to exit", and when called, raise :exc:`SystemExit` with the + (i.e. EOF) to exit", and when accessed directly in the interactive + interpreter or called as functions, raise :exc:`SystemExit` with the specified exit code. .. data:: help :noindex: Object that when printed, prints the message "Type help() for interactive - help, or help(object) for help about object.", and when called, - acts as described :func:`elsewhere `. + help, or help(object) for help about object.", and when accessed directly + in the interactive interpreter, invokes the built-in help system + (see :func:`help`). .. data:: copyright credits diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index f5b349441bcfee..d0fa645093a3e9 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -49,7 +49,7 @@ Functions and classes provided: While many objects natively support use in with statements, sometimes a resource needs to be managed that isn't a context manager in its own right, - and doesn't implement a ``close()`` method for use with ``contextlib.closing`` + and doesn't implement a ``close()`` method for use with ``contextlib.closing``. An abstract example would be the following to ensure correct resource management:: @@ -151,9 +151,9 @@ Functions and classes provided: created by :func:`asynccontextmanager` to meet the requirement that context managers support multiple invocations in order to be used as decorators. - .. versionchanged:: 3.10 - Async context managers created with :func:`asynccontextmanager` can - be used as decorators. + .. versionchanged:: 3.10 + Async context managers created with :func:`asynccontextmanager` can + be used as decorators. .. function:: closing(thing) @@ -327,10 +327,10 @@ Functions and classes provided: .. function:: redirect_stdout(new_target) Context manager for temporarily redirecting :data:`sys.stdout` to - another file or file-like object. + another :term:`file object`. This tool adds flexibility to existing functions or classes whose output - is hardwired to stdout. + is hardwired to :data:`sys.stdout`. For example, the output of :func:`help` normally is sent to *sys.stdout*. You can capture that output in a string by redirecting the output to an @@ -366,8 +366,8 @@ Functions and classes provided: .. function:: redirect_stderr(new_target) - Similar to :func:`~contextlib.redirect_stdout` but redirecting - :data:`sys.stderr` to another file or file-like object. + Similar to :func:`~contextlib.redirect_stdout` but redirecting the global + :data:`sys.stderr` to another :term:`file object`. This context manager is :ref:`reentrant `. @@ -629,7 +629,8 @@ Functions and classes provided: The :meth:`~ExitStack.close` method is not implemented; :meth:`aclose` must be used instead. - .. coroutinemethod:: enter_async_context(cm) + .. method:: enter_async_context(cm) + :async: Similar to :meth:`ExitStack.enter_context` but expects an asynchronous context manager. @@ -647,7 +648,8 @@ Functions and classes provided: Similar to :meth:`ExitStack.callback` but expects a coroutine function. - .. coroutinemethod:: aclose() + .. method:: aclose() + :async: Similar to :meth:`ExitStack.close` but properly handles awaitables. diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 2b1fb9fdd29cd8..653d8b597c2362 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -77,6 +77,32 @@ Context Variables to restore the variable to its previous value via the :meth:`ContextVar.reset` method. + For convenience, the token object can be used as a context manager + to avoid calling :meth:`ContextVar.reset` manually:: + + var = ContextVar('var', default='default value') + + with var.set('new value'): + assert var.get() == 'new value' + + assert var.get() == 'default value' + + It is a shorthand for:: + + var = ContextVar('var', default='default value') + + token = var.set('new value') + try: + assert var.get() == 'new value' + finally: + var.reset(token) + + assert var.get() == 'default value' + + .. versionadded:: 3.14 + + Added support for using tokens as context managers. + .. method:: reset(token) Reset the context variable to the value it had before the @@ -93,13 +119,22 @@ Context Variables # After the reset call the var has no value again, so # var.get() would raise a LookupError. + The same *token* cannot be used twice. + .. class:: Token *Token* objects are returned by the :meth:`ContextVar.set` method. They can be passed to the :meth:`ContextVar.reset` method to revert the value of the variable to what it was before the corresponding - *set*. + *set*. A single token cannot reset a context variable more than once. + + Tokens support the :ref:`context manager protocol ` + to automatically reset context variables. See :meth:`ContextVar.set`. + + .. versionadded:: 3.14 + + Added support for usage as a context manager. .. attribute:: Token.var @@ -298,7 +333,7 @@ client:: addr = writer.transport.get_extra_info('socket').getpeername() client_addr_var.set(addr) - # In any code that we call is now possible to get + # In any code that we call, it is now possible to get the # client's address by calling 'client_addr_var.get()'. while True: diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 95b41f988a035b..210ad7188003e6 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -122,6 +122,8 @@ and only supports named tuples created by :func:`~collections.namedtuple`, This method should create a new object of the same type, replacing fields with values from *changes*. + .. versionadded:: 3.13 + .. seealso:: diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst new file mode 100644 index 00000000000000..647cb4925f31da --- /dev/null +++ b/Doc/library/crypt.rst @@ -0,0 +1,20 @@ +:mod:`!crypt` --- Function to check Unix passwords +================================================== + +.. module:: crypt + :synopsis: Removed in 3.13. + :deprecated: + +.. deprecated-removed:: 3.11 3.13 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.13 ` after +being deprecated in Python 3.11. The removal was decided in :pep:`594`. + +Applications can use the :mod:`hashlib` module from the standard library. +Other possible replacements are third-party libraries from PyPI: +:pypi:`legacycrypt`, :pypi:`bcrypt`, or :pypi:`argon2-cffi`. +These are not supported or maintained by the Python core team. + +The last version of Python that provided the :mod:`!crypt` module was +`Python 3.12 `_. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 533cdf13974be6..4a033d823e6a7e 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -53,7 +53,7 @@ The :mod:`csv` module defines the following functions: .. index:: single: universal newlines; csv.reader function -.. function:: reader(csvfile, dialect='excel', **fmtparams) +.. function:: reader(csvfile, /, dialect='excel', **fmtparams) Return a :ref:`reader object ` that will process lines from the given *csvfile*. A csvfile must be an iterable of @@ -70,7 +70,7 @@ The :mod:`csv` module defines the following functions: section :ref:`csv-fmt-params`. Each row read from the csv file is returned as a list of strings. No - automatic data type conversion is performed unless the ``QUOTE_NONNUMERIC`` format + automatic data type conversion is performed unless the :data:`QUOTE_NONNUMERIC` format option is specified (in which case unquoted fields are transformed into floats). A short usage example:: @@ -84,7 +84,7 @@ The :mod:`csv` module defines the following functions: Spam, Lovely Spam, Wonderful Spam -.. function:: writer(csvfile, dialect='excel', **fmtparams) +.. function:: writer(csvfile, /, dialect='excel', **fmtparams) Return a writer object responsible for converting the user's data into delimited strings on the given file-like object. *csvfile* can be any object with a @@ -113,7 +113,7 @@ The :mod:`csv` module defines the following functions: spamwriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam']) -.. function:: register_dialect(name[, dialect[, **fmtparams]]) +.. function:: register_dialect(name, /, dialect='excel', **fmtparams) Associate *dialect* with *name*. *name* must be a string. The dialect can be specified either by passing a sub-class of :class:`Dialect`, or @@ -139,7 +139,8 @@ The :mod:`csv` module defines the following functions: Return the names of all registered dialects. -.. function:: field_size_limit([new_limit]) +.. function:: field_size_limit() + field_size_limit(new_limit) Returns the current maximum field size allowed by the parser. If *new_limit* is given, this becomes the new limit. @@ -294,8 +295,8 @@ The :mod:`csv` module defines the following classes: - the second through n-th rows contain strings where at least one value's length differs from that of the putative header of that column. - Twenty rows after the first row are sampled; if more than half of columns + - rows meet the criteria, :const:`True` is returned. + Twenty-one rows after the header are sampled; if more than half of the + columns + rows meet the criteria, :const:`True` is returned. .. note:: @@ -323,23 +324,32 @@ The :mod:`csv` module defines the following constants: .. data:: QUOTE_MINIMAL Instructs :class:`writer` objects to only quote those fields which contain - special characters such as *delimiter*, *quotechar* or any of the characters in - *lineterminator*. + special characters such as *delimiter*, *quotechar*, ``'\r'``, ``'\n'`` + or any of the characters in *lineterminator*. .. data:: QUOTE_NONNUMERIC Instructs :class:`writer` objects to quote all non-numeric fields. - Instructs :class:`reader` objects to convert all non-quoted fields to type *float*. + Instructs :class:`reader` objects to convert all non-quoted fields to type :class:`float`. + .. note:: + Some numeric types, such as :class:`bool`, :class:`~fractions.Fraction`, + or :class:`~enum.IntEnum`, have a string representation that cannot be + converted to :class:`float`. + They cannot be read in the :data:`QUOTE_NONNUMERIC` and + :data:`QUOTE_STRINGS` modes. .. data:: QUOTE_NONE - Instructs :class:`writer` objects to never quote fields. When the current - *delimiter* occurs in output data it is preceded by the current *escapechar* - character. If *escapechar* is not set, the writer will raise :exc:`Error` if + Instructs :class:`writer` objects to never quote fields. + When the current *delimiter*, *quotechar*, *escapechar*, ``'\r'``, ``'\n'`` + or any of the characters in *lineterminator* occurs in output data + it is preceded by the current *escapechar* character. + If *escapechar* is not set, the writer will raise :exc:`Error` if any characters that require escaping are encountered. + Set *quotechar* to ``None`` to prevent its escaping. Instructs :class:`reader` objects to perform no special processing of quote characters. @@ -408,9 +418,16 @@ Dialects support the following attributes: .. attribute:: Dialect.escapechar - A one-character string used by the writer to escape the *delimiter* if *quoting* - is set to :const:`QUOTE_NONE` and the *quotechar* if *doublequote* is - :const:`False`. On reading, the *escapechar* removes any special meaning from + A one-character string used by the writer to escape characters that + require escaping: + + * the *delimiter*, the *quotechar*, ``'\r'``, ``'\n'`` and any of the + characters in *lineterminator* are escaped if *quoting* is set to + :const:`QUOTE_NONE`; + * the *quotechar* is escaped if *doublequote* is :const:`False`; + * the *escapechar* itself. + + On reading, the *escapechar* removes any special meaning from the following character. It defaults to :const:`None`, which disables escaping. .. versionchanged:: 3.11 @@ -430,9 +447,12 @@ Dialects support the following attributes: .. attribute:: Dialect.quotechar - A one-character string used to quote fields containing special characters, such - as the *delimiter* or *quotechar*, or which contain new-line characters. It - defaults to ``'"'``. + A one-character string used to quote fields containing special characters, + such as the *delimiter* or the *quotechar*, or which contain new-line + characters (``'\r'``, ``'\n'`` or any of the characters in *lineterminator*). + It defaults to ``'"'``. + Can be set to ``None`` to prevent escaping ``'"'`` if *quoting* is set + to :const:`QUOTE_NONE`. .. versionchanged:: 3.11 An empty *quotechar* is not allowed. @@ -441,13 +461,15 @@ Dialects support the following attributes: Controls when quotes should be generated by the writer and recognised by the reader. It can take on any of the :ref:`QUOTE_\* constants ` - and defaults to :const:`QUOTE_MINIMAL`. + and defaults to :const:`QUOTE_MINIMAL` if *quotechar* is not ``None``, + and :const:`QUOTE_NONE` otherwise. .. attribute:: Dialect.skipinitialspace When :const:`True`, spaces immediately following the *delimiter* are ignored. - The default is :const:`False`. + The default is :const:`False`. When combining ``delimiter=' '`` with + ``skipinitialspace=True``, unquoted empty fields are not allowed. .. attribute:: Dialect.strict @@ -506,7 +528,7 @@ out surrounded by parens. This may cause some problems for other programs which read CSV files (assuming they support complex numbers at all). -.. method:: csvwriter.writerow(row) +.. method:: csvwriter.writerow(row, /) Write the *row* parameter to the writer's file object, formatted according to the current :class:`Dialect`. Return the return value of the call to the @@ -515,7 +537,7 @@ read CSV files (assuming they support complex numbers at all). .. versionchanged:: 3.5 Added support of arbitrary iterables. -.. method:: csvwriter.writerows(rows) +.. method:: csvwriter.writerows(rows, /) Write all elements in *rows* (an iterable of *row* objects as described above) to the writer's file object, formatted according to the current @@ -603,7 +625,7 @@ A slightly more advanced use of the reader --- catching and reporting errors:: for row in reader: print(row) except csv.Error as e: - sys.exit('file {}, line {}: {}'.format(filename, reader.line_num, e)) + sys.exit(f'file {filename}, line {reader.line_num}: {e}') And while the module doesn't directly support parsing strings, it can easily be done:: @@ -616,7 +638,7 @@ done:: .. rubric:: Footnotes .. [1] If ``newline=''`` is not specified, newlines embedded inside quoted fields - will not be interpreted correctly, and on platforms that use ``\r\n`` linendings + will not be interpreted correctly, and on platforms that use ``\r\n`` line endings on write an extra ``\r`` will be added. It should always be safe to specify ``newline=''``, since the csv module does its own (:term:`universal `) newline handling. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 99909de20ef439..6038af99009d02 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -14,6 +14,8 @@ data types, and allows calling functions in DLLs or shared libraries. It can be used to wrap these libraries in pure Python. +.. include:: ../includes/optional-module.rst + .. _ctypes-ctypes-tutorial: @@ -232,8 +234,24 @@ Fundamental data types +----------------------+------------------------------------------+----------------------------+ | :class:`c_int` | :c:expr:`int` | int | +----------------------+------------------------------------------+----------------------------+ +| :class:`c_int8` | :c:type:`int8_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int16` | :c:type:`int16_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int32` | :c:type:`int32_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int64` | :c:type:`int64_t` | int | ++----------------------+------------------------------------------+----------------------------+ | :class:`c_uint` | :c:expr:`unsigned int` | int | +----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint8` | :c:type:`uint8_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint16` | :c:type:`uint16_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint32` | :c:type:`uint32_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint64` | :c:type:`uint64_t` | int | ++----------------------+------------------------------------------+----------------------------+ | :class:`c_long` | :c:expr:`long` | int | +----------------------+------------------------------------------+----------------------------+ | :class:`c_ulong` | :c:expr:`unsigned long` | int | @@ -266,8 +284,8 @@ Fundamental data types (1) The constructor accepts any object with a truth value. -Additionally, if IEC 60559 compatible complex arithmetic (Annex G) is supported, the following -complex types are available: +Additionally, if IEC 60559 compatible complex arithmetic (Annex G) is supported +in both C and ``libffi``, the following complex types are available: +----------------------------------+---------------------------------+-----------------+ | ctypes type | C type | Python type | @@ -306,7 +324,7 @@ Since these types are mutable, their value can also be changed afterwards:: Assigning a new value to instances of the pointer types :class:`c_char_p`, :class:`c_wchar_p`, and :class:`c_void_p` changes the *memory location* they point to, *not the contents* of the memory block (of course not, because Python -bytes objects are immutable):: +string objects are immutable):: >>> s = "Hello, World" >>> c_s = c_wchar_p(s) @@ -657,12 +675,13 @@ Nested structures can also be initialized in the constructor in several ways:: >>> r = RECT((1, 2), (3, 4)) Field :term:`descriptor`\s can be retrieved from the *class*, they are useful -for debugging because they can provide useful information:: +for debugging because they can provide useful information. +See :class:`CField`:: - >>> print(POINT.x) - - >>> print(POINT.y) - + >>> POINT.x + + >>> POINT.y + >>> @@ -683,14 +702,10 @@ compiler does it. It is possible to override this behavior entirely by specifyi :attr:`~Structure._layout_` class attribute in the subclass definition; see the attribute documentation for details. -It is possible to specify the maximum alignment for the fields by setting -the :attr:`~Structure._pack_` class attribute to a positive integer. -This matches what ``#pragma pack(n)`` does in MSVC. - -It is also possible to set a minimum alignment for how the subclass itself is packed in the -same way ``#pragma align(n)`` works in MSVC. -This can be achieved by specifying a ::attr:`~Structure._align_` class attribute -in the subclass definition. +It is possible to specify the maximum alignment for the fields and/or for the +structure itself by setting the class attributes :attr:`~Structure._pack_` +and/or :attr:`~Structure._align_`, respectively. +See the attribute documentation for details. :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the @@ -713,10 +728,16 @@ item in the :attr:`~Structure._fields_` tuples:: ... ("second_16", c_int, 16)] ... >>> print(Int.first_16) - + >>> print(Int.second_16) - - >>> + + +It is important to note that bit field allocation and layout in memory are not +defined as a C standard; their implementation is compiler-specific. +By default, Python will attempt to match the behavior of a "native" compiler +for the current platform. +See the :attr:`~Structure._layout_` attribute for details on the default +behavior and how to change it. .. _ctypes-arrays: @@ -870,6 +891,36 @@ invalid non-\ ``NULL`` pointers would crash Python):: ValueError: NULL pointer access >>> +.. _ctypes-thread-safety: + +Thread safety without the GIL +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +From Python 3.13 onward, the :term:`GIL` can be disabled on :term:`free threaded ` builds. +In ctypes, reads and writes to a single object concurrently is safe, but not across multiple objects: + + .. code-block:: pycon + + >>> number = c_int(42) + >>> pointer_a = pointer(number) + >>> pointer_b = pointer(number) + +In the above, it's only safe for one object to read and write to the address at once if the GIL is disabled. +So, ``pointer_a`` can be shared and written to across multiple threads, but only if ``pointer_b`` +is not also attempting to do the same. If this is an issue, consider using a :class:`threading.Lock` +to synchronize access to memory: + + .. code-block:: pycon + + >>> import threading + >>> lock = threading.Lock() + >>> # Thread 1 + >>> with lock: + ... pointer_a.contents = 24 + >>> # Thread 2 + >>> with lock: + ... pointer_b.contents = 42 + .. _ctypes-type-conversions: @@ -1337,6 +1388,9 @@ On Linux, :func:`~ctypes.util.find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the library file. It returns the filename of the library file. +Note that if the output of these programs does not correspond to the dynamic +linker used by Python, the result of this function may be misleading. + .. versionchanged:: 3.6 On Linux, the value of the environment variable ``LD_LIBRARY_PATH`` is used when searching for libraries, if a library cannot be found by any other means. @@ -1376,6 +1430,28 @@ the shared library name at development time, and hardcode that into the wrapper module instead of using :func:`~ctypes.util.find_library` to locate the library at runtime. +.. _ctypes-listing-loaded-shared-libraries: + +Listing loaded shared libraries +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When writing code that relies on code loaded from shared libraries, it can be +useful to know which shared libraries have already been loaded into the current +process. + +The :mod:`!ctypes.util` module provides the :func:`~ctypes.util.dllist` function, +which calls the different APIs provided by the various platforms to help determine +which shared libraries have already been loaded into the current process. + +The exact output of this function will be system dependent. On most platforms, +the first entry of this list represents the current process itself, which may +be an empty string. +For example, on glibc-based Linux, the return may look like:: + + >>> from ctypes.util import dllist + >>> dllist() + ['', 'linux-vdso.so.1', '/lib/x86_64-linux-gnu/libm.so.6', '/lib/x86_64-linux-gnu/libc.so.6', ... ] + .. _ctypes-loading-shared-libraries: Loading shared libraries @@ -1413,13 +1489,15 @@ way is to instantiate one of the following classes: .. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - Windows only: Instances of this class represent loaded shared libraries, + Instances of this class represent loaded shared libraries, functions in these libraries use the ``stdcall`` calling convention, and are assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT` values contain information specifying whether the function call failed or succeeded, together with additional error code. If the return value signals a failure, an :class:`OSError` is automatically raised. + .. availability:: Windows + .. versionchanged:: 3.3 :exc:`WindowsError` used to be raised, which is now an alias of :exc:`OSError`. @@ -1431,14 +1509,17 @@ way is to instantiate one of the following classes: .. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - Windows only: Instances of this class represent loaded shared libraries, + Instances of this class represent loaded shared libraries, functions in these libraries use the ``stdcall`` calling convention, and are assumed to return :c:expr:`int` by default. + .. availability:: Windows + .. versionchanged:: 3.12 The *name* parameter can now be a :term:`path-like object`. + The Python :term:`global interpreter lock` is released before calling any function exported by these libraries, and reacquired afterwards. @@ -1574,13 +1655,17 @@ These prefabricated library loaders are available: .. data:: windll :noindex: - Windows only: Creates :class:`WinDLL` instances. + Creates :class:`WinDLL` instances. + + .. availability:: Windows .. data:: oledll :noindex: - Windows only: Creates :class:`OleDLL` instances. + Creates :class:`OleDLL` instances. + + .. availability:: Windows .. data:: pydll @@ -1707,12 +1792,6 @@ in :mod:`!ctypes`) which inherits from the private :class:`_CFuncPtr` class: and raise an exception if the foreign function call failed. -.. exception:: ArgumentError - - This exception is raised when a foreign function call cannot convert one of the - passed arguments. - - .. audit-event:: ctypes.set_exception code foreign-functions On Windows, when a foreign function call raises a system exception (for @@ -1723,7 +1802,8 @@ in :mod:`!ctypes`) which inherits from the private :class:`_CFuncPtr` class: .. audit-event:: ctypes.call_function func_pointer,arguments foreign-functions - Some ways to invoke foreign function calls may raise an auditing event + Some ways to invoke foreign function calls as well as some of the + functions in this module may raise an auditing event ``ctypes.call_function`` with arguments ``function pointer`` and ``arguments``. .. _ctypes-function-prototypes: @@ -1752,11 +1832,13 @@ See :ref:`ctypes-callback-functions` for examples. .. function:: WINFUNCTYPE(restype, *argtypes, use_errno=False, use_last_error=False) - Windows only: The returned function prototype creates functions that use the + The returned function prototype creates functions that use the ``stdcall`` calling convention. The function will release the GIL during the call. *use_errno* and *use_last_error* have the same meaning as above. + .. availability:: Windows + .. function:: PYFUNCTYPE(restype, *argtypes) @@ -1799,10 +1881,17 @@ different ways, depending on the type and number of the parameters in the call: integer. *name* is name of the COM method. *iid* is an optional pointer to the interface identifier which is used in extended error reporting. + If *iid* is not specified, an :exc:`OSError` is raised if the COM method + call fails. If *iid* is specified, a :exc:`~ctypes.COMError` is raised + instead. + COM methods use a special calling convention: They require a pointer to the COM interface as first argument, in addition to those parameters that are specified in the :attr:`!argtypes` tuple. + .. availability:: Windows + + The optional *paramflags* parameter creates foreign function wrappers with much more functionality than the features described above. @@ -1939,6 +2028,24 @@ Utility functions It behaves similar to ``pointer(obj)``, but the construction is a lot faster. +.. function:: CopyComPointer(src, dst) + + Copies a COM pointer from *src* to *dst* and returns the Windows specific + :c:type:`!HRESULT` value. + + If *src* is not ``NULL``, its ``AddRef`` method is called, incrementing the + reference count. + + In contrast, the reference count of *dst* will not be decremented before + assigning the new value. Unless *dst* is ``NULL``, the caller is responsible + for decrementing the reference count by calling its ``Release`` method when + necessary. + + .. availability:: Windows + + .. versionadded:: 3.14 + + .. function:: cast(obj, type) This function is similar to the cast operator in C. It returns a new instance @@ -1947,52 +2054,76 @@ Utility functions pointer. -.. function:: create_string_buffer(init_or_size, size=None) +.. function:: create_string_buffer(init, size=None) + create_string_buffer(size) This function creates a mutable character buffer. The returned object is a ctypes array of :class:`c_char`. - *init_or_size* must be an integer which specifies the size of the array, or a - bytes object which will be used to initialize the array items. + If *size* is given (and not ``None``), it must be an :class:`int`. + It specifies the size of the returned array. + + If the *init* argument is given, it must be :class:`bytes`. It is used + to initialize the array items. Bytes not initialized this way are + set to zero (NUL). + + If *size* is not given (or if it is ``None``), the buffer is made one element + larger than *init*, effectively adding a NUL terminator. + + If both arguments are given, *size* must not be less than ``len(init)``. - If a bytes object is specified as first argument, the buffer is made one item - larger than its length so that the last element in the array is a NUL - termination character. An integer can be passed as second argument which allows - specifying the size of the array if the length of the bytes should not be used. + .. warning:: + + If *size* is equal to ``len(init)``, a NUL terminator is + not added. Do not treat such a buffer as a C string. + + For example:: + + >>> bytes(create_string_buffer(2)) + b'\x00\x00' + >>> bytes(create_string_buffer(b'ab')) + b'ab\x00' + >>> bytes(create_string_buffer(b'ab', 2)) + b'ab' + >>> bytes(create_string_buffer(b'ab', 4)) + b'ab\x00\x00' + >>> bytes(create_string_buffer(b'abcdef', 2)) + Traceback (most recent call last): + ... + ValueError: byte string too long .. audit-event:: ctypes.create_string_buffer init,size ctypes.create_string_buffer -.. function:: create_unicode_buffer(init_or_size, size=None) +.. function:: create_unicode_buffer(init, size=None) + create_unicode_buffer(size) This function creates a mutable unicode character buffer. The returned object is a ctypes array of :class:`c_wchar`. - *init_or_size* must be an integer which specifies the size of the array, or a - string which will be used to initialize the array items. - - If a string is specified as first argument, the buffer is made one item - larger than the length of the string so that the last element in the array is a - NUL termination character. An integer can be passed as second argument which - allows specifying the size of the array if the length of the string should not - be used. + The function takes the same arguments as :func:`~create_string_buffer` except + *init* must be a string and *size* counts :class:`c_wchar`. .. audit-event:: ctypes.create_unicode_buffer init,size ctypes.create_unicode_buffer .. function:: DllCanUnloadNow() - Windows only: This function is a hook which allows implementing in-process + This function is a hook which allows implementing in-process COM servers with ctypes. It is called from the DllCanUnloadNow function that the _ctypes extension dll exports. + .. availability:: Windows + .. function:: DllGetClassObject() - Windows only: This function is a hook which allows implementing in-process + This function is a hook which allows implementing in-process COM servers with ctypes. It is called from the DllGetClassObject function that the ``_ctypes`` extension dll exports. + .. availability:: Windows + .. function:: find_library(name) :module: ctypes.util @@ -2004,11 +2135,13 @@ Utility functions The exact functionality is system dependent. + See :ref:`ctypes-finding-shared-libraries` for complete documentation. + .. function:: find_msvcrt() :module: ctypes.util - Windows only: return the filename of the VC runtime library used by Python, + Returns the filename of the VC runtime library used by Python, and by the extension modules. If the name of the library cannot be determined, ``None`` is returned. @@ -2016,20 +2149,41 @@ Utility functions with a call to the ``free(void *)``, it is important that you use the function in the same library that allocated the memory. + .. availability:: Windows + + +.. function:: dllist() + :module: ctypes.util + + Try to provide a list of paths of the shared libraries loaded into the current + process. These paths are not normalized or processed in any way. The function + can raise :exc:`OSError` if the underlying platform APIs fail. + The exact functionality is system dependent. + + On most platforms, the first element of the list represents the current + executable file. It may be an empty string. + + .. availability:: Windows, macOS, iOS, glibc, BSD libc, musl + .. versionadded:: 3.14 .. function:: FormatError([code]) - Windows only: Returns a textual description of the error code *code*. If no - error code is specified, the last error code is used by calling the Windows - api function GetLastError. + Returns a textual description of the error code *code*. If no error code is + specified, the last error code is used by calling the Windows API function + :func:`GetLastError`. + + .. availability:: Windows .. function:: GetLastError() - Windows only: Returns the last error code set by Windows in the calling thread. + Returns the last error code set by Windows in the calling thread. This function calls the Windows ``GetLastError()`` function directly, it does not return the ctypes-private copy of the error code. + .. availability:: Windows + + .. function:: get_errno() Returns the current value of the ctypes-private copy of the system @@ -2039,11 +2193,14 @@ Utility functions .. function:: get_last_error() - Windows only: returns the current value of the ctypes-private copy of the system + Returns the current value of the ctypes-private copy of the system :data:`!LastError` variable in the calling thread. + .. availability:: Windows + .. audit-event:: ctypes.get_last_error "" ctypes.get_last_error + .. function:: memmove(dst, src, count) Same as the standard C memmove library function: copies *count* bytes from @@ -2060,10 +2217,20 @@ Utility functions .. function:: POINTER(type, /) - Create and return a new ctypes pointer type. Pointer types are cached and + Create or return a ctypes pointer type. Pointer types are cached and reused internally, so calling this function repeatedly is cheap. *type* must be a ctypes type. + .. impl-detail:: + + The resulting pointer type is cached in the ``__pointer_type__`` + attribute of *type*. + It is possible to set this attribute before the first call to + ``POINTER`` in order to set a custom pointer type. + However, doing this is discouraged: manually creating a suitable + pointer type is difficult without relying on implementation + details that may change in future Python versions. + .. function:: pointer(obj, /) @@ -2092,10 +2259,12 @@ Utility functions .. function:: set_last_error(value) - Windows only: set the current value of the ctypes-private copy of the system + Sets the current value of the ctypes-private copy of the system :data:`!LastError` variable in the calling thread to *value* and return the previous value. + .. availability:: Windows + .. audit-event:: ctypes.set_last_error error ctypes.set_last_error @@ -2116,12 +2285,13 @@ Utility functions .. function:: WinError(code=None, descr=None) - Windows only: this function is probably the worst-named thing in ctypes. It - creates an instance of :exc:`OSError`. If *code* is not specified, - ``GetLastError`` is called to determine the error code. If *descr* is not + Creates an instance of :exc:`OSError`. If *code* is not specified, + :func:`GetLastError` is called to determine the error code. If *descr* is not specified, :func:`FormatError` is called to get a textual description of the error. + .. availability:: Windows + .. versionchanged:: 3.3 An instance of :exc:`WindowsError` used to be created, which is now an alias of :exc:`OSError`. @@ -2137,6 +2307,28 @@ Utility functions .. audit-event:: ctypes.wstring_at ptr,size ctypes.wstring_at +.. function:: memoryview_at(ptr, size, readonly=False) + + Return a :class:`memoryview` object of length *size* that references memory + starting at *void \*ptr*. + + If *readonly* is true, the returned :class:`!memoryview` object can + not be used to modify the underlying memory. + (Changes made by other means will still be reflected in the returned + object.) + + This function is similar to :func:`string_at` with the key + difference of not making a copy of the specified memory. + It is a semantically equivalent (but more efficient) alternative to + ``memoryview((c_byte * size).from_address(ptr))``. + (While :meth:`~_CData.from_address` only takes integers, *ptr* can also + be given as a :class:`ctypes.POINTER` or a :func:`~ctypes.byref` object.) + + .. audit-event:: ctypes.memoryview_at address,size,readonly + + .. versionadded:: 3.14 + + .. _ctypes-data-types: Data types @@ -2203,6 +2395,16 @@ Data types library. *name* is the name of the symbol that exports the data, *library* is the loaded shared library. + Common class variables of ctypes data types: + + .. attribute:: __pointer_type__ + + The pointer type that was created by calling + :func:`POINTER` for corresponding ctypes data type. If a pointer type + was not yet created, the attribute is missing. + + .. versionadded:: 3.14 + Common instance variables of ctypes data types: .. attribute:: _b_base_ @@ -2341,7 +2543,7 @@ These are the fundamental ctypes data types: .. class:: c_int8 - Represents the C 8-bit :c:expr:`signed int` datatype. Usually an alias for + Represents the C 8-bit :c:expr:`signed int` datatype. It is an alias for :class:`c_byte`. @@ -2416,7 +2618,7 @@ These are the fundamental ctypes data types: .. class:: c_uint8 - Represents the C 8-bit :c:expr:`unsigned int` datatype. Usually an alias for + Represents the C 8-bit :c:expr:`unsigned int` datatype. It is an alias for :class:`c_ubyte`. @@ -2485,15 +2687,20 @@ These are the fundamental ctypes data types: .. class:: HRESULT - Windows only: Represents a :c:type:`!HRESULT` value, which contains success or + Represents a :c:type:`!HRESULT` value, which contains success or error information for a function or method call. + .. availability:: Windows + .. class:: py_object Represents the C :c:expr:`PyObject *` datatype. Calling this without an argument creates a ``NULL`` :c:expr:`PyObject *` pointer. + .. versionchanged:: 3.14 + :class:`!py_object` is now a :term:`generic type`. + The :mod:`!ctypes.wintypes` module provides quite some other Windows specific data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, or :c:type:`!DWORD`. Some useful structures like :c:type:`!MSG` or :c:type:`!RECT` are also defined. @@ -2588,16 +2795,46 @@ fields, or any other data types containing pointer type fields. .. attribute:: _pack_ An optional small integer that allows overriding the alignment of - structure fields in the instance. :attr:`_pack_` must already be defined + structure fields in the instance. + + This is only implemented for the MSVC-compatible memory layout + (see :attr:`_layout_`). + + Setting :attr:`!_pack_` to 0 is the same as not setting it at all. + Otherwise, the value must be a positive power of two. + The effect is equivalent to ``#pragma pack(N)`` in C, except + :mod:`ctypes` may allow larger *n* than what the compiler accepts. + + :attr:`!_pack_` must already be defined when :attr:`_fields_` is assigned, otherwise it will have no effect. - Setting this attribute to 0 is the same as not setting it at all. + .. deprecated-removed:: 3.14 3.19 + + For historical reasons, if :attr:`!_pack_` is non-zero, + the MSVC-compatible layout will be used by default. + On non-Windows platforms, this default is deprecated and is slated to + become an error in Python 3.19. + If it is intended, set :attr:`~Structure._layout_` to ``'ms'`` + explicitly. .. attribute:: _align_ - An optional small integer that allows overriding the alignment of + An optional small integer that allows increasing the alignment of the structure when being packed or unpacked to/from memory. - Setting this attribute to 0 is the same as not setting it at all. + + The value must not be negative. + The effect is equivalent to ``__attribute__((aligned(N)))`` on GCC + or ``#pragma align(N)`` on MSVC, except :mod:`ctypes` may allow + values that the compiler would reject. + + :attr:`!_align_` can only *increase* a structure's alignment + requirements. Setting it to 0 or 1 has no effect. + + Using values that are not powers of two is discouraged and may lead to + surprising behavior. + + :attr:`!_align_` must already be defined + when :attr:`_fields_` is assigned, otherwise it will have no effect. .. versionadded:: 3.13 @@ -2620,12 +2857,15 @@ fields, or any other data types containing pointer type fields. Currently the default will be: - On Windows: ``"ms"`` - - When :attr:`~Structure._pack_` is specified: ``"ms"`` + - When :attr:`~Structure._pack_` is specified: ``"ms"``. + (This is deprecated; see :attr:`~Structure._pack_` documentation.) - Otherwise: ``"gcc-sysv"`` :attr:`!_layout_` must already be defined when :attr:`~Structure._fields_` is assigned, otherwise it will have no effect. + .. versionadded:: 3.14 + .. attribute:: _anonymous_ An optional sequence that lists the names of unnamed (anonymous) fields. @@ -2675,6 +2915,98 @@ fields, or any other data types containing pointer type fields. present in :attr:`_fields_`. +.. class:: CField(*args, **kw) + + Descriptor for fields of a :class:`Structure` and :class:`Union`. + For example:: + + >>> class Color(Structure): + ... _fields_ = ( + ... ('red', c_uint8), + ... ('green', c_uint8), + ... ('blue', c_uint8), + ... ('intense', c_bool, 1), + ... ('blinking', c_bool, 1), + ... ) + ... + >>> Color.red + + >>> Color.green.type + + >>> Color.blue.byte_offset + 2 + >>> Color.intense + + >>> Color.blinking.bit_offset + 1 + + All attributes are read-only. + + :class:`!CField` objects are created via :attr:`~Structure._fields_`; + do not instantiate the class directly. + + .. versionadded:: 3.14 + + Previously, descriptors only had ``offset`` and ``size`` attributes + and a readable string representation; the :class:`!CField` class was not + available directly. + + .. attribute:: name + + Name of the field, as a string. + + .. attribute:: type + + Type of the field, as a :ref:`ctypes class `. + + .. attribute:: offset + byte_offset + + Offset of the field, in bytes. + + For bitfields, this is the offset of the underlying byte-aligned + *storage unit*; see :attr:`~CField.bit_offset`. + + .. attribute:: byte_size + + Size of the field, in bytes. + + For bitfields, this is the size of the underlying *storage unit*. + Typically, it has the same size as the bitfield's type. + + .. attribute:: size + + For non-bitfields, equivalent to :attr:`~CField.byte_size`. + + For bitfields, this contains a backwards-compatible bit-packed + value that combines :attr:`~CField.bit_size` and + :attr:`~CField.bit_offset`. + Prefer using the explicit attributes instead. + + .. attribute:: is_bitfield + + True if this is a bitfield. + + .. attribute:: bit_offset + bit_size + + The location of a bitfield within its *storage unit*, that is, within + :attr:`~CField.byte_size` bytes of memory starting at + :attr:`~CField.byte_offset`. + + To get the field's value, read the storage unit as an integer, + :ref:`shift left ` by :attr:`!bit_offset` and + take the :attr:`!bit_size` least significant bits. + + For non-bitfields, :attr:`!bit_offset` is zero + and :attr:`!bit_size` is equal to ``byte_size * 8``. + + .. attribute:: is_anonymous + + True if this field is anonymous, that is, it contains nested sub-fields + that should be merged into a containing structure or union. + + .. _ctypes-arrays-pointers: Arrays and pointers @@ -2741,3 +3073,41 @@ Arrays and pointers Returns the object to which to pointer points. Assigning to this attribute changes the pointer to point to the assigned object. + + +.. _ctypes-exceptions: + +Exceptions +^^^^^^^^^^ + +.. exception:: ArgumentError + + This exception is raised when a foreign function call cannot convert one of the + passed arguments. + + +.. exception:: COMError(hresult, text, details) + + This exception is raised when a COM method call failed. + + .. attribute:: hresult + + The integer value representing the error code. + + .. attribute:: text + + The error message. + + .. attribute:: details + + The 5-tuple ``(descr, source, helpfile, helpcontext, progid)``. + + *descr* is the textual description. *source* is the language-dependent + ``ProgID`` for the class or application that raised the error. *helpfile* + is the path of the help file. *helpcontext* is the help context + identifier. *progid* is the ``ProgID`` of the interface that defined the + error. + + .. availability:: Windows + + .. versionadded:: 3.14 diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 6c7fc721a3e0fb..057d338edda92a 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -23,6 +23,8 @@ Linux and the BSD variants of Unix. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + .. note:: Whenever the documentation mentions a *character* it can be specified @@ -68,6 +70,21 @@ The module :mod:`curses` defines the following exception: The module :mod:`curses` defines the following functions: +.. function:: assume_default_colors(fg, bg, /) + + Allow use of default values for colors on terminals supporting this feature. + Use this to support transparency in your application. + + * Assign terminal default foreground/background colors to color number ``-1``. + So ``init_pair(x, COLOR_RED, -1)`` will initialize pair *x* as red + on default background and ``init_pair(x, -1, COLOR_BLUE)`` will + initialize pair *x* as default foreground on blue. + + * Change the definition of the color-pair ``0`` to ``(fg, bg)``. + + .. versionadded:: 3.14 + + .. function:: baudrate() Return the output speed of the terminal in bits per second. On software @@ -290,9 +307,11 @@ The module :mod:`curses` defines the following functions: Change the definition of a color-pair. It takes three arguments: the number of the color-pair to be changed, the foreground color number, and the background color number. The value of *pair_number* must be between ``1`` and - ``COLOR_PAIRS - 1`` (the ``0`` color pair is wired to white on black and cannot - be changed). The value of *fg* and *bg* arguments must be between ``0`` and - ``COLORS - 1``, or, after calling :func:`use_default_colors`, ``-1``. + ``COLOR_PAIRS - 1`` (the ``0`` color pair can only be changed by + :func:`use_default_colors` and :func:`assume_default_colors`). + The value of *fg* and *bg* arguments must be between ``0`` and + ``COLORS - 1``, or, after calling :func:`!use_default_colors` or + :func:`!assume_default_colors`, ``-1``. If the color-pair was previously initialized, the screen is refreshed and all occurrences of that color-pair are changed to the new definition. @@ -678,11 +697,7 @@ The module :mod:`curses` defines the following functions: .. function:: use_default_colors() - Allow use of default values for colors on terminals supporting this feature. Use - this to support transparency in your application. The default color is assigned - to the color number ``-1``. After calling this function, ``init_pair(x, - curses.COLOR_RED, -1)`` initializes, for instance, color pair *x* to a red - foreground color on the default background. + Equivalent to ``assume_default_colors(-1, -1)``. .. function:: wrapper(func, /, *args, **kwargs) @@ -703,8 +718,10 @@ The module :mod:`curses` defines the following functions: Window Objects -------------- -Window objects, as returned by :func:`initscr` and :func:`newwin` above, have -the following methods and attributes: +.. class:: window + + Window objects, as returned by :func:`initscr` and :func:`newwin` above, have + the following methods and attributes: .. method:: window.addch(ch[, attr]) @@ -757,7 +774,7 @@ the following methods and attributes: .. method:: window.attron(attr) - Add attribute *attr* from the "background" set applied to all writes to the + Add attribute *attr* to the "background" set applied to all writes to the current window. @@ -975,6 +992,10 @@ the following methods and attributes: window.getstr(y, x, n) Read a bytes object from the user, with primitive line editing capacity. + The maximum value for *n* is 2047. + + .. versionchanged:: 3.14 + The maximum value for *n* was increased from 1023 to 2047. .. method:: window.getyx() @@ -1066,6 +1087,10 @@ the following methods and attributes: current cursor position, or at *y*, *x* if specified. Attributes are stripped from the characters. If *n* is specified, :meth:`instr` returns a string at most *n* characters long (exclusive of the trailing NUL). + The maximum value for *n* is 2047. + + .. versionchanged:: 3.14 + The maximum value for *n* was increased from 1023 to 2047. .. method:: window.is_linetouched(line) @@ -1326,7 +1351,6 @@ The :mod:`curses` module defines the following data members: .. data:: version -.. data:: __version__ A bytes object representing the current version of the module. diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index e34b2db0210960..cff36e258224d3 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -121,8 +121,11 @@ Module contents :meth:`!__le__`, :meth:`!__gt__`, or :meth:`!__ge__`, then :exc:`TypeError` is raised. - - *unsafe_hash*: If ``False`` (the default), a :meth:`~object.__hash__` method - is generated according to how *eq* and *frozen* are set. + - *unsafe_hash*: If true, force ``dataclasses`` to create a + :meth:`~object.__hash__` method, even though it may not be safe to do so. + Otherwise, generate a :meth:`~object.__hash__` method according to how + *eq* and *frozen* are set. + The default value is ``False``. :meth:`!__hash__` is used by built-in :meth:`hash`, and when objects are added to hashed collections such as dictionaries and sets. Having a @@ -158,13 +161,15 @@ Module contents :class:`object`, this means it will fall back to id-based hashing). - *frozen*: If true (the default is ``False``), assigning to fields will - generate an exception. This emulates read-only frozen instances. If - :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class, then - :exc:`TypeError` is raised. See the discussion below. + generate an exception. This emulates read-only frozen instances. + See the :ref:`discussion ` below. + + If :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class + and *frozen* is true, then :exc:`TypeError` is raised. - *match_args*: If true (the default is ``True``), the :attr:`~object.__match_args__` tuple will be created from the list of - parameters to the generated :meth:`~object.__init__` method (even if + non keyword-only parameters to the generated :meth:`~object.__init__` method (even if :meth:`!__init__` is not generated, see above). If false, or if :attr:`!__match_args__` is already defined in the class, then :attr:`!__match_args__` will not be generated. @@ -175,11 +180,12 @@ Module contents fields will be marked as keyword-only. If a field is marked as keyword-only, then the only effect is that the :meth:`~object.__init__` parameter generated from a keyword-only field must be specified - with a keyword when :meth:`!__init__` is called. There is no - effect on any other aspect of dataclasses. See the - :term:`parameter` glossary entry for details. Also see the + with a keyword when :meth:`!__init__` is called. See the :term:`parameter` + glossary entry for details. Also see the :const:`KW_ONLY` section. + Keyword-only fields are not included in :attr:`!__match_args__`. + .. versionadded:: 3.10 - *slots*: If true (the default is ``False``), :attr:`~object.__slots__` attribute @@ -270,10 +276,11 @@ Module contents string returned by the generated :meth:`~object.__repr__` method. - *hash*: This can be a bool or ``None``. If true, this field is - included in the generated :meth:`~object.__hash__` method. If ``None`` (the - default), use the value of *compare*: this would normally be - the expected behavior. A field should be considered in the hash - if it's used for comparisons. Setting this value to anything + included in the generated :meth:`~object.__hash__` method. If false, + this field is excluded from the generated :meth:`~object.__hash__`. + If ``None`` (the default), use the value of *compare*: this would + normally be the expected behavior, since a field should be included + in the hash if it's used for comparisons. Setting this value to anything other than ``None`` is discouraged. One possible reason to set ``hash=False`` but ``compare=True`` @@ -298,17 +305,19 @@ Module contents This is used when the generated :meth:`~object.__init__` method's parameters are computed. + Keyword-only fields are also not included in :attr:`!__match_args__`. + .. versionadded:: 3.10 - - ``doc``: optional docstring for this field. + - *doc*: optional docstring for this field. - .. versionadded:: 3.13 + .. versionadded:: 3.14 If the default value of a field is specified by a call to :func:`!field`, then the class attribute for this field will be replaced by the specified *default* value. If *default* is not provided, then the class attribute will be deleted. The intent is - that after the :func:`@dataclass ` decorator runs, the class + that after the :deco:`dataclass` decorator runs, the class attributes will all contain the default values for the fields, just as if the default value itself were specified. For example, after:: @@ -340,6 +349,15 @@ Module contents Other attributes may exist, but they are private and must not be inspected or relied on. +.. class:: InitVar + + ``InitVar[T]`` type annotations describe variables that are :ref:`init-only + `. Fields annotated with :class:`!InitVar` + are considered pseudo-fields, and thus are neither returned by the + :func:`fields` function nor used in any way except adding them as + parameters to :meth:`~object.__init__` and an optional + :meth:`__post_init__`. + .. function:: fields(class_or_instance) Returns a tuple of :class:`Field` objects that define the fields for this @@ -409,7 +427,7 @@ Module contents :data:`typing.Any` is used for ``type``. The values of *init*, *repr*, *eq*, *order*, *unsafe_hash*, *frozen*, *match_args*, *kw_only*, *slots*, and *weakref_slot* have - the same meaning as they do in :func:`@dataclass `. + the same meaning as they do in :deco:`dataclass`. If *module* is defined, the :attr:`!__module__` attribute of the dataclass is set to that value. @@ -417,12 +435,12 @@ Module contents The *decorator* parameter is a callable that will be used to create the dataclass. It should take the class object as a first argument and the same keyword arguments - as :func:`@dataclass `. By default, the :func:`@dataclass ` + as :deco:`dataclass`. By default, the :deco:`dataclass` function is used. This function is not strictly required, because any Python - mechanism for creating a new class with :attr:`!__annotations__` can - then apply the :func:`@dataclass ` function to convert that class to + mechanism for creating a new class with :attr:`~object.__annotations__` can + then apply the :deco:`dataclass` function to convert that class to a dataclass. This function is provided as a convenience. For example:: @@ -551,7 +569,7 @@ Post-init processing def __post_init__(self): self.c = self.a + self.b -The :meth:`~object.__init__` method generated by :func:`@dataclass ` does not call base +The :meth:`~object.__init__` method generated by :deco:`dataclass` does not call base class :meth:`!__init__` methods. If the base class has an :meth:`!__init__` method that has to be called, it is common to call this method in a :meth:`__post_init__` method:: @@ -581,7 +599,7 @@ parameters to :meth:`!__post_init__`. Also see the warning about how Class variables --------------- -One of the few places where :func:`@dataclass ` actually inspects the type +One of the few places where :deco:`dataclass` actually inspects the type of a field is to determine if a field is a class variable as defined in :pep:`526`. It does this by checking if the type of the field is :data:`typing.ClassVar`. If a field is a ``ClassVar``, it is excluded @@ -594,10 +612,10 @@ module-level :func:`fields` function. Init-only variables ------------------- -Another place where :func:`@dataclass ` inspects a type annotation is to +Another place where :deco:`dataclass` inspects a type annotation is to determine if a field is an init-only variable. It does this by seeing -if the type of a field is of type ``dataclasses.InitVar``. If a field -is an ``InitVar``, it is considered a pseudo-field called an init-only +if the type of a field is of type :class:`InitVar`. If a field +is an :class:`InitVar`, it is considered a pseudo-field called an init-only field. As it is not a true field, it is not returned by the module-level :func:`fields` function. Init-only fields are added as parameters to the generated :meth:`~object.__init__` method, and are passed to @@ -628,7 +646,7 @@ Frozen instances ---------------- It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :func:`@dataclass ` decorator you can +by passing ``frozen=True`` to the :deco:`dataclass` decorator you can emulate immutability. In that case, dataclasses will add :meth:`~object.__setattr__` and :meth:`~object.__delattr__` methods to the class. These methods will raise a :exc:`FrozenInstanceError` when invoked. @@ -644,7 +662,7 @@ must use :meth:`!object.__setattr__`. Inheritance ----------- -When the dataclass is being created by the :func:`@dataclass ` decorator, +When the dataclass is being created by the :deco:`dataclass` decorator, it looks through all of the class's base classes in reverse MRO (that is, starting at :class:`object`) and, for each dataclass that it finds, adds the fields from that base class to an ordered mapping of fields. @@ -768,7 +786,7 @@ for :attr:`!x` when creating a class instance will share the same copy of :attr:`!x`. Because dataclasses just use normal Python class creation they also share this behavior. There is no general way for Data Classes to detect this condition. Instead, the -:func:`@dataclass ` decorator will raise a :exc:`ValueError` if it +:deco:`dataclass` decorator will raise a :exc:`ValueError` if it detects an unhashable default parameter. The assumption is that if a value is unhashable, it is mutable. This is a partial solution, but it does protect against many common errors. @@ -802,7 +820,7 @@ default value have the following special behaviors: :meth:`~object.__get__` or :meth:`!__set__` method is called rather than returning or overwriting the descriptor object. -* To determine whether a field contains a default value, :func:`@dataclass ` +* To determine whether a field contains a default value, :deco:`dataclass` will call the descriptor's :meth:`!__get__` method using its class access form: ``descriptor.__get__(obj=None, type=cls)``. If the descriptor returns a value in this case, it will be used as the diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 2f81080d525f86..48e7080da6c525 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -93,7 +93,7 @@ The :mod:`!datetime` module exports the following constants: The largest year number allowed in a :class:`date` or :class:`.datetime` object. :const:`MAXYEAR` is 9999. -.. attribute:: UTC +.. data:: UTC Alias for the UTC time zone singleton :attr:`datetime.timezone.utc`. @@ -261,6 +261,22 @@ A :class:`timedelta` object represents a duration, the difference between two >>> (d.days, d.seconds, d.microseconds) (-1, 86399, 999999) + Since the string representation of :class:`!timedelta` objects can be confusing, + use the following recipe to produce a more readable format: + + .. code-block:: pycon + + >>> def pretty_timedelta(td): + ... if td.days >= 0: + ... return str(td) + ... return f'-({-td!s})' + ... + >>> d = timedelta(hours=-1) + >>> str(d) # not human-friendly + '-1 day, 23:00:00' + >>> pretty_timedelta(d) + '-(1:00:00)' + Class attributes: @@ -519,6 +535,9 @@ Other constructors, all class methods: :c:func:`localtime` function. Raise :exc:`OSError` instead of :exc:`ValueError` on :c:func:`localtime` failure. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + .. classmethod:: date.fromordinal(ordinal) @@ -699,8 +718,8 @@ Instance methods: .. method:: date.replace(year=self.year, month=self.month, day=self.day) - Return a date with the same value, except for those parameters given new - values by whichever keyword arguments are specified. + Return a new :class:`date` object with the same values, but with specified + parameters updated. Example:: @@ -709,8 +728,8 @@ Instance methods: >>> d.replace(day=26) datetime.date(2002, 12, 26) - :class:`date` objects are also supported by generic function - :func:`copy.replace`. + The generic function :func:`copy.replace` also supports :class:`date` + objects. .. method:: date.timetuple() @@ -970,7 +989,7 @@ Other constructors, all class methods: .. deprecated:: 3.12 - Use :meth:`datetime.now` with :attr:`UTC` instead. + Use :meth:`datetime.now` with :const:`UTC` instead. .. classmethod:: datetime.fromtimestamp(timestamp, tz=None) @@ -1004,6 +1023,10 @@ Other constructors, all class methods: .. versionchanged:: 3.6 :meth:`fromtimestamp` may return instances with :attr:`.fold` set to 1. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + + .. classmethod:: datetime.utcfromtimestamp(timestamp) Return the UTC :class:`.datetime` corresponding to the POSIX timestamp, with @@ -1042,7 +1065,10 @@ Other constructors, all class methods: .. deprecated:: 3.12 - Use :meth:`datetime.fromtimestamp` with :attr:`UTC` instead. + Use :meth:`datetime.fromtimestamp` with :const:`UTC` instead. + + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. .. classmethod:: datetime.fromordinal(ordinal) @@ -1348,10 +1374,10 @@ Instance methods: hour=self.hour, minute=self.minute, second=self.second, microsecond=self.microsecond, \ tzinfo=self.tzinfo, *, fold=0) - Return a datetime with the same attributes, except for those attributes given - new values by whichever keyword arguments are specified. Note that - ``tzinfo=None`` can be specified to create a naive datetime from an aware - datetime with no conversion of date and time data. + Return a new :class:`datetime` object with the same attributes, but with + specified parameters updated. Note that ``tzinfo=None`` can be specified to + create a naive datetime from an aware datetime with no conversion of date + and time data. :class:`.datetime` objects are also supported by generic function :func:`copy.replace`. @@ -1486,11 +1512,11 @@ Instance methods: returned by :func:`time.time`. Naive :class:`.datetime` instances are assumed to represent local - time and this method relies on the platform C :c:func:`mktime` - function to perform the conversion. Since :class:`.datetime` - supports wider range of values than :c:func:`mktime` on many - platforms, this method may raise :exc:`OverflowError` or :exc:`OSError` - for times far in the past or far in the future. + time and this method relies on platform C functions to perform + the conversion. Since :class:`.datetime` supports a wider range of + values than the platform C functions on many platforms, this + method may raise :exc:`OverflowError` or :exc:`OSError` for times + far in the past or far in the future. For aware :class:`.datetime` instances, the return value is computed as:: @@ -1503,6 +1529,10 @@ Instance methods: The :meth:`timestamp` method uses the :attr:`.fold` attribute to disambiguate the times during a repeated interval. + .. versionchanged:: 3.6 + This method no longer relies on the platform C :c:func:`mktime` + function to perform conversions. + .. note:: There is no method to obtain the POSIX timestamp directly from a @@ -1942,10 +1972,10 @@ Instance methods: .. method:: time.replace(hour=self.hour, minute=self.minute, second=self.second, \ microsecond=self.microsecond, tzinfo=self.tzinfo, *, fold=0) - Return a :class:`.time` with the same value, except for those attributes given - new values by whichever keyword arguments are specified. Note that - ``tzinfo=None`` can be specified to create a naive :class:`.time` from an - aware :class:`.time`, without conversion of the time data. + Return a new :class:`.time` with the same values, but with specified + parameters updated. Note that ``tzinfo=None`` can be specified to create a + naive :class:`.time` from an aware :class:`.time`, without conversion of the + time data. :class:`.time` objects are also supported by generic function :func:`copy.replace`. @@ -2363,7 +2393,7 @@ locations where different offsets are used in different days of the year or where historical changes have been made to civil time. -.. class:: timezone(offset, name=None) +.. class:: timezone(offset[, name]) The *offset* argument must be specified as a :class:`timedelta` object representing the difference between the local time and UTC. It must @@ -2609,7 +2639,10 @@ differences between platforms in handling of unsupported format specifiers. ``%G``, ``%u`` and ``%V`` were added. .. versionadded:: 3.12 - ``%:z`` was added. + ``%:z`` was added for :meth:`~.datetime.strftime` + +.. versionadded:: 3.15 + ``%:z`` was added for :meth:`~.datetime.strptime` Technical Detail ^^^^^^^^^^^^^^^^ @@ -2618,9 +2651,42 @@ Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())`` although not all objects support a :meth:`~date.timetuple` method. -For the :meth:`.datetime.strptime` class method, the default value is -``1900-01-01T00:00:00.000``: any components not specified in the format string -will be pulled from the default value. [#]_ +For the :meth:`.datetime.strptime` and :meth:`.date.strptime` class methods, +the default value is ``1900-01-01T00:00:00.000``: any components not specified +in the format string will be pulled from the default value. + +.. note:: + When used to parse partial dates lacking a year, :meth:`.datetime.strptime` + and :meth:`.date.strptime` will raise when encountering February 29 because + the default year of 1900 is *not* a leap year. Always add a default leap + year to partial date strings before parsing. + + +.. testsetup:: + + # doctest seems to turn the warning into an error which makes it + # show up and require matching and prevents the actual interesting + # exception from being raised. + # Manually apply the catch_warnings context manager + import warnings + catch_warnings = warnings.catch_warnings() + catch_warnings.__enter__() + warnings.simplefilter("ignore") + +.. testcleanup:: + + catch_warnings.__exit__() + +.. doctest:: + + >>> from datetime import datetime + >>> value = "2/29" + >>> datetime.strptime(value, "%m/%d") + Traceback (most recent call last): + ... + ValueError: day 29 must be in range 1..28 for month 2 in year 1900 + >>> datetime.strptime(f"1904 {value}", "%Y %m/%d") + datetime.datetime(1904, 2, 29, 0, 0) Using ``datetime.strptime(date_string, format)`` is equivalent to:: @@ -2704,12 +2770,18 @@ Notes: When the ``%z`` directive is provided to the :meth:`~.datetime.strptime` method, the UTC offsets can have a colon as a separator between hours, minutes and seconds. - For example, ``'+01:00:00'`` will be parsed as an offset of one hour. - In addition, providing ``'Z'`` is identical to ``'+00:00'``. + For example, both ``'+010000'`` and ``'+01:00:00'`` will be parsed as an offset + of one hour. In addition, providing ``'Z'`` is identical to ``'+00:00'``. ``%:z`` - Behaves exactly as ``%z``, but has a colon separator added between - hours, minutes and seconds. + When used with :meth:`~.datetime.strftime`, behaves exactly as ``%z``, + except that a colon separator is added between hours, minutes and seconds. + + When used with :meth:`~.datetime.strptime`, the UTC offset is *required* + to have a colon as a separator between hours, minutes and seconds. + For example, ``'+01:00:00'`` (but *not* ``'+010000'``) will be parsed as + an offset of one hour. In addition, providing ``'Z'`` is identical to + ``'+00:00'``. ``%Z`` In :meth:`~.datetime.strftime`, ``%Z`` is replaced by an empty string if @@ -2751,7 +2823,7 @@ Notes: include a year in the format. If the value you need to parse lacks a year, append an explicit dummy leap year. Otherwise your code will raise an exception when it encounters leap day because the default year used by the - parser is not a leap year. Users run into this bug every four years... + parser (1900) is not a leap year. Users run into that bug every leap year. .. doctest:: @@ -2778,5 +2850,3 @@ Notes: .. [#] See R. H. van Gent's `guide to the mathematics of the ISO 8601 calendar `_ for a good explanation. - -.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since 1900 is not a leap year. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 6c659ea52ad821..02eb68d7b49b04 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -15,10 +15,16 @@ * :mod:`dbm.ndbm` If none of these modules are installed, the -slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There +slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There is a `third party interface `_ to the Oracle Berkeley DB. +.. note:: + None of the underlying modules will automatically shrink the disk space used by + the database file. However, :mod:`dbm.sqlite3`, :mod:`dbm.gnu` and :mod:`dbm.dumb` + provide a :meth:`!reorganize` method that can be used for this purpose. + + .. exception:: error A tuple containing the exceptions that can be raised by each of the supported @@ -84,10 +90,13 @@ the Oracle Berkeley DB. .. versionchanged:: 3.11 *file* accepts a :term:`path-like object`. -The object returned by :func:`~dbm.open` supports the same basic functionality as a -:class:`dict`; keys and their corresponding values can be stored, retrieved, and -deleted, and the :keyword:`in` operator and the :meth:`!keys` method are -available, as well as :meth:`!get` and :meth:`!setdefault` methods. +The object returned by :func:`~dbm.open` supports the basic +functionality of mutable :term:`mappings `; +keys and their corresponding values can be stored, retrieved, and +deleted, and iteration, the :keyword:`in` operator and methods :meth:`!keys`, +:meth:`!get`, :meth:`!setdefault` and :meth:`!clear` are available. +The :meth:`!keys` method returns a list instead of a view object. +The :meth:`!setdefault` method requires two arguments. Key and values are always stored as :class:`bytes`. This means that when strings are used they are implicitly converted to the default encoding before @@ -108,6 +117,10 @@ will automatically close them when done. Deleting a key from a read-only database raises a database module specific exception instead of :exc:`KeyError`. +.. versionchanged:: 3.13 + :meth:`!clear` methods are now available for all :mod:`dbm` backends. + + The following example records some hostnames and a corresponding title, and then prints out the contents of the database:: @@ -167,9 +180,6 @@ or any other SQLite browser, including the SQLite CLI. .. function:: open(filename, /, flag="r", mode=0o666) Open an SQLite database. - The returned object behaves like a :term:`mapping`, - implements a :meth:`!close` method, - and supports a "closing" context manager via the :keyword:`with` keyword. :param filename: The path to the database to be opened. @@ -186,6 +196,29 @@ or any other SQLite browser, including the SQLite CLI. The Unix file access mode of the file (default: octal ``0o666``), used only when the database has to be created. + The returned database object behaves similar to a mutable :term:`mapping`, + but the :meth:`!keys` method returns a list, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. + + The following methods are also provided: + + .. method:: sqlite3.close() + + Close the SQLite database. + + .. method:: sqlite3.reorganize() + + If you have carried out a lot of deletions and would like to shrink the space + used on disk, this method will reorganize the database; otherwise, deleted file + space will be kept and reused as new (key, value) pairs are added. + + .. note:: + While reorganizing, as much as two times the size of the original database is required + in free disk space. However, be aware that this factor changes for each :mod:`dbm` submodule. + + .. versionadded:: 3.15 + :mod:`dbm.gnu` --- GNU database manager --------------------------------------- @@ -215,6 +248,11 @@ functionality like crash tolerance. raised for general mapping errors like specifying an incorrect key. +.. data:: open_flags + + A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. + + .. function:: open(filename, flag="r", mode=0o666, /) Open a GDBM database and return a :class:`!gdbm` object. @@ -250,14 +288,25 @@ functionality like crash tolerance. .. versionchanged:: 3.11 *filename* accepts a :term:`path-like object`. - .. data:: open_flags + :class:`!gdbm` objects behave similar to mutable :term:`mappings `, + but methods :meth:`!items`, :meth:`!values`, :meth:`!pop`, :meth:`!popitem`, + and :meth:`!update` are not supported, + the :meth:`!keys` method returns a list, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. - A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. + .. versionchanged:: 3.2 + Added the :meth:`!get` and :meth:`!setdefault` methods. + + .. versionchanged:: 3.13 + Added the :meth:`!clear` method. - :class:`!gdbm` objects behave similar to :term:`mappings `, - but :meth:`!items` and :meth:`!values` methods are not supported. The following methods are also provided: + .. method:: gdbm.close() + + Close the GDBM database. + .. method:: gdbm.firstkey() It's possible to loop over every key in the database using this method and the @@ -284,21 +333,15 @@ functionality like crash tolerance. reorganization; otherwise, deleted file space will be kept and reused as new (key, value) pairs are added. + .. note:: + While reorganizing, as much as one time the size of the original database is required + in free disk space. However, be aware that this factor changes for each :mod:`dbm` submodule. + .. method:: gdbm.sync() When the database has been opened in fast mode, this method forces any unwritten data to be written to the disk. - .. method:: gdbm.close() - - Close the GDBM database. - - .. method:: gdbm.clear() - - Remove all items from the GDBM database. - - .. versionadded:: 3.13 - :mod:`dbm.ndbm` --- New Database Manager ---------------------------------------- @@ -359,22 +402,27 @@ This module can be used with the "classic" NDBM interface or the :param int mode: |mode_param_doc| - :class:`!ndbm` objects behave similar to :term:`mappings `, - but :meth:`!items` and :meth:`!values` methods are not supported. - The following methods are also provided: - .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. - .. method:: ndbm.close() + :class:`!ndbm` objects behave similar to mutable :term:`mappings `, + but methods :meth:`!items`, :meth:`!values`, :meth:`!pop`, :meth:`!popitem`, + and :meth:`!update` are not supported, + the :meth:`!keys` method returns a list, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. - Close the NDBM database. + .. versionchanged:: 3.2 + Added the :meth:`!get` and :meth:`!setdefault` methods. + + .. versionchanged:: 3.13 + Added the :meth:`!clear` method. - .. method:: ndbm.clear() + The following method is also provided: - Remove all items from the NDBM database. + .. method:: ndbm.close() - .. versionadded:: 3.13 + Close the NDBM database. :mod:`dbm.dumb` --- Portable DBM implementation @@ -412,9 +460,6 @@ The :mod:`!dbm.dumb` module defines the following: .. function:: open(filename, flag="c", mode=0o666) Open a :mod:`!dbm.dumb` database. - The returned database object behaves similar to a :term:`mapping`, - in addition to providing :meth:`~dumbdbm.sync` and :meth:`~dumbdbm.close` - methods. :param filename: The basename of the database file (without extensions). @@ -438,6 +483,11 @@ The :mod:`!dbm.dumb` module defines the following: with a sufficiently large/complex entry due to stack depth limitations in Python's AST compiler. + .. warning:: + :mod:`dbm.dumb` does not support concurrent read/write access. (Multiple + simultaneous read accesses are safe.) When a program has the database open + for writing, no other program should have it open for reading or writing. + .. versionchanged:: 3.5 :func:`~dbm.dumb.open` always creates a new database when *flag* is ``'n'``. @@ -448,15 +498,30 @@ The :mod:`!dbm.dumb` module defines the following: .. versionchanged:: 3.11 *filename* accepts a :term:`path-like object`. - In addition to the methods provided by the - :class:`collections.abc.MutableMapping` class, - the following methods are provided: + The returned database object behaves similar to a mutable :term:`mapping`, + but the :meth:`!keys` and :meth:`!items` methods return lists, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. - .. method:: dumbdbm.sync() - - Synchronize the on-disk directory and data files. This method is called - by the :meth:`Shelve.sync` method. + The following methods are also provided: .. method:: dumbdbm.close() Close the database. + + .. method:: dumbdbm.reorganize() + + If you have carried out a lot of deletions and would like to shrink the space + used on disk, this method will reorganize the database; otherwise, deleted file + space will not be reused. + + .. note:: + While reorganizing, no additional free disk space is required. However, be aware + that this factor changes for each :mod:`dbm` submodule. + + .. versionadded:: 3.15 + + .. method:: dumbdbm.sync() + + Synchronize the on-disk directory and data files. This method is called + by the :meth:`shelve.Shelf.sync` method. diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst index 60223657a44043..f87c2481fb89cd 100644 --- a/Doc/library/debug.rst +++ b/Doc/library/debug.rst @@ -1,5 +1,5 @@ *********************** -Debugging and Profiling +Debugging and profiling *********************** These libraries help you with Python development: the debugger enables you to @@ -15,7 +15,8 @@ intrusive debugging or patching. bdb.rst faulthandler.rst pdb.rst - profile.rst + profiling.rst + pstats.rst timeit.rst trace.rst tracemalloc.rst diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index c9a3e448cad063..376bcc7aaf9eb2 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -2,7 +2,7 @@ ===================================================================== .. module:: decimal - :synopsis: Implementation of the General Decimal Arithmetic Specification. + :synopsis: Implementation of the General Decimal Arithmetic Specification. .. moduleauthor:: Eric Price .. moduleauthor:: Facundo Batista @@ -34,10 +34,12 @@ The :mod:`decimal` module provides support for fast correctly rounded decimal floating-point arithmetic. It offers several advantages over the :class:`float` datatype: -* Decimal "is based on a floating-point model which was designed with people - in mind, and necessarily has a paramount guiding principle -- computers must - provide an arithmetic that works in the same way as the arithmetic that - people learn at school." -- excerpt from the decimal arithmetic specification. +* Decimal "is based on a `floating-point model + `__ which was designed + with people in mind, and necessarily has a paramount guiding principle -- + computers must provide an arithmetic that works in the same way as the + arithmetic that people learn at school." -- excerpt from the decimal + arithmetic specification. * Decimal numbers can be represented exactly. In contrast, numbers like ``1.1`` and ``2.2`` do not have exact representations in binary @@ -121,7 +123,7 @@ reset them before monitoring a calculation. .. _decimal-tutorial: -Quick-start Tutorial +Quick-start tutorial -------------------- The usual start to using decimals is importing the module, viewing the current @@ -238,6 +240,26 @@ floating-point flying circus: >>> c % a Decimal('0.77') +Decimals can be formatted (with :func:`format` built-in or :ref:`f-strings`) in +fixed-point or scientific notation, using the same formatting syntax (see +:ref:`formatspec`) as builtin :class:`float` type: + +.. doctest:: + + >>> format(Decimal('2.675'), "f") + '2.675' + >>> format(Decimal('2.675'), ".2f") + '2.68' + >>> f"{Decimal('2.675'):.2f}" + '2.68' + >>> format(Decimal('2.675'), ".2e") + '2.68e+0' + >>> with localcontext() as ctx: + ... ctx.rounding = ROUND_DOWN + ... print(format(Decimal('2.675'), ".2f")) + ... + 2.67 + And some mathematical functions are also available to Decimal: >>> getcontext().prec = 28 @@ -264,7 +286,7 @@ allows the settings to be changed. This approach meets the needs of most applications. For more advanced work, it may be useful to create alternate contexts using the -Context() constructor. To make an alternate active, use the :func:`setcontext` +:meth:`Context` constructor. To make an alternate active, use the :func:`setcontext` function. In accordance with the standard, the :mod:`decimal` module provides two ready to @@ -367,6 +389,8 @@ Decimal objects appears above. These include decimal digits from various other alphabets (for example, Arabic-Indic and Devanāgarī digits) along with the fullwidth digits ``'\uff10'`` through ``'\uff19'``. + Case is not significant, so, for example, ``inf``, ``Inf``, ``INFINITY``, + and ``iNfINity`` are all acceptable spellings for positive infinity. If *value* is a :class:`tuple`, it should have three components, a sign (``0`` for positive or ``1`` for negative), a :class:`tuple` of @@ -571,7 +595,7 @@ Decimal objects >>> Decimal(321).exp() Decimal('2.561702493119680037517373933E+139') - .. classmethod:: from_float(f) + .. classmethod:: from_float(f, /) Alternative constructor that only accepts instances of :class:`float` or :class:`int`. @@ -598,7 +622,7 @@ Decimal objects .. versionadded:: 3.1 - .. classmethod:: from_number(number) + .. classmethod:: from_number(number, /) Alternative constructor that only accepts instances of :class:`float`, :class:`int` or :class:`Decimal`, but not strings @@ -989,7 +1013,7 @@ Each thread has its own current context which is accessed or changed using the Return the current context for the active thread. -.. function:: setcontext(c) +.. function:: setcontext(c, /) Set the current context for the active thread to *c*. @@ -1029,11 +1053,19 @@ function to temporarily change the active context. .. versionchanged:: 3.11 :meth:`localcontext` now supports setting context attributes through the use of keyword arguments. +.. function:: IEEEContext(bits) + + Return a context object initialized to the proper values for one of the + IEEE interchange formats. The argument must be a multiple of 32 and less + than :const:`IEEE_CONTEXT_MAX_BITS`. + + .. versionadded:: 3.14 + New contexts can also be created using the :class:`Context` constructor described below. In addition, the module provides three pre-made contexts: -.. class:: BasicContext +.. data:: BasicContext This is a standard context defined by the General Decimal Arithmetic Specification. Precision is set to nine. Rounding is set to @@ -1044,7 +1076,7 @@ described below. In addition, the module provides three pre-made contexts: Because many of the traps are enabled, this context is useful for debugging. -.. class:: ExtendedContext +.. data:: ExtendedContext This is a standard context defined by the General Decimal Arithmetic Specification. Precision is set to nine. Rounding is set to @@ -1057,7 +1089,7 @@ described below. In addition, the module provides three pre-made contexts: presence of conditions that would otherwise halt the program. -.. class:: DefaultContext +.. data:: DefaultContext This context is used by the :class:`Context` constructor as a prototype for new contexts. Changing a field (such a precision) has the effect of changing the @@ -1086,40 +1118,52 @@ In addition to the three supplied contexts, new contexts can be created with the default values are copied from the :const:`DefaultContext`. If the *flags* field is not specified or is :const:`None`, all flags are cleared. - *prec* is an integer in the range [``1``, :const:`MAX_PREC`] that sets - the precision for arithmetic operations in the context. + .. attribute:: prec + + An integer in the range [``1``, :const:`MAX_PREC`] that sets + the precision for arithmetic operations in the context. + + .. attribute:: rounding - The *rounding* option is one of the constants listed in the section - `Rounding Modes`_. + One of the constants listed in the section `Rounding Modes`_. - The *traps* and *flags* fields list any signals to be set. Generally, new - contexts should only set traps and leave the flags clear. + .. attribute:: traps + flags - The *Emin* and *Emax* fields are integers specifying the outer limits allowable - for exponents. *Emin* must be in the range [:const:`MIN_EMIN`, ``0``], - *Emax* in the range [``0``, :const:`MAX_EMAX`]. + Lists of any signals to be set. Generally, new contexts should only set + traps and leave the flags clear. - The *capitals* field is either ``0`` or ``1`` (the default). If set to - ``1``, exponents are printed with a capital ``E``; otherwise, a - lowercase ``e`` is used: ``Decimal('6.02e+23')``. + .. attribute:: Emin + Emax - The *clamp* field is either ``0`` (the default) or ``1``. - If set to ``1``, the exponent ``e`` of a :class:`Decimal` - instance representable in this context is strictly limited to the - range ``Emin - prec + 1 <= e <= Emax - prec + 1``. If *clamp* is - ``0`` then a weaker condition holds: the adjusted exponent of - the :class:`Decimal` instance is at most :attr:`~Context.Emax`. When *clamp* is - ``1``, a large normal number will, where possible, have its - exponent reduced and a corresponding number of zeros added to its - coefficient, in order to fit the exponent constraints; this - preserves the value of the number but loses information about - significant trailing zeros. For example:: + Integers specifying the outer limits allowable for exponents. *Emin* must + be in the range [:const:`MIN_EMIN`, ``0``], *Emax* in the range + [``0``, :const:`MAX_EMAX`]. - >>> Context(prec=6, Emax=999, clamp=1).create_decimal('1.23e999') - Decimal('1.23000E+999') + .. attribute:: capitals - A *clamp* value of ``1`` allows compatibility with the - fixed-width decimal interchange formats specified in IEEE 754. + Either ``0`` or ``1`` (the default). If set to + ``1``, exponents are printed with a capital ``E``; otherwise, a + lowercase ``e`` is used: ``Decimal('6.02e+23')``. + + .. attribute:: clamp + + Either ``0`` (the default) or ``1``. If set to ``1``, the exponent ``e`` + of a :class:`Decimal` instance representable in this context is strictly + limited to the range ``Emin - prec + 1 <= e <= Emax - prec + 1``. + If *clamp* is ``0`` then a weaker condition holds: the adjusted exponent of + the :class:`Decimal` instance is at most :attr:`~Context.Emax`. When *clamp* is + ``1``, a large normal number will, where possible, have its + exponent reduced and a corresponding number of zeros added to its + coefficient, in order to fit the exponent constraints; this + preserves the value of the number but loses information about + significant trailing zeros. For example:: + + >>> Context(prec=6, Emax=999, clamp=1).create_decimal('1.23e999') + Decimal('1.23000E+999') + + A *clamp* value of ``1`` allows compatibility with the + fixed-width decimal interchange formats specified in IEEE 754. The :class:`Context` class defines several general purpose methods as well as a large number of methods for doing arithmetic directly in a given context. @@ -1146,11 +1190,11 @@ In addition to the three supplied contexts, new contexts can be created with the Return a duplicate of the context. - .. method:: copy_decimal(num) + .. method:: copy_decimal(num, /) Return a copy of the Decimal instance num. - .. method:: create_decimal(num) + .. method:: create_decimal(num='0', /) Creates a new Decimal instance from *num* but using *self* as context. Unlike the :class:`Decimal` constructor, the context precision, @@ -1174,7 +1218,7 @@ In addition to the three supplied contexts, new contexts can be created with the If the argument is a string, no leading or trailing whitespace or underscores are permitted. - .. method:: create_decimal_from_float(f) + .. method:: create_decimal_from_float(f, /) Creates a new Decimal instance from a float *f* but rounding using *self* as the context. Unlike the :meth:`Decimal.from_float` class method, @@ -1212,222 +1256,222 @@ In addition to the three supplied contexts, new contexts can be created with the recounted here. - .. method:: abs(x) + .. method:: abs(x, /) Returns the absolute value of *x*. - .. method:: add(x, y) + .. method:: add(x, y, /) Return the sum of *x* and *y*. - .. method:: canonical(x) + .. method:: canonical(x, /) Returns the same Decimal object *x*. - .. method:: compare(x, y) + .. method:: compare(x, y, /) Compares *x* and *y* numerically. - .. method:: compare_signal(x, y) + .. method:: compare_signal(x, y, /) Compares the values of the two operands numerically. - .. method:: compare_total(x, y) + .. method:: compare_total(x, y, /) Compares two operands using their abstract representation. - .. method:: compare_total_mag(x, y) + .. method:: compare_total_mag(x, y, /) Compares two operands using their abstract representation, ignoring sign. - .. method:: copy_abs(x) + .. method:: copy_abs(x, /) Returns a copy of *x* with the sign set to 0. - .. method:: copy_negate(x) + .. method:: copy_negate(x, /) Returns a copy of *x* with the sign inverted. - .. method:: copy_sign(x, y) + .. method:: copy_sign(x, y, /) Copies the sign from *y* to *x*. - .. method:: divide(x, y) + .. method:: divide(x, y, /) Return *x* divided by *y*. - .. method:: divide_int(x, y) + .. method:: divide_int(x, y, /) Return *x* divided by *y*, truncated to an integer. - .. method:: divmod(x, y) + .. method:: divmod(x, y, /) Divides two numbers and returns the integer part of the result. - .. method:: exp(x) + .. method:: exp(x, /) Returns ``e ** x``. - .. method:: fma(x, y, z) + .. method:: fma(x, y, z, /) Returns *x* multiplied by *y*, plus *z*. - .. method:: is_canonical(x) + .. method:: is_canonical(x, /) Returns ``True`` if *x* is canonical; otherwise returns ``False``. - .. method:: is_finite(x) + .. method:: is_finite(x, /) Returns ``True`` if *x* is finite; otherwise returns ``False``. - .. method:: is_infinite(x) + .. method:: is_infinite(x, /) Returns ``True`` if *x* is infinite; otherwise returns ``False``. - .. method:: is_nan(x) + .. method:: is_nan(x, /) Returns ``True`` if *x* is a qNaN or sNaN; otherwise returns ``False``. - .. method:: is_normal(x) + .. method:: is_normal(x, /) Returns ``True`` if *x* is a normal number; otherwise returns ``False``. - .. method:: is_qnan(x) + .. method:: is_qnan(x, /) Returns ``True`` if *x* is a quiet NaN; otherwise returns ``False``. - .. method:: is_signed(x) + .. method:: is_signed(x, /) Returns ``True`` if *x* is negative; otherwise returns ``False``. - .. method:: is_snan(x) + .. method:: is_snan(x, /) Returns ``True`` if *x* is a signaling NaN; otherwise returns ``False``. - .. method:: is_subnormal(x) + .. method:: is_subnormal(x, /) Returns ``True`` if *x* is subnormal; otherwise returns ``False``. - .. method:: is_zero(x) + .. method:: is_zero(x, /) Returns ``True`` if *x* is a zero; otherwise returns ``False``. - .. method:: ln(x) + .. method:: ln(x, /) Returns the natural (base e) logarithm of *x*. - .. method:: log10(x) + .. method:: log10(x, /) Returns the base 10 logarithm of *x*. - .. method:: logb(x) + .. method:: logb(x, /) Returns the exponent of the magnitude of the operand's MSD. - .. method:: logical_and(x, y) + .. method:: logical_and(x, y, /) Applies the logical operation *and* between each operand's digits. - .. method:: logical_invert(x) + .. method:: logical_invert(x, /) Invert all the digits in *x*. - .. method:: logical_or(x, y) + .. method:: logical_or(x, y, /) Applies the logical operation *or* between each operand's digits. - .. method:: logical_xor(x, y) + .. method:: logical_xor(x, y, /) Applies the logical operation *xor* between each operand's digits. - .. method:: max(x, y) + .. method:: max(x, y, /) Compares two values numerically and returns the maximum. - .. method:: max_mag(x, y) + .. method:: max_mag(x, y, /) Compares the values numerically with their sign ignored. - .. method:: min(x, y) + .. method:: min(x, y, /) Compares two values numerically and returns the minimum. - .. method:: min_mag(x, y) + .. method:: min_mag(x, y, /) Compares the values numerically with their sign ignored. - .. method:: minus(x) + .. method:: minus(x, /) Minus corresponds to the unary prefix minus operator in Python. - .. method:: multiply(x, y) + .. method:: multiply(x, y, /) Return the product of *x* and *y*. - .. method:: next_minus(x) + .. method:: next_minus(x, /) Returns the largest representable number smaller than *x*. - .. method:: next_plus(x) + .. method:: next_plus(x, /) Returns the smallest representable number larger than *x*. - .. method:: next_toward(x, y) + .. method:: next_toward(x, y, /) Returns the number closest to *x*, in direction towards *y*. - .. method:: normalize(x) + .. method:: normalize(x, /) Reduces *x* to its simplest form. - .. method:: number_class(x) + .. method:: number_class(x, /) Returns an indication of the class of *x*. - .. method:: plus(x) + .. method:: plus(x, /) Plus corresponds to the unary prefix plus operator in Python. This operation applies the context precision and rounding, so it is *not* an @@ -1468,7 +1512,7 @@ In addition to the three supplied contexts, new contexts can be created with the always exact. - .. method:: quantize(x, y) + .. method:: quantize(x, y, /) Returns a value equal to *x* (rounded), having the exponent of *y*. @@ -1478,7 +1522,7 @@ In addition to the three supplied contexts, new contexts can be created with the Just returns 10, as this is Decimal, :) - .. method:: remainder(x, y) + .. method:: remainder(x, y, /) Returns the remainder from integer division. @@ -1486,43 +1530,43 @@ In addition to the three supplied contexts, new contexts can be created with the dividend. - .. method:: remainder_near(x, y) + .. method:: remainder_near(x, y, /) Returns ``x - y * n``, where *n* is the integer nearest the exact value of ``x / y`` (if the result is 0 then its sign will be the sign of *x*). - .. method:: rotate(x, y) + .. method:: rotate(x, y, /) Returns a rotated copy of *x*, *y* times. - .. method:: same_quantum(x, y) + .. method:: same_quantum(x, y, /) Returns ``True`` if the two operands have the same exponent. - .. method:: scaleb (x, y) + .. method:: scaleb (x, y, /) Returns the first operand after adding the second value its exp. - .. method:: shift(x, y) + .. method:: shift(x, y, /) Returns a shifted copy of *x*, *y* times. - .. method:: sqrt(x) + .. method:: sqrt(x, /) Square root of a non-negative number to context precision. - .. method:: subtract(x, y) + .. method:: subtract(x, y, /) Return the difference between *x* and *y*. - .. method:: to_eng_string(x) + .. method:: to_eng_string(x, /) Convert to a string, using engineering notation if an exponent is needed. @@ -1531,12 +1575,12 @@ In addition to the three supplied contexts, new contexts can be created with the require the addition of either one or two trailing zeros. - .. method:: to_integral_exact(x) + .. method:: to_integral_exact(x, /) Rounds to an integer. - .. method:: to_sci_string(x) + .. method:: to_sci_string(x, /) Converts a number to a string using scientific notation. @@ -1547,21 +1591,31 @@ In addition to the three supplied contexts, new contexts can be created with the Constants --------- -The constants in this section are only relevant for the C module. They -are also included in the pure Python version for compatibility. +.. data:: SPEC_VERSION -+---------------------+---------------------+-------------------------------+ -| | 32-bit | 64-bit | -+=====================+=====================+===============================+ -| .. data:: MAX_PREC | ``425000000`` | ``999999999999999999`` | -+---------------------+---------------------+-------------------------------+ -| .. data:: MAX_EMAX | ``425000000`` | ``999999999999999999`` | -+---------------------+---------------------+-------------------------------+ -| .. data:: MIN_EMIN | ``-425000000`` | ``-999999999999999999`` | -+---------------------+---------------------+-------------------------------+ -| .. data:: MIN_ETINY | ``-849999999`` | ``-1999999999999999997`` | -+---------------------+---------------------+-------------------------------+ + The highest version of the General Decimal Arithmetic + Specification that this implementation complies with. + See https://speleotrove.com/decimal/decarith.html for the specification. + .. versionadded:: 3.15 + + +The following constants are only relevant for the C module. They +are also included in the pure Python version for compatibility. + ++---------------------------------+---------------------+-------------------------------+ +| | 32-bit | 64-bit | ++=================================+=====================+===============================+ +| .. data:: MAX_PREC | ``425000000`` | ``999999999999999999`` | ++---------------------------------+---------------------+-------------------------------+ +| .. data:: MAX_EMAX | ``425000000`` | ``999999999999999999`` | ++---------------------------------+---------------------+-------------------------------+ +| .. data:: MIN_EMIN | ``-425000000`` | ``-999999999999999999`` | ++---------------------------------+---------------------+-------------------------------+ +| .. data:: MIN_ETINY | ``-849999999`` | ``-1999999999999999997`` | ++---------------------------------+---------------------+-------------------------------+ +| .. data:: IEEE_CONTEXT_MAX_BITS | ``256`` | ``512`` | ++---------------------------------+---------------------+-------------------------------+ .. data:: HAVE_THREADS @@ -1758,7 +1812,7 @@ The following table summarizes the hierarchy of signals:: .. _decimal-notes: -Floating-Point Notes +Floating-point notes -------------------- @@ -1884,13 +1938,20 @@ the current thread. If :func:`setcontext` has not been called before :func:`getcontext`, then :func:`getcontext` will automatically create a new context for use in the -current thread. +current thread. New context objects have default values set from the +:data:`decimal.DefaultContext` object. + +The :data:`sys.flags.thread_inherit_context` flag affects the context for +new threads. If the flag is false, new threads will start with an empty +context. In this case, :func:`getcontext` will create a new context object +when called and use the default values from *DefaultContext*. If the flag +is true, new threads will start with a copy of context from the caller of +:meth:`threading.Thread.start`. -The new context is copied from a prototype context called *DefaultContext*. To -control the defaults so that each thread will use the same values throughout the -application, directly modify the *DefaultContext* object. This should be done -*before* any threads are started so that there won't be a race condition between -threads calling :func:`getcontext`. For example:: +To control the defaults so that each thread will use the same values throughout +the application, directly modify the *DefaultContext* object. This should be +done *before* any threads are started so that there won't be a race condition +between threads calling :func:`getcontext`. For example:: # Set applicationwide defaults for all threads about to be launched DefaultContext.prec = 12 @@ -2070,20 +2131,20 @@ to work with the :class:`Decimal` class:: Decimal FAQ ----------- -Q. It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to +Q: It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to minimize typing when using the interactive interpreter? -A. Some users abbreviate the constructor to just a single letter: +A: Some users abbreviate the constructor to just a single letter: >>> D = decimal.Decimal >>> D('1.23') + D('3.45') Decimal('4.68') -Q. In a fixed-point application with two decimal places, some inputs have many +Q: In a fixed-point application with two decimal places, some inputs have many places and need to be rounded. Others are not supposed to have excess digits and need to be validated. What methods should be used? -A. The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If +A: The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If the :const:`Inexact` trap is set, it is also useful for validation: >>> TWOPLACES = Decimal(10) ** -2 # same as Decimal('0.01') @@ -2101,10 +2162,10 @@ the :const:`Inexact` trap is set, it is also useful for validation: ... Inexact: None -Q. Once I have valid two place inputs, how do I maintain that invariant +Q: Once I have valid two place inputs, how do I maintain that invariant throughout an application? -A. Some operations like addition, subtraction, and multiplication by an integer +A: Some operations like addition, subtraction, and multiplication by an integer will automatically preserve fixed point. Others operations, like division and non-integer multiplication, will change the number of decimal places and need to be followed-up with a :meth:`~Decimal.quantize` step: @@ -2136,21 +2197,21 @@ to handle the :meth:`~Decimal.quantize` step: >>> div(b, a) Decimal('0.03') -Q. There are many ways to express the same value. The numbers ``200``, +Q: There are many ways to express the same value. The numbers ``200``, ``200.000``, ``2E2``, and ``.02E+4`` all have the same value at various precisions. Is there a way to transform them to a single recognizable canonical value? -A. The :meth:`~Decimal.normalize` method maps all equivalent values to a single +A: The :meth:`~Decimal.normalize` method maps all equivalent values to a single representative: >>> values = map(Decimal, '200 200.000 2E2 .02E+4'.split()) >>> [v.normalize() for v in values] [Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2')] -Q. When does rounding occur in a computation? +Q: When does rounding occur in a computation? -A. It occurs *after* the computation. The philosophy of the decimal +A: It occurs *after* the computation. The philosophy of the decimal specification is that numbers are considered exact and are created independent of the current context. They can even have greater precision than current context. Computations process with those @@ -2168,10 +2229,10 @@ applied to the *result* of the computation:: >>> pi + 0 - Decimal('0.00005'). # Intermediate values are rounded Decimal('3.1416') -Q. Some decimal values always print with exponential notation. Is there a way +Q: Some decimal values always print with exponential notation. Is there a way to get a non-exponential representation? -A. For some values, exponential notation is the only way to express the number +A: For some values, exponential notation is the only way to express the number of significant places in the coefficient. For example, expressing ``5.0E+3`` as ``5000`` keeps the value constant but cannot show the original's two-place significance. @@ -2186,9 +2247,9 @@ value unchanged: >>> remove_exponent(Decimal('5E+3')) Decimal('5000') -Q. Is there a way to convert a regular float to a :class:`Decimal`? +Q: Is there a way to convert a regular float to a :class:`Decimal`? -A. Yes, any binary floating-point number can be exactly expressed as a +A: Yes, any binary floating-point number can be exactly expressed as a Decimal though an exact conversion may take more precision than intuition would suggest: @@ -2197,19 +2258,19 @@ suggest: >>> Decimal(math.pi) Decimal('3.141592653589793115997963468544185161590576171875') -Q. Within a complex calculation, how can I make sure that I haven't gotten a +Q: Within a complex calculation, how can I make sure that I haven't gotten a spurious result because of insufficient precision or rounding anomalies. -A. The decimal module makes it easy to test results. A best practice is to +A: The decimal module makes it easy to test results. A best practice is to re-run calculations using greater precision and with various rounding modes. Widely differing results indicate insufficient precision, rounding mode issues, ill-conditioned inputs, or a numerically unstable algorithm. -Q. I noticed that context precision is applied to the results of operations but +Q: I noticed that context precision is applied to the results of operations but not to the inputs. Is there anything to watch out for when mixing values of different precisions? -A. Yes. The principle is that all values are considered to be exact and so is +A: Yes. The principle is that all values are considered to be exact and so is the arithmetic on those values. Only the results are rounded. The advantage for inputs is that "what you type is what you get". A disadvantage is that the results can look odd if you forget that the inputs haven't been rounded: @@ -2237,9 +2298,9 @@ Alternatively, inputs can be rounded upon creation using the >>> Context(prec=5, rounding=ROUND_DOWN).create_decimal('1.2345678') Decimal('1.2345') -Q. Is the CPython implementation fast for large numbers? +Q: Is the CPython implementation fast for large numbers? -A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of +A: Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of the decimal module integrate the high speed `libmpdec `_ library for arbitrary precision correctly rounded decimal floating-point arithmetic [#]_. @@ -2262,7 +2323,7 @@ value for :attr:`~Context.prec` as well [#]_:: Decimal('904625697166532776746648320380374280103671755200316906558262375061821325312') -For inexact results, :attr:`MAX_PREC` is far too large on 64-bit platforms and +For inexact results, :const:`MAX_PREC` is far too large on 64-bit platforms and the available memory will be insufficient:: >>> Decimal(1) / 3 diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst index 191e0da12103fa..e0693e8eb6ed22 100644 --- a/Doc/library/dialog.rst +++ b/Doc/library/dialog.rst @@ -220,7 +220,7 @@ is the base class for dialogs defined in other supporting modules. .. class:: Dialog(master=None, **options) - .. method:: show(color=None, **options) + .. method:: show(**options) Render the Dialog window. diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index ce948a6860f02c..9e5a62d8fe5260 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -231,7 +231,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. *linejunk*: A function that accepts a single string argument, and returns true if the string is junk, or false if not. The default is ``None``. There is also a module-level function :func:`IS_LINE_JUNK`, which filters out lines - without visible characters, except for at most one pound character (``'#'``) + without visible characters, except for at most one hash character (``'#'``) -- however the underlying :class:`SequenceMatcher` class does a dynamic analysis of which lines are so frequent as to constitute noise, and this usually works better than using this function. @@ -278,7 +278,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. emu -.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n') +.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n', *, color=False) Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` generating the delta lines) in unified diff format. @@ -297,6 +297,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. For inputs that do not have trailing newlines, set the *lineterm* argument to ``""`` so that the output will be uniformly newline free. + Set *color* to ``True`` to enable output in color, similar to + :program:`git diff --color`. Even if enabled, it can be + :ref:`controlled using environment variables `. + The unified diff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for *fromfile*, *tofile*, *fromfiledate*, and *tofiledate*. The modification times are normally @@ -319,6 +323,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. See :ref:`difflib-interface` for a more detailed example. + .. versionchanged:: 3.15 + Added the *color* parameter. + + .. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n') Compare *a* and *b* (lists of bytes objects) using *dfunc*; yield a @@ -351,9 +359,9 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. seealso:: - `Pattern Matching: The Gestalt Approach `_ + `Pattern Matching: The Gestalt Approach `_ Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This - was published in `Dr. Dobb's Journal `_ in July, 1988. + was published in Dr. Dobb's Journal in July, 1988. .. _sequence-matcher: diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index a2e44e09ffc925..755d681b9df2be 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -60,6 +60,8 @@ interpreter. The :option:`-P ` command-line option and the ``show_positions`` argument were added. + The :option:`-S ` command-line option is added. + Example: Given the function :func:`!myfunc`:: def myfunc(alist): @@ -74,7 +76,7 @@ the following command can be used to display the disassembly of 2 RESUME 0 3 LOAD_GLOBAL 1 (len + NULL) - LOAD_FAST 0 (alist) + LOAD_FAST_BORROW 0 (alist) CALL 1 RETURN_VALUE @@ -89,28 +91,40 @@ The :mod:`dis` module can be invoked as a script from the command line: .. code-block:: sh - python -m dis [-h] [-C] [-O] [-P] [infile] + python -m dis [-h] [-C] [-O] [-P] [-S] [infile] The following options are accepted: .. program:: dis -.. cmdoption:: -h, --help +.. option:: -h, --help Display usage and exit. -.. cmdoption:: -C, --show-caches +.. option:: -C, --show-caches Show inline caches. -.. cmdoption:: -O, --show-offsets + .. versionadded:: 3.13 + +.. option:: -O, --show-offsets Show offsets of instructions. -.. cmdoption:: -P, --show-positions + .. versionadded:: 3.13 + +.. option:: -P, --show-positions Show positions of instructions in the source code. + .. versionadded:: 3.14 + +.. option:: -S, --specialized + + Show specialized bytecode. + + .. versionadded:: 3.14 + If :file:`infile` is specified, its disassembled code will be written to stdout. Otherwise, disassembly is performed on compiled source code received from stdin. @@ -201,7 +215,7 @@ Example: ... RESUME LOAD_GLOBAL - LOAD_FAST + LOAD_FAST_BORROW CALL RETURN_VALUE @@ -521,7 +535,7 @@ details of bytecode instructions as :class:`Instruction` instances: :class:`dis.Positions` object holding the start and end locations that are covered by this instruction. - .. data::cache_info + .. data:: cache_info Information about the cache entries of this instruction, as triplets of the form ``(name, size, data)``, where the ``name`` @@ -571,6 +585,22 @@ operations on it as if it was a Python list. The top of the stack corresponds to generate line tracing events. +.. opcode:: NOT_TAKEN + + Do nothing code. + Used by the interpreter to record :monitoring-event:`BRANCH_LEFT` + and :monitoring-event:`BRANCH_RIGHT` events for :mod:`sys.monitoring`. + + .. versionadded:: 3.14 + + +.. opcode:: POP_ITER + + Removes the iterator from the top of the stack. + + .. versionadded:: 3.14 + + .. opcode:: POP_TOP Removes the top-of-stack item:: @@ -697,15 +727,8 @@ not have to be) the original ``STACK[-2]``. STACK.append(lhs op rhs) .. versionadded:: 3.11 - - -.. opcode:: BINARY_SUBSCR - - Implements:: - - key = STACK.pop() - container = STACK.pop() - STACK.append(container[key]) + .. versionchanged:: 3.14 + With oparg :``NB_SUBSCR``, implements binary subscript (replaces opcode ``BINARY_SUBSCR``) .. opcode:: STORE_SUBSCR @@ -745,7 +768,7 @@ not have to be) the original ``STACK[-2]``. end = STACK.pop() start = STACK.pop() container = STACK.pop() - values = STACK.pop() + value = STACK.pop() container[start:end] = value .. versionadded:: 3.12 @@ -862,13 +885,6 @@ iterations of the loop. Returns with ``STACK[-1]`` to the caller of the function. -.. opcode:: RETURN_CONST (consti) - - Returns with ``co_consts[consti]`` to the caller of the function. - - .. versionadded:: 3.12 - - .. opcode:: YIELD_VALUE Yields ``STACK.pop()`` from a :term:`generator`. @@ -1086,6 +1102,14 @@ iterations of the loop. Pushes ``co_consts[consti]`` onto the stack. +.. opcode:: LOAD_SMALL_INT (i) + + Pushes the integer ``i`` onto the stack. + ``i`` must be in ``range(256)`` + + .. versionadded:: 3.14 + + .. opcode:: LOAD_NAME (namei) Pushes the value associated with ``co_names[namei]`` onto the stack. @@ -1112,6 +1136,48 @@ iterations of the loop. .. versionadded:: 3.12 +.. opcode:: BUILD_TEMPLATE + + Constructs a new :class:`~string.templatelib.Template` instance from a tuple + of strings and a tuple of interpolations and pushes the resulting object + onto the stack:: + + interpolations = STACK.pop() + strings = STACK.pop() + STACK.append(_build_template(strings, interpolations)) + + .. versionadded:: 3.14 + + +.. opcode:: BUILD_INTERPOLATION (format) + + Constructs a new :class:`~string.templatelib.Interpolation` instance from a + value and its source expression and pushes the resulting object onto the + stack. + + If no conversion or format specification is present, ``format`` is set to + ``2``. + + If the low bit of ``format`` is set, it indicates that the interpolation + contains a format specification. + + If ``format >> 2`` is non-zero, it indicates that the interpolation + contains a conversion. The value of ``format >> 2`` is the conversion type + (``0`` for no conversion, ``1`` for ``!s``, ``2`` for ``!r``, and + ``3`` for ``!a``):: + + conversion = format >> 2 + if format & 1: + format_spec = STACK.pop() + else: + format_spec = None + expression = STACK.pop() + value = STACK.pop() + STACK.append(_build_interpolation(value, expression, conversion, format_spec)) + + .. versionadded:: 3.14 + + .. opcode:: BUILD_TUPLE (count) Creates a tuple consuming *count* items from the stack, and pushes the @@ -1338,9 +1404,6 @@ iterations of the loop. If ``STACK[-1]`` is not ``None``, increments the bytecode counter by *delta*. ``STACK[-1]`` is popped. - This opcode is a pseudo-instruction, replaced in final bytecode by - the directed versions (forward/backward). - .. versionadded:: 3.11 .. versionchanged:: 3.12 @@ -1352,9 +1415,6 @@ iterations of the loop. If ``STACK[-1]`` is ``None``, increments the bytecode counter by *delta*. ``STACK[-1]`` is popped. - This opcode is a pseudo-instruction, replaced in final bytecode by - the directed versions (forward/backward). - .. versionadded:: 3.11 .. versionchanged:: 3.12 @@ -1386,6 +1446,28 @@ iterations of the loop. This opcode is now only used in situations where the local variable is guaranteed to be initialized. It cannot raise :exc:`UnboundLocalError`. +.. opcode:: LOAD_FAST_BORROW (var_num) + + Pushes a borrowed reference to the local ``co_varnames[var_num]`` onto the + stack. + + .. versionadded:: 3.14 + +.. opcode:: LOAD_FAST_LOAD_FAST (var_nums) + + Pushes references to ``co_varnames[var_nums >> 4]`` and + ``co_varnames[var_nums & 15]`` onto the stack. + + .. versionadded:: 3.13 + + +.. opcode:: LOAD_FAST_BORROW_LOAD_FAST_BORROW (var_nums) + + Pushes borrowed references to ``co_varnames[var_nums >> 4]`` and + ``co_varnames[var_nums & 15]`` onto the stack. + + .. versionadded:: 3.14 + .. opcode:: LOAD_FAST_CHECK (var_num) Pushes a reference to the local ``co_varnames[var_num]`` onto the stack, @@ -1406,6 +1488,20 @@ iterations of the loop. Stores ``STACK.pop()`` into the local ``co_varnames[var_num]``. +.. opcode:: STORE_FAST_STORE_FAST (var_nums) + + Stores ``STACK[-1]`` into ``co_varnames[var_nums >> 4]`` + and ``STACK[-2]`` into ``co_varnames[var_nums & 15]``. + + .. versionadded:: 3.13 + +.. opcode:: STORE_FAST_LOAD_FAST (var_nums) + + Stores ``STACK.pop()`` into the local ``co_varnames[var_nums >> 4]`` + and pushes a reference to the local ``co_varnames[var_nums & 15]`` + onto the stack. + + .. versionadded:: 3.13 .. opcode:: DELETE_FAST (var_num) @@ -1546,14 +1642,14 @@ iterations of the loop. Pushes a ``NULL`` to the stack. Used in the call sequence to match the ``NULL`` pushed by - :opcode:`LOAD_METHOD` for non-method calls. + :opcode:`!LOAD_METHOD` for non-method calls. .. versionadded:: 3.11 .. opcode:: MAKE_FUNCTION - Pushes a new function object on the stack built from the code object at ``STACK[1]``. + Pushes a new function object on the stack built from the code object at ``STACK[-1]``. .. versionchanged:: 3.10 Flag value ``0x04`` is a tuple of strings instead of dictionary @@ -1577,9 +1673,13 @@ iterations of the loop. * ``0x02`` a dictionary of keyword-only parameters' default values * ``0x04`` a tuple of strings containing parameters' annotations * ``0x08`` a tuple containing cells for free variables, making a closure + * ``0x10`` the :term:`annotate function` for the function object .. versionadded:: 3.13 + .. versionchanged:: 3.14 + Added ``0x10`` to indicate the annotate function for the function object. + .. opcode:: BUILD_SLICE (argc) @@ -1621,7 +1721,7 @@ iterations of the loop. * ``oparg == 2``: call :func:`repr` on *value* * ``oparg == 3``: call :func:`ascii` on *value* - Used for implementing formatted literal strings (f-strings). + Used for implementing formatted string literals (f-strings). .. versionadded:: 3.13 @@ -1634,11 +1734,11 @@ iterations of the loop. result = value.__format__("") STACK.append(result) - Used for implementing formatted literal strings (f-strings). + Used for implementing formatted string literals (f-strings). .. versionadded:: 3.13 -.. opcode:: FORMAT_SPEC +.. opcode:: FORMAT_WITH_SPEC Formats the given value with the given format spec:: @@ -1647,7 +1747,7 @@ iterations of the loop. result = value.__format__(spec) STACK.append(result) - Used for implementing formatted literal strings (f-strings). + Used for implementing formatted string literals (f-strings). .. versionadded:: 3.13 @@ -1867,14 +1967,15 @@ but are replaced by real opcodes or removed before bytecode is generated. Marks the end of the code block associated with the last ``SETUP_FINALLY``, ``SETUP_CLEANUP`` or ``SETUP_WITH``. + .. opcode:: JUMP -.. opcode:: JUMP_NO_INTERRUPT + JUMP_NO_INTERRUPT Undirected relative jump instructions which are replaced by their directed (forward/backward) counterparts by the assembler. .. opcode:: JUMP_IF_TRUE -.. opcode:: JUMP_IF_FALSE + JUMP_IF_FALSE Conditional jumps which do not impact the stack. Replaced by the sequence ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. @@ -1890,12 +1991,6 @@ but are replaced by real opcodes or removed before bytecode is generated. This opcode is now a pseudo-instruction. -.. opcode:: LOAD_METHOD - - Optimized unbound method lookup. Emitted as a ``LOAD_ATTR`` opcode - with a flag set in the arg. - - .. _opcode_collections: Opcode collections @@ -1986,4 +2081,3 @@ instructions: .. deprecated:: 3.13 All jumps are now relative. This list is empty. - diff --git a/Doc/library/distutils.rst b/Doc/library/distutils.rst new file mode 100644 index 00000000000000..af63e035bf3c4a --- /dev/null +++ b/Doc/library/distutils.rst @@ -0,0 +1,17 @@ +:mod:`!distutils` --- Building and installing Python modules +============================================================ + +.. module:: distutils + :synopsis: Removed in 3.12. + :deprecated: + +.. deprecated-removed:: 3.10 3.12 + +This module is no longer part of the Python standard library. +It was :ref:`removed in Python 3.12 ` after +being deprecated in Python 3.10. The removal was decided in :pep:`632`, +which has `migration advice +`_. + +The last version of Python that provided the :mod:`!distutils` module was +`Python 3.11 `_. diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 6b0282eed49566..df3de8f622a091 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -136,6 +136,10 @@ examples of doctests in the standard Python test suite and libraries. Especially useful examples can be found in the standard test file :file:`Lib/test/test_doctest/test_doctest.py`. +.. versionadded:: 3.13 + Output is colorized by default and can be + :ref:`controlled using environment variables `. + .. _doctest-simple-testmod: @@ -170,18 +174,11 @@ with assorted summaries at the end. You can force verbose mode by passing ``verbose=True`` to :func:`testmod`, or prohibit it by passing ``verbose=False``. In either of those cases, -``sys.argv`` is not examined by :func:`testmod` (so passing ``-v`` or not +:data:`sys.argv` is not examined by :func:`testmod` (so passing ``-v`` or not has no effect). -There is also a command line shortcut for running :func:`testmod`. You can -instruct the Python interpreter to run the doctest module directly from the -standard library and pass the module name(s) on the command line:: - - python -m doctest -v example.py - -This will import :file:`example.py` as a standalone module and run -:func:`testmod` on it. Note that this may not work correctly if the file is -part of a package and imports other submodules from that package. +There is also a command line shortcut for running :func:`testmod`, see section +:ref:`doctest-cli`. For more information on :func:`testmod`, see section :ref:`doctest-basic-api`. @@ -234,7 +231,7 @@ documentation:: As with :func:`testmod`, :func:`testfile` won't display anything unless an example fails. If an example does fail, then the failing example(s) and the cause(s) of the failure(s) are printed to stdout, using the same format as -:func:`testmod`. +:func:`!testmod`. By default, :func:`testfile` looks for files in the calling module's directory. See section :ref:`doctest-basic-api` for a description of the optional arguments @@ -244,16 +241,53 @@ Like :func:`testmod`, :func:`testfile`'s verbosity can be set with the ``-v`` command-line switch or with the optional keyword argument *verbose*. -There is also a command line shortcut for running :func:`testfile`. You can -instruct the Python interpreter to run the doctest module directly from the -standard library and pass the file name(s) on the command line:: +There is also a command line shortcut for running :func:`testfile`, see section +:ref:`doctest-cli`. - python -m doctest -v example.txt +For more information on :func:`testfile`, see section :ref:`doctest-basic-api`. -Because the file name does not end with :file:`.py`, :mod:`doctest` infers that -it must be run with :func:`testfile`, not :func:`testmod`. -For more information on :func:`testfile`, see section :ref:`doctest-basic-api`. +.. _doctest-cli: + +Command-line Usage +------------------ + +The :mod:`doctest` module can be invoked as a script from the command line: + +.. code-block:: bash + + python -m doctest [-v] [-o OPTION] [-f] file [file ...] + +.. program:: doctest + +.. option:: -v, --verbose + + Detailed report of all examples tried is printed to standard output, + along with assorted summaries at the end:: + + python -m doctest -v example.py + + This will import :file:`example.py` as a standalone module and run + :func:`testmod` on it. Note that this may not work correctly if the + file is part of a package and imports other submodules from that package. + + If the file name does not end with :file:`.py`, :mod:`!doctest` infers + that it must be run with :func:`testfile` instead:: + + python -m doctest -v example.txt + +.. option:: -o, --option