gh-112301: Enable warning emitting options and ignore warnings in CI

.github/workflows/reusable-macos.yml

@@ -35,7 +35,7 @@ jobs:
  path: config.cache
  key: ${{github.job }}-${{inputs.os }}-${{env.IMAGE_VERSION }}-${{inputs.config_hash }}
  - name: Install Homebrew dependencies
- run: brew install pkg-config [email protected] xz gdbm tcl-tk
+ run: brew install pkg-config [email protected] xz gdbm tcl-tk make
  - name: Configure CPython
  run: |
  GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
@@ -48,10 +48,22 @@ jobs:
  --prefix=/opt/python-dev \
  --with-openssl="$(brew --prefix [email protected])"
  - name: Build CPython
- run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt
+ if : ${{inputs.free-threading || inputs.os != 'macos-13'}}
+ run: gmake -j8
+ - name: Build CPython for compiler warning check
+ if : ${{!inputs.free-threading && inputs.os == 'macos-13'}}
+ run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt
  - name: Display build info
  run: make pythoninfo
  - name: Check compiler warnings
- run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang
+ if : ${{!inputs.free-threading && inputs.os == 'macos-13'}}
+ run: >-
+ python3 Tools/build/check_warnings.py
+ --compiler-output-file-path=compiler_output_macos.txt
+ --warning-ignore-file-path=Tools/build/.warningignore_macos
+ --compiler-output-type=clang
+ --fail-on-regression
+ --fail-on-improvement
+ --path-prefix="./"
  - name: Tests
  run: make test

.github/workflows/reusable-ubuntu.yml

@@ -67,20 +67,32 @@ jobs:
  working-directory: ${{env.CPYTHON_BUILDDIR }}
  run: >-
  ../cpython-ro-srcdir/configure
- CFLAGS="-fdiagnostics-format=json"
  --config-cache
  --with-pydebug
  --enable-slower-safety
  --with-openssl=$OPENSSL_DIR
  ${{fromJSON(inputs.free-threading) && '--disable-gil' || ''}}
  - name: Build CPython out-of-tree
+ if: ${{inputs.free-threading }}
  working-directory: ${{env.CPYTHON_BUILDDIR }}
- run: set -o pipefail; make -j4 2>&1 | tee compiler_output.txt
+ run: make -j4
+ - name: Build CPython out-of-tree (for compiler warning check)
+ if: ${{!inputs.free-threading}}
+ working-directory: ${{env.CPYTHON_BUILDDIR }}
+ run: set -o pipefail; make -j4 2>&1 | tee compiler_output_ubuntu.txt
  - name: Display build info
  working-directory: ${{env.CPYTHON_BUILDDIR }}
  run: make pythoninfo
  - name: Check compiler warnings
- run: python Tools/build/check_warnings.py --compiler-output-file-path=${{env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json
+ if: ${{!inputs.free-threading }}
+ run: >-
+ python Tools/build/check_warnings.py
+ --compiler-output-file-path=${{env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt
+ --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu
+ --compiler-output-type=gcc
+ --fail-on-regression
+ --fail-on-improvement
+ --path-prefix="../cpython-ro-srcdir/"
  - name: Remount sources writable for tests
  # some tests write to srcdir, lack of pyc files slows down testing
  run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw

Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst

@@ -0,0 +1 @@
+Enable compiler options that warn of potential security vulnerabilities.

Tools/build/.warningignore_macos

@@ -3,3 +3,230 @@
 # Keep lines sorted lexicographically to help avoid merge conflicts.
 # Format example:
 # /path/to/file (number of warnings in file)
+Include/internal/mimalloc/mimalloc/internal.h 4
+Include/internal/pycore_backoff.h 1
+Include/internal/pycore_dict.h 2
+Include/internal/pycore_gc.h 1
+Include/internal/pycore_long.h 2
+Include/internal/pycore_object.h 4
+Modules/_asynciomodule.c 3
+Modules/_bisectmodule.c 2
+Modules/_bz2module.c 5
+Modules/_collectionsmodule.c 2
+Modules/_csv.c 3
+Modules/_ctypes/_ctypes.c 37
+Modules/_ctypes/_ctypes_test_generated.c.h 141
+Modules/_ctypes/callbacks.c 6
+Modules/_ctypes/callproc.c 15
+Modules/_ctypes/cfield.c 59
+Modules/_ctypes/malloc_closure.c 3
+Modules/_ctypes/stgdict.c 17
+Modules/_cursesmodule.c 24
+Modules/_datetimemodule.c 28
+Modules/_dbmmodule.c 8
+Modules/_decimal/_decimal.c 15
+Modules/_elementtree.c 42
+Modules/_functoolsmodule.c 6
+Modules/_gdbmmodule.c 5
+Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84
+Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84
+Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 24
+Modules/_hashopenssl.c 16
+Modules/_interpchannelsmodule.c 1
+Modules/_interpqueuesmodule.c 1
+Modules/_io/_iomodule.c 1
+Modules/_io/bufferedio.c 4
+Modules/_io/bytesio.c 11
+Modules/_io/fileio.c 9
+Modules/_io/stringio.c 8
+Modules/_io/textio.c 11
+Modules/_json.c 19
+Modules/_localemodule.c 3
+Modules/_lzmamodule.c 10
+Modules/_multiprocessing/semaphore.c 2
+Modules/_operator.c 5
+Modules/_pickle.c 71
+Modules/_posixsubprocess.c 8
+Modules/_queuemodule.c 4
+Modules/_randommodule.c 3
+Modules/_scproxy.c 3
+Modules/_sqlite/connection.c 4
+Modules/_sqlite/cursor.c 3
+Modules/_sqlite/module.c 2
+Modules/_sre/sre.c 18
+Modules/_sre/sre_lib.h 62
+Modules/_ssl.c 29
+Modules/_struct.c 1
+Modules/_testbuffer.c 22
+Modules/_testcapi/heaptype.c 1
+Modules/_testcapi/long.c 2
+Modules/_testcapi/mem.c 2
+Modules/_testcapi/monitoring.c 3
+Modules/_testcapi/pyatomic.c 1
+Modules/_testcapi/unicode.c 2
+Modules/_testcapi/vectorcall.c 3
+Modules/_testcapi/watchers.c 3
+Modules/_testcapimodule.c 3
+Modules/_testclinic.c 14
+Modules/_testexternalinspection.c 8
+Modules/_testinternalcapi.c 8
+Modules/_testinternalcapi/pytime.c 8
+Modules/_testinternalcapi/test_critical_sections.c 1
+Modules/_testinternalcapi/test_lock.c 2
+Modules/_testlimitedcapi/heaptype_relative.c 4
+Modules/_testlimitedcapi/object.c 2
+Modules/_testlimitedcapi/unicode.c 2
+Modules/_threadmodule.c 2
+Modules/_tkinter.c 6
+Modules/_xxtestfuzz/_xxtestfuzz.c 1
+Modules/_xxtestfuzz/fuzzer.c 11
+Modules/_zoneinfo.c 14
+Modules/arraymodule.c 32
+Modules/atexitmodule.c 1
+Modules/binascii.c 206
+Modules/blake2module.c 6
+Modules/cjkcodecs/_codecs_cn.c 1
+Modules/cjkcodecs/_codecs_iso2022.c 2
+Modules/cjkcodecs/_codecs_jp.c 14
+Modules/cjkcodecs/_codecs_kr.c 3
+Modules/cjkcodecs/cjkcodecs.h 1
+Modules/cjkcodecs/multibytecodec.c 2
+Modules/clinic/_testclinic.c.h 1
+Modules/clinic/arraymodule.c.h 1
+Modules/clinic/unicodedata.c.h 10
+Modules/cmathmodule.c 1
+Modules/expat/siphash.h 8
+Modules/expat/xmlparse.c 45
+Modules/expat/xmltok.c 17
+Modules/expat/xmltok_impl.c 34
+Modules/faulthandler.c 3
+Modules/fcntlmodule.c 1
+Modules/getpath.c 7
+Modules/grpmodule.c 4
+Modules/itertoolsmodule.c 7
+Modules/main.c 2
+Modules/mathmodule.c 15
+Modules/mmapmodule.c 20
+Modules/posixmodule.c 67
+Modules/pwdmodule.c 4
+Modules/pyexpat.c 20
+Modules/readline.c 1
+Modules/resource.c 3
+Modules/rotatingtree.c 1
+Modules/selectmodule.c 6
+Modules/sha3module.c 4
+Modules/signalmodule.c 1
+Modules/socketmodule.c 44
+Modules/syslogmodule.c 3
+Modules/timemodule.c 4
+Modules/unicodedata.c 28
+Modules/unicodedata_db.h 1
+Modules/xxsubtype.c 2
+Modules/zlibmodule.c 16
+Objects/abstract.c 2
+Objects/bytearrayobject.c 34
+Objects/bytes_methods.c 9
+Objects/bytesobject.c 35
+Objects/call.c 13
+Objects/classobject.c 4
+Objects/codeobject.c 15
+Objects/descrobject.c 2
+Objects/dictobject.c 28
+Objects/fileobject.c 3
+Objects/floatobject.c 30
+Objects/frameobject.c 19
+Objects/funcobject.c 1
+Objects/genobject.c 5
+Objects/listobject.c 43
+Objects/longobject.c 46
+Objects/memoryobject.c 6
+Objects/methodobject.c 1
+Objects/mimalloc/alloc.c 6
+Objects/mimalloc/arena.c 6
+Objects/mimalloc/heap.c 1
+Objects/mimalloc/init.c 2
+Objects/mimalloc/options.c 1
+Objects/mimalloc/os.c 4
+Objects/mimalloc/page-queue.c 2
+Objects/mimalloc/page.c 1
+Objects/mimalloc/prim/osx/../unix/prim.c 2
+Objects/mimalloc/random.c 1
+Objects/mimalloc/segment.c 11
+Objects/mimalloc/stats.c 1
+Objects/moduleobject.c 2
+Objects/object.c 1
+Objects/obmalloc.c 6
+Objects/odictobject.c 3
+Objects/rangeobject.c 10
+Objects/setobject.c 13
+Objects/sliceobject.c 4
+Objects/stringlib/codecs.h 26
+Objects/stringlib/eq.h 1
+Objects/stringlib/fastsearch.h 14
+Objects/stringlib/join.h 1
+Objects/stringlib/replace.h 4
+Objects/stringlib/repr.h 21
+Objects/stringlib/transmogrify.h 5
+Objects/structseq.c 14
+Objects/tupleobject.c 10
+Objects/typeobject.c 17
+Objects/unicodectype.c 7
+Objects/unicodeobject.c 113
+Parser/action_helpers.c 4
+Parser/lexer/buffer.c 1
+Parser/lexer/lexer.c 12
+Parser/parser.c 116
+Parser/pegen.c 7
+Parser/string_parser.c 7
+Parser/tokenizer/file_tokenizer.c 8
+Parser/tokenizer/helpers.c 7
+Parser/tokenizer/readline_tokenizer.c 3
+Programs/_freeze_module.c 1
+Python/Python-ast.c 15
+Python/asdl.c 3
+Python/assemble.c 7
+Python/ast_opt.c 7
+Python/bltinmodule.c 9
+Python/bootstrap_hash.c 4
+Python/ceval.c 8
+Python/ceval_gil.c 2
+Python/codecs.c 32
+Python/codegen.c 6
+Python/compile.c 2
+Python/context.c 1
+Python/crossinterp.c 2
+Python/crossinterp_data_lookup.h 1
+Python/dtoa.c 34
+Python/errors.c 1
+Python/fileutils.c 7
+Python/flowgraph.c 8
+Python/formatter_unicode.c 7
+Python/frame.c 4
+Python/gc.c 8
+Python/generated_cases.c.h 35
+Python/getargs.c 11
+Python/import.c 5
+Python/initconfig.c 11
+Python/instrumentation.c 31
+Python/intrinsics.c 1
+Python/legacy_tracing.c 3
+Python/lock.c 4
+Python/marshal.c 11
+Python/modsupport.c 3
+Python/mystrtoul.c 4
+Python/pathconfig.c 1
+Python/preconfig.c 2
+Python/pyarena.c 1
+Python/pyhash.c 2
+Python/pylifecycle.c 7
+Python/pystate.c 6
+Python/pystrhex.c 19
+Python/pystrtod.c 3
+Python/qsbr.c 2
+Python/specialize.c 10
+Python/suggestions.c 12
+Python/symtable.c 18
+Python/sysmodule.c 2
+Python/thread_pthread.h 1
+Python/traceback.c 6
+Python/tracemalloc.c 6