path: config.cache
key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
- name: Install Homebrew dependencies
- run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk
+ run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk make
- name: Configure CPython
run: |
GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
--prefix=/opt/python-dev \
--with-openssl="$(brew --prefix openssl@3.0)"
- name: Build CPython
- run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt
+ if : ${{ inputs.free-threading || inputs.os != 'macos-13' }}
+ run: gmake -j8
+ - name: Build CPython for compiler warning check
+ if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+ run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt
- name: Display build info
run: make pythoninfo
- name: Check compiler warnings
- run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang
+ if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+ run: >-
+ python3 Tools/build/check_warnings.py
+ --compiler-output-file-path=compiler_output_macos.txt
+ --warning-ignore-file-path=Tools/build/.warningignore_macos
+ --compiler-output-type=clang
+ --fail-on-regression
+ --fail-on-improvement
+ --path-prefix="./"
- name: Tests
run: make test
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: >-
../cpython-ro-srcdir/configure
- CFLAGS="-fdiagnostics-format=json"
--config-cache
--with-pydebug
--enable-slower-safety
--with-openssl=$OPENSSL_DIR
${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
- name: Build CPython out-of-tree
+ if: ${{ inputs.free-threading }}
working-directory: ${{ env.CPYTHON_BUILDDIR }}
- run: set -o pipefail; make -j4 2>&1 | tee compiler_output.txt
+ run: make -j4
+ - name: Build CPython out-of-tree (for compiler warning check)
+ if: ${{ !inputs.free-threading}}
+ working-directory: ${{ env.CPYTHON_BUILDDIR }}
+ run: set -o pipefail; make -j4 2>&1 | tee compiler_output_ubuntu.txt
- name: Display build info
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: make pythoninfo
- name: Check compiler warnings
- run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json
+ if: ${{ !inputs.free-threading }}
+ run: >-
+ python Tools/build/check_warnings.py
+ --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt
+ --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu
+ --compiler-output-type=gcc
+ --fail-on-regression
+ --fail-on-improvement
+ --path-prefix="../cpython-ro-srcdir/"
- name: Remount sources writable for tests
# some tests write to srcdir, lack of pyc files slows down testing
run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
--- /dev/null
+Enable compiler options that warn of potential security vulnerabilities.
# Keep lines sorted lexicographically to help avoid merge conflicts.
# Format example:
# /path/to/file (number of warnings in file)
+Include/internal/mimalloc/mimalloc/internal.h 4
+Include/internal/pycore_backoff.h 1
+Include/internal/pycore_dict.h 2
+Include/internal/pycore_gc.h 1
+Include/internal/pycore_long.h 2
+Include/internal/pycore_object.h 4
+Modules/_asynciomodule.c 3
+Modules/_bisectmodule.c 2
+Modules/_bz2module.c 5
+Modules/_collectionsmodule.c 2
+Modules/_csv.c 3
+Modules/_ctypes/_ctypes.c 37
+Modules/_ctypes/_ctypes_test_generated.c.h 141
+Modules/_ctypes/callbacks.c 6
+Modules/_ctypes/callproc.c 15
+Modules/_ctypes/cfield.c 59
+Modules/_ctypes/malloc_closure.c 3
+Modules/_ctypes/stgdict.c 17
+Modules/_cursesmodule.c 24
+Modules/_datetimemodule.c 28
+Modules/_dbmmodule.c 8
+Modules/_decimal/_decimal.c 15
+Modules/_elementtree.c 42
+Modules/_functoolsmodule.c 6
+Modules/_gdbmmodule.c 5
+Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84
+Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84
+Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 24
+Modules/_hashopenssl.c 16
+Modules/_interpchannelsmodule.c 1
+Modules/_interpqueuesmodule.c 1
+Modules/_io/_iomodule.c 1
+Modules/_io/bufferedio.c 4
+Modules/_io/bytesio.c 11
+Modules/_io/fileio.c 9
+Modules/_io/stringio.c 8
+Modules/_io/textio.c 11
+Modules/_json.c 19
+Modules/_localemodule.c 3
+Modules/_lzmamodule.c 10
+Modules/_multiprocessing/semaphore.c 2
+Modules/_operator.c 5
+Modules/_pickle.c 71
+Modules/_posixsubprocess.c 8
+Modules/_queuemodule.c 4
+Modules/_randommodule.c 3
+Modules/_scproxy.c 3
+Modules/_sqlite/connection.c 4
+Modules/_sqlite/cursor.c 3
+Modules/_sqlite/module.c 2
+Modules/_sre/sre.c 18
+Modules/_sre/sre_lib.h 62
+Modules/_ssl.c 29
+Modules/_struct.c 1
+Modules/_testbuffer.c 22
+Modules/_testcapi/heaptype.c 1
+Modules/_testcapi/long.c 2
+Modules/_testcapi/mem.c 2
+Modules/_testcapi/monitoring.c 3
+Modules/_testcapi/pyatomic.c 1
+Modules/_testcapi/unicode.c 2
+Modules/_testcapi/vectorcall.c 3
+Modules/_testcapi/watchers.c 3
+Modules/_testcapimodule.c 3
+Modules/_testclinic.c 14
+Modules/_testexternalinspection.c 8
+Modules/_testinternalcapi.c 8
+Modules/_testinternalcapi/pytime.c 8
+Modules/_testinternalcapi/test_critical_sections.c 1
+Modules/_testinternalcapi/test_lock.c 2
+Modules/_testlimitedcapi/heaptype_relative.c 4
+Modules/_testlimitedcapi/object.c 2
+Modules/_testlimitedcapi/unicode.c 2
+Modules/_threadmodule.c 2
+Modules/_tkinter.c 6
+Modules/_xxtestfuzz/_xxtestfuzz.c 1
+Modules/_xxtestfuzz/fuzzer.c 11
+Modules/_zoneinfo.c 14
+Modules/arraymodule.c 32
+Modules/atexitmodule.c 1
+Modules/binascii.c 206
+Modules/blake2module.c 6
+Modules/cjkcodecs/_codecs_cn.c 1
+Modules/cjkcodecs/_codecs_iso2022.c 2
+Modules/cjkcodecs/_codecs_jp.c 14
+Modules/cjkcodecs/_codecs_kr.c 3
+Modules/cjkcodecs/cjkcodecs.h 1
+Modules/cjkcodecs/multibytecodec.c 2
+Modules/clinic/_testclinic.c.h 1
+Modules/clinic/arraymodule.c.h 1
+Modules/clinic/unicodedata.c.h 10
+Modules/cmathmodule.c 1
+Modules/expat/siphash.h 8
+Modules/expat/xmlparse.c 45
+Modules/expat/xmltok.c 17
+Modules/expat/xmltok_impl.c 34
+Modules/faulthandler.c 3
+Modules/fcntlmodule.c 1
+Modules/getpath.c 7
+Modules/grpmodule.c 4
+Modules/itertoolsmodule.c 7
+Modules/main.c 2
+Modules/mathmodule.c 15
+Modules/mmapmodule.c 20
+Modules/posixmodule.c 67
+Modules/pwdmodule.c 4
+Modules/pyexpat.c 20
+Modules/readline.c 1
+Modules/resource.c 3
+Modules/rotatingtree.c 1
+Modules/selectmodule.c 6
+Modules/sha3module.c 4
+Modules/signalmodule.c 1
+Modules/socketmodule.c 44
+Modules/syslogmodule.c 3
+Modules/timemodule.c 4
+Modules/unicodedata.c 28
+Modules/unicodedata_db.h 1
+Modules/xxsubtype.c 2
+Modules/zlibmodule.c 16
+Objects/abstract.c 2
+Objects/bytearrayobject.c 34
+Objects/bytes_methods.c 9
+Objects/bytesobject.c 35
+Objects/call.c 13
+Objects/classobject.c 4
+Objects/codeobject.c 15
+Objects/descrobject.c 2
+Objects/dictobject.c 28
+Objects/fileobject.c 3
+Objects/floatobject.c 30
+Objects/frameobject.c 19
+Objects/funcobject.c 1
+Objects/genobject.c 5
+Objects/listobject.c 43
+Objects/longobject.c 46
+Objects/memoryobject.c 6
+Objects/methodobject.c 1
+Objects/mimalloc/alloc.c 6
+Objects/mimalloc/arena.c 6
+Objects/mimalloc/heap.c 1
+Objects/mimalloc/init.c 2
+Objects/mimalloc/options.c 1
+Objects/mimalloc/os.c 4
+Objects/mimalloc/page-queue.c 2
+Objects/mimalloc/page.c 1
+Objects/mimalloc/prim/osx/../unix/prim.c 2
+Objects/mimalloc/random.c 1
+Objects/mimalloc/segment.c 11
+Objects/mimalloc/stats.c 1
+Objects/moduleobject.c 2
+Objects/object.c 1
+Objects/obmalloc.c 6
+Objects/odictobject.c 3
+Objects/rangeobject.c 10
+Objects/setobject.c 13
+Objects/sliceobject.c 4
+Objects/stringlib/codecs.h 26
+Objects/stringlib/eq.h 1
+Objects/stringlib/fastsearch.h 14
+Objects/stringlib/join.h 1
+Objects/stringlib/replace.h 4
+Objects/stringlib/repr.h 21
+Objects/stringlib/transmogrify.h 5
+Objects/structseq.c 14
+Objects/tupleobject.c 10
+Objects/typeobject.c 17
+Objects/unicodectype.c 7
+Objects/unicodeobject.c 113
+Parser/action_helpers.c 4
+Parser/lexer/buffer.c 1
+Parser/lexer/lexer.c 12
+Parser/parser.c 116
+Parser/pegen.c 7
+Parser/string_parser.c 7
+Parser/tokenizer/file_tokenizer.c 8
+Parser/tokenizer/helpers.c 7
+Parser/tokenizer/readline_tokenizer.c 3
+Programs/_freeze_module.c 1
+Python/Python-ast.c 15
+Python/asdl.c 3
+Python/assemble.c 7
+Python/ast_opt.c 7
+Python/bltinmodule.c 9
+Python/bootstrap_hash.c 4
+Python/ceval.c 8
+Python/ceval_gil.c 2
+Python/codecs.c 32
+Python/codegen.c 6
+Python/compile.c 2
+Python/context.c 1
+Python/crossinterp.c 2
+Python/crossinterp_data_lookup.h 1
+Python/dtoa.c 34
+Python/errors.c 1
+Python/fileutils.c 7
+Python/flowgraph.c 8
+Python/formatter_unicode.c 7
+Python/frame.c 4
+Python/gc.c 8
+Python/generated_cases.c.h 35
+Python/getargs.c 11
+Python/import.c 5
+Python/initconfig.c 11
+Python/instrumentation.c 31
+Python/intrinsics.c 1
+Python/legacy_tracing.c 3
+Python/lock.c 4
+Python/marshal.c 11
+Python/modsupport.c 3
+Python/mystrtoul.c 4
+Python/pathconfig.c 1
+Python/preconfig.c 2
+Python/pyarena.c 1
+Python/pyhash.c 2
+Python/pylifecycle.c 7
+Python/pystate.c 6
+Python/pystrhex.c 19
+Python/pystrtod.c 3
+Python/qsbr.c 2
+Python/specialize.c 10
+Python/suggestions.c 12
+Python/symtable.c 18
+Python/sysmodule.c 2
+Python/thread_pthread.h 1
+Python/traceback.c 6
+Python/tracemalloc.c 6
# Keep lines sorted lexicographically to help avoid merge conflicts.
# Format example:
# /path/to/file (number of warnings in file)
+/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/evp.h 2
+/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/ssl.h 4
+/usr/include/tcl8.6/tclTomMathDecls.h 1
+Include/cpython/bytearrayobject.h 1
+Include/cpython/bytesobject.h 3
+Include/cpython/dictobject.h 2
+Include/cpython/listobject.h 1
+Include/cpython/pyctype.h 2
+Include/cpython/tupleobject.h 1
+Include/cpython/unicodeobject.h 7
+Include/internal/mimalloc/mimalloc/internal.h 4
+Include/internal/mimalloc/mimalloc/types.h 2
+Include/internal/pycore_asdl.h 1
+Include/internal/pycore_backoff.h 3
+Include/internal/pycore_blocks_output_buffer.h 1
+Include/internal/pycore_dict.h 2
+Include/internal/pycore_gc.h 1
+Include/internal/pycore_gc.h 1
+Include/internal/pycore_interp.h 1
+Include/internal/pycore_list.h 1
+Include/internal/pycore_long.h 3
+Include/internal/pycore_object.h 4
+Include/internal/pycore_obmalloc.h 1
+Include/internal/pycore_pymath.h 1
+Include/internal/pycore_runtime_init.h 1
+Include/longobject.h 1
+Include/object.h 4
+Include/opcode_ids.h 1
+Include/pymacro.h 4
+Include/pymath.h 1
+Include/pymem.h 2
+Include/pyport.h 2
+Modules/_asynciomodule.c 3
+Modules/_bisectmodule.c 4
+Modules/_bz2module.c 5
+Modules/_collectionsmodule.c 2
+Modules/_csv.c 2
+Modules/_ctypes/_ctypes.c 53
+Modules/_ctypes/_ctypes_test.c 7
+Modules/_ctypes/_ctypes_test_generated.c.h 2
+Modules/_ctypes/callbacks.c 3
+Modules/_ctypes/callproc.c 13
+Modules/_ctypes/cfield.c 33
+Modules/_ctypes/stgdict.c 17
+Modules/_cursesmodule.c 27
+Modules/_datetimemodule.c 38
+Modules/_datetimemodule.c 38
+Modules/_dbmmodule.c 7
+Modules/_decimal/_decimal.c 19
+Modules/_elementtree.c 37
+Modules/_functoolsmodule.c 6
+Modules/_gdbmmodule.c 4
+Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84
+Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84
+Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84
+Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84
+Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4
+Modules/_hashopenssl.c 13
+Modules/_io/_iomodule.c 1
+Modules/_io/bufferedio.c 15
+Modules/_io/bytesio.c 14
+Modules/_io/fileio.c 9
+Modules/_io/stringio.c 8
+Modules/_io/textio.c 17
+Modules/_json.c 19
+Modules/_localemodule.c 2
+Modules/_lsprof.c 5
+Modules/_lzmamodule.c 6
+Modules/_multiprocessing/posixshmem.c 1
+Modules/_multiprocessing/semaphore.c 1
+Modules/_operator.c 5
+Modules/_pickle.c 73
+Modules/_posixsubprocess.c 11
+Modules/_queuemodule.c 4
+Modules/_randommodule.c 3
+Modules/_sqlite/connection.c 5
+Modules/_sqlite/cursor.c 3
+Modules/_sqlite/module.c 2
+Modules/_sre/sre.c 14
+Modules/_sre/sre_lib.h 25
+Modules/_ssl.c 26
+Modules/_struct.c 3
+Modules/_testbuffer.c 27
+Modules/_testcapi/bytes.c 1
+Modules/_testcapi/heaptype.c 1
+Modules/_testcapi/long.c 2
+Modules/_testcapi/mem.c 2
+Modules/_testcapi/monitoring.c 3
+Modules/_testcapi/pyatomic.c 4
+Modules/_testcapi/pyatomic.c 4
+Modules/_testcapi/unicode.c 1
+Modules/_testcapi/vectorcall.c 3
+Modules/_testcapi/watchers.c 3
+Modules/_testcapimodule.c 1
+Modules/_testclinic.c 14
+Modules/_testclinic.c 14
+Modules/_testexternalinspection.c 7
+Modules/_testinternalcapi.c 10
+Modules/_testinternalcapi/test_critical_sections.c 1
+Modules/_testinternalcapi/test_lock.c 4
+Modules/_testlimitedcapi/heaptype_relative.c 3
+Modules/_testlimitedcapi/object.c 2
+Modules/_testlimitedcapi/unicode.c 1
+Modules/_testmultiphase.c 1
+Modules/_tkinter.c 8
+Modules/_xxtestfuzz/_xxtestfuzz.c 1
+Modules/_xxtestfuzz/fuzzer.c 13
+Modules/_zoneinfo.c 17
+Modules/arraymodule.c 48
+Modules/binascii.c 208
+Modules/blake2module.c 8
+Modules/cjkcodecs/_codecs_iso2022.c 1
+Modules/cjkcodecs/_codecs_jp.c 17
+Modules/cjkcodecs/_codecs_kr.c 7
+Modules/cjkcodecs/alg_jisx0201.h 2
+Modules/cjkcodecs/cjkcodecs.h 1
+Modules/cjkcodecs/multibytecodec.c 12
+Modules/expat/pyexpatns.h 3
+Modules/expat/siphash.h 1
+Modules/expat/xmlparse.c 43
+Modules/expat/xmltok.c 15
+Modules/expat/xmltok.c 15
+Modules/expat/xmltok_impl.c 8
+Modules/faulthandler.c 5
+Modules/fcntlmodule.c 6
+Modules/getpath.c 7
+Modules/grpmodule.c 4
+Modules/itertoolsmodule.c 4
+Modules/main.c 2
+Modules/mathmodule.c 14
+Modules/mmapmodule.c 22
+Modules/mmapmodule.c 22
+Modules/posixmodule.c 79
+Modules/pwdmodule.c 4
+Modules/pyexpat.c 10
+Modules/readline.c 1
+Modules/resource.c 4
+Modules/rotatingtree.c 2
+Modules/selectmodule.c 1
+Modules/sha3module.c 4
+Modules/signalmodule.c 3
+Modules/socketmodule.c 75
+Modules/syslogmodule.c 3
+Modules/termios.c 1
+Modules/timemodule.c 10
+Modules/unicodedata.c 24
+Modules/unicodedata_db.h 1
+Modules/zlibmodule.c 24
+Objects/abstract.c 6
+Objects/bytearrayobject.c 42
+Objects/bytes_methods.c 4
+Objects/bytesobject.c 45
+Objects/call.c 12
+Objects/classobject.c 4
+Objects/codeobject.c 19
+Objects/descrobject.c 2
+Objects/dictobject.c 31
+Objects/fileobject.c 3
+Objects/floatobject.c 10
+Objects/frameobject.c 16
+Objects/funcobject.c 1
+Objects/genobject.c 3
+Objects/listobject.c 38
+Objects/longobject.c 47
+Objects/memoryobject.c 12
+Objects/methodobject.c 1
+Objects/mimalloc/alloc.c 6
+Objects/mimalloc/arena.c 6
+Objects/mimalloc/heap.c 2
+Objects/mimalloc/init.c 2
+Objects/mimalloc/options.c 4
+Objects/mimalloc/os.c 4
+Objects/mimalloc/page-queue.c 2
+Objects/mimalloc/page.c 2
+Objects/mimalloc/prim/unix/prim.c 6
+Objects/mimalloc/random.c 1
+Objects/mimalloc/segment.c 11
+Objects/mimalloc/stats.c 5
+Objects/moduleobject.c 4
+Objects/object.c 1
+Objects/obmalloc.c 6
+Objects/odictobject.c 6
+Objects/rangeobject.c 10
+Objects/setobject.c 13
+Objects/sliceobject.c 2
+Objects/stringlib/codecs.h 12
+Objects/stringlib/eq.h 1
+Objects/stringlib/fastsearch.h 8
+Objects/stringlib/join.h 3
+Objects/stringlib/replace.h 4
+Objects/stringlib/repr.h 21
+Objects/stringlib/transmogrify.h 26
+Objects/structseq.c 10
+Objects/tupleobject.c 8
+Objects/typeobject.c 38
+Objects/unicodectype.c 7
+Objects/unicodeobject.c 135
+Parser/action_helpers.c 3
+Parser/lexer/buffer.c 1
+Parser/lexer/lexer.c 14
+Parser/parser.c 116
+Parser/pegen.c 8
+Parser/string_parser.c 7
+Parser/tokenizer/file_tokenizer.c 9
+Parser/tokenizer/helpers.c 7
+Parser/tokenizer/readline_tokenizer.c 4
+Python/assemble.c 11
+Python/ast_opt.c 5
+Python/bltinmodule.c 8
+Python/bootstrap_hash.c 7
+Python/ceval.c 8
+Python/ceval_gil.c 2
+Python/codecs.c 28
+Python/codegen.c 6
+Python/compile.c 2
+Python/context.c 1
+Python/crossinterp.c 2
+Python/crossinterp_data_lookup.h 1
+Python/dtoa.c 30
+Python/errors.c 1
+Python/fileutils.c 11
+Python/flowgraph.c 7
+Python/formatter_unicode.c 6
+Python/frame.c 3
+Python/gc.c 9
+Python/gc.c 9
+Python/generated_cases.c.h 27
+Python/generated_cases.c.h 27
+Python/getargs.c 7
+Python/hashtable.c 1
+Python/import.c 6
+Python/import.c 7
+Python/initconfig.c 11
+Python/instrumentation.c 43
+Python/intrinsics.c 1
+Python/legacy_tracing.c 3
+Python/lock.c 4
+Python/marshal.c 16
+Python/modsupport.c 3
+Python/mystrtoul.c 4
+Python/pathconfig.c 1
+Python/perf_jit_trampoline.c 32
+Python/perf_trampoline.c 12
+Python/preconfig.c 2
+Python/pyarena.c 1
+Python/pyhash.c 4
+Python/pylifecycle.c 3
+Python/pystate.c 4
+Python/pystrhex.c 15
+Python/pystrtod.c 12
+Python/pytime.c 2
+Python/qsbr.c 2
+Python/specialize.c 9
+Python/suggestions.c 12
+Python/symtable.c 15
+Python/sysmodule.c 2
+Python/thread.c 1
+Python/thread_pthread.h 6
+Python/traceback.c 6
+Python/tracemalloc.c 6
"""
-Parses compiler output with -fdiagnostics-format=json and checks that warnings
+Parses compiler output from Clang or GCC and checks that warnings
exist only in files that are expected to have warnings.
"""
import argparse
from collections import defaultdict
-import json
import re
import sys
from pathlib import Path
from typing import NamedTuple
+
class FileWarnings(NamedTuple):
name: str
count: int
-def extract_warnings_from_compiler_output_clang(
+def extract_warnings_from_compiler_output(
compiler_output: str,
+ compiler_output_type: str,
+ path_prefix: str = "",
) -> list[dict]:
"""
- Extracts warnings from the compiler output when using clang
+ Extracts warnings from the compiler output based on compiler
+ output type. Removes path prefix from file paths if provided.
+ Compatible with GCC and Clang compiler output.
"""
- # Regex to find warnings in the compiler output
- clang_warning_regex = re.compile(
- r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: "
- r"(?P<message>.*) (?P<option>\[-[^\]]+\])$"
- )
+ # Choose pattern and compile regex for particular compiler output
+ if compiler_output_type == "gcc":
+ regex_pattern = (
+ r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: "
+ r"(?P<message>.*?)(?: (?P<option>\[-[^\]]+\]))?$"
+ )
+ elif compiler_output_type == "clang":
+ regex_pattern = (
+ r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: "
+ r"(?P<message>.*) (?P<option>\[-[^\]]+\])$"
+ )
+ compiled_regex = re.compile(regex_pattern)
compiler_warnings = []
for line in compiler_output.splitlines():
- if match := clang_warning_regex.match(line):
+ if match := compiled_regex.match(line):
compiler_warnings.append(
{
- "file": match.group("file"),
+ "file": match.group("file").removeprefix(path_prefix),
"line": match.group("line"),
"column": match.group("column"),
"message": match.group("message"),
return compiler_warnings
-def extract_warnings_from_compiler_output_json(
- compiler_output: str,
-) -> list[dict]:
- """
- Extracts warnings from the compiler output when using
- -fdiagnostics-format=json.
-
- Compiler output as a whole is not a valid json document,
- but includes many json objects and may include other output
- that is not json.
- """
- # Regex to find json arrays at the top level of the file
- # in the compiler output
- json_arrays = re.findall(r"\[(?:[^[\]]|\[[^]]*])*]", compiler_output)
- compiler_warnings = []
- for array in json_arrays:
- try:
- json_data = json.loads(array)
- json_objects_in_array = [entry for entry in json_data]
- warning_list = [
- entry
- for entry in json_objects_in_array
- if entry.get("kind") == "warning"
- ]
- for warning in warning_list:
- locations = warning["locations"]
- for location in locations:
- for key in ["caret", "start", "end"]:
- if key in location:
- compiler_warnings.append(
- {
- # Remove leading current directory if present
- "file": location[key]["file"].lstrip("./"),
- "line": location[key]["line"],
- "column": location[key]["column"],
- "message": warning["message"],
- "option": warning["option"],
- }
- )
- # Found a caret, start, or end in location so
- # break out completely to address next warning
- break
- else:
- continue
- break
-
- except json.JSONDecodeError:
- continue # Skip malformed JSON
-
- return compiler_warnings
-
-
def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]:
"""
- Returns a dictionary where the key is the file and the data is the warnings
- in that file. Does not include duplicate warnings for a file from list of
- provided warnings.
+ Returns a dictionary where the key is the file and the data is the
+ warnings in that file. Does not include duplicate warnings for a
+ file from list of provided warnings.
"""
warnings_by_file = defaultdict(list)
warnings_added = set()
are associated with a file that is not found in the list of files
with expected warnings
"""
- unexpected_warnings = []
+ unexpected_warnings = {}
for file in files_with_warnings.keys():
found_file_in_ignore_list = False
for ignore_file in files_with_expected_warnings:
if file == ignore_file.name:
if len(files_with_warnings[file]) > ignore_file.count:
- unexpected_warnings.extend(files_with_warnings[file])
+ unexpected_warnings[file] = (files_with_warnings[file], ignore_file.count)
found_file_in_ignore_list = True
break
if not found_file_in_ignore_list:
- unexpected_warnings.extend(files_with_warnings[file])
+ unexpected_warnings[file] = (files_with_warnings[file], 0)
if unexpected_warnings:
print("Unexpected warnings:")
- for warning in unexpected_warnings:
- print(warning)
+ for file in unexpected_warnings:
+ print(
+ f"{file} expected {unexpected_warnings[file][1]} warnings,"
+ f" found {len(unexpected_warnings[file][0])}"
+ )
+ for warning in unexpected_warnings[file][0]:
+ print(warning)
+
return 1
return 0
unexpected_improvements = []
for file in files_with_expected_warnings:
if file.name not in files_with_warnings.keys():
- unexpected_improvements.append(file)
+ unexpected_improvements.append((file.name, file.count, 0))
elif len(files_with_warnings[file.name]) < file.count:
- unexpected_improvements.append(file)
+ unexpected_improvements.append((file.name, file.count, len(files_with_warnings[file.name])))
if unexpected_improvements:
print("Unexpected improvements:")
for file in unexpected_improvements:
- print(file.name)
+ print(f"{file[0]} expected {file[1]} warnings, found {file[2]}")
return 1
return 0
"--compiler-output-type",
type=str,
required=True,
- choices=["json", "clang"],
- help="Type of compiler output file (json or clang)",
+ choices=["gcc", "clang"],
+ help="Type of compiler output file (GCC or Clang)",
+ )
+ parser.add_argument(
+ "-p",
+ "--path-prefix",
+ type=str,
+ help="Path prefix to remove from the start of file paths"
+ " in compiler output",
)
args = parser.parse_args(argv)
# where the first element is the file name and the second element
# is the number of warnings expected in that file
files_with_expected_warnings = {
- FileWarnings(file.strip().split()[0], int(file.strip().split()[1]))
+ FileWarnings(
+ file.strip().split()[0], int(file.strip().split()[1])
+ )
for file in clean_files
if file.strip() and not file.startswith("#")
}
with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f:
compiler_output_file_contents = f.read()
- if args.compiler_output_type == "json":
- warnings = extract_warnings_from_compiler_output_json(
- compiler_output_file_contents
- )
- elif args.compiler_output_type == "clang":
- warnings = extract_warnings_from_compiler_output_clang(
- compiler_output_file_contents
- )
+ warnings = extract_warnings_from_compiler_output(
+ compiler_output_file_contents,
+ args.compiler_output_type,
+ args.path_prefix
+ )
files_with_warnings = get_warnings_by_file(warnings)
if args.fail_on_improvement:
exit_code |= status
+ print(
+ "For information about this tool and its configuration"
+ " visit https://devguide.python.org/development-tools/warnings/"
+ )
+
return exit_code
else $as_nop
{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wtrampolines not supported" >&5
printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;}
+fi
+
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wconversion" >&5
+printf %s "checking whether C compiler accepts -Wconversion... " >&6; }
+if test ${ax_cv_check_cflags__Werror__Wconversion+y}
+then :
+ printf %s "(cached) " >&6
+else $as_nop
+
+ ax_check_save_flags=$CFLAGS
+ CFLAGS="$CFLAGS -Werror -Wconversion"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main (void)
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+ ax_cv_check_cflags__Werror__Wconversion=yes
+else $as_nop
+ ax_cv_check_cflags__Werror__Wconversion=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+ CFLAGS=$ax_check_save_flags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wconversion" >&5
+printf "%s\n" "$ax_cv_check_cflags__Werror__Wconversion" >&6; }
+if test "x$ax_cv_check_cflags__Werror__Wconversion" = xyes
+then :
+ CFLAGS_NODIST="$CFLAGS_NODIST -Wconversion"
+else $as_nop
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wconversion not supported" >&5
+printf "%s\n" "$as_me: WARNING: -Wconversion not supported" >&2;}
+fi
+
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wimplicit-fallthrough" >&5
+printf %s "checking whether C compiler accepts -Wimplicit-fallthrough... " >&6; }
+if test ${ax_cv_check_cflags__Werror__Wimplicit_fallthrough+y}
+then :
+ printf %s "(cached) " >&6
+else $as_nop
+
+ ax_check_save_flags=$CFLAGS
+ CFLAGS="$CFLAGS -Werror -Wimplicit-fallthrough"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main (void)
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+ ax_cv_check_cflags__Werror__Wimplicit_fallthrough=yes
+else $as_nop
+ ax_cv_check_cflags__Werror__Wimplicit_fallthrough=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+ CFLAGS=$ax_check_save_flags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&5
+printf "%s\n" "$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&6; }
+if test "x$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" = xyes
+then :
+ CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough"
+else $as_nop
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wimplicit-fallthrough not supported" >&5
+printf "%s\n" "$as_me: WARNING: -Wimplicit-fallthrough not supported" >&2;}
+fi
+
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Werror=format-security" >&5
+printf %s "checking whether C compiler accepts -Werror=format-security... " >&6; }
+if test ${ax_cv_check_cflags__Werror__Werror_format_security+y}
+then :
+ printf %s "(cached) " >&6
+else $as_nop
+
+ ax_check_save_flags=$CFLAGS
+ CFLAGS="$CFLAGS -Werror -Werror=format-security"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main (void)
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+ ax_cv_check_cflags__Werror__Werror_format_security=yes
+else $as_nop
+ ax_cv_check_cflags__Werror__Werror_format_security=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+ CFLAGS=$ax_check_save_flags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Werror_format_security" >&5
+printf "%s\n" "$ax_cv_check_cflags__Werror__Werror_format_security" >&6; }
+if test "x$ax_cv_check_cflags__Werror__Werror_format_security" = xyes
+then :
+ CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security"
+else $as_nop
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Werror=format-security not supported" >&5
+printf "%s\n" "$as_me: WARNING: -Werror=format-security not supported" >&2;}
+fi
+
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wbidi-chars=any" >&5
+printf %s "checking whether C compiler accepts -Wbidi-chars=any... " >&6; }
+if test ${ax_cv_check_cflags__Werror__Wbidi_chars_any+y}
+then :
+ printf %s "(cached) " >&6
+else $as_nop
+
+ ax_check_save_flags=$CFLAGS
+ CFLAGS="$CFLAGS -Werror -Wbidi-chars=any"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main (void)
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+ ax_cv_check_cflags__Werror__Wbidi_chars_any=yes
+else $as_nop
+ ax_cv_check_cflags__Werror__Wbidi_chars_any=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+ CFLAGS=$ax_check_save_flags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wbidi_chars_any" >&5
+printf "%s\n" "$ax_cv_check_cflags__Werror__Wbidi_chars_any" >&6; }
+if test "x$ax_cv_check_cflags__Werror__Wbidi_chars_any" = xyes
+then :
+ CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any"
+else $as_nop
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wbidi-chars=any not supported" >&5
+printf "%s\n" "$as_me: WARNING: -Wbidi-chars=any not supported" >&2;}
+fi
+
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wall" >&5
+printf %s "checking whether C compiler accepts -Wall... " >&6; }
+if test ${ax_cv_check_cflags__Werror__Wall+y}
+then :
+ printf %s "(cached) " >&6
+else $as_nop
+
+ ax_check_save_flags=$CFLAGS
+ CFLAGS="$CFLAGS -Werror -Wall"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+int
+main (void)
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+ ax_cv_check_cflags__Werror__Wall=yes
+else $as_nop
+ ax_cv_check_cflags__Werror__Wall=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+ CFLAGS=$ax_check_save_flags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wall" >&5
+printf "%s\n" "$ax_cv_check_cflags__Werror__Wall" >&6; }
+if test "x$ax_cv_check_cflags__Werror__Wall" = xyes
+then :
+ CFLAGS_NODIST="$CFLAGS_NODIST -Wall"
+else $as_nop
+ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wall not supported" >&5
+printf "%s\n" "$as_me: WARNING: -Wall not supported" >&2;}
fi
fi
then
AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [CFLAGS_NODIST="$CFLAGS_NODIST -fstack-protector-strong"], [AC_MSG_WARN([-fstack-protector-strong not supported])], [-Werror])
AX_CHECK_COMPILE_FLAG([-Wtrampolines], [CFLAGS_NODIST="$CFLAGS_NODIST -Wtrampolines"], [AC_MSG_WARN([-Wtrampolines not supported])], [-Werror])
+ AX_CHECK_COMPILE_FLAG([-Wconversion], [CFLAGS_NODIST="$CFLAGS_NODIST -Wconversion"], [AC_MSG_WARN([-Wconversion not supported])], [-Werror])
+ AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough"], [AC_MSG_WARN([-Wimplicit-fallthrough not supported])], [-Werror])
+ AX_CHECK_COMPILE_FLAG([-Werror=format-security], [CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security"], [AC_MSG_WARN([-Werror=format-security not supported])], [-Werror])
+ AX_CHECK_COMPILE_FLAG([-Wbidi-chars=any], [CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any"], [AC_MSG_WARN([-Wbidi-chars=any not supported])], [-Werror])
+ AX_CHECK_COMPILE_FLAG([-Wall], [CFLAGS_NODIST="$CFLAGS_NODIST -Wall"], [AC_MSG_WARN([-Wall not supported])], [-Werror])
fi
AC_MSG_CHECKING([for --enable-slower-safety])
projects / thirdparty / Python / cpython.git / commitdiff
