bpo-36876: Make sure the c-analyzer is checking all the source files.' (gh-31264)

https://bugs.python.org/issue36876

diff --git a/Tools/c-analyzer/cpython/_analyzer.py b/Tools/c-analyzer/cpython/_analyzer.py
index 49de9d887284a654021f41fc1a984b4820c85701..741fbaddf16d95a4c05f989a454700a53d816251 100644 (file)
--- a/Tools/c-analyzer/cpython/_analyzer.py
+++ b/Tools/c-analyzer/cpython/_analyzer.py
@@ -307,7 +307,7 @@ class Analyzed(_info.Analyzed):
         # XXX ???
         #if itemonly:
         #    yield from rendered
-        supported = self._supported
+        supported = self.supported
         if fmt in ('line', 'brief'):
             rendered, = rendered
             parts = [

diff --git a/Tools/c-analyzer/cpython/_files.py b/Tools/c-analyzer/cpython/_files.py
index 3e397880977ab242ffe2d75f56095b0ef6c970af..ee9e46f7e5e95f5510772e10e9a49d3d12c65668 100644 (file)
--- a/Tools/c-analyzer/cpython/_files.py
+++ b/Tools/c-analyzer/cpython/_files.py
@@ -6,15 +6,17 @@ from . import REPO_ROOT, INCLUDE_DIRS, SOURCE_DIRS
 
 GLOBS = [
     'Include/*.h',
+    # Technically, this is covered by "Include/*.h":
+    #'Include/cpython/*.h',
     'Include/internal/*.h',
     'Modules/**/*.h',
     'Modules/**/*.c',
     'Objects/**/*.h',
     'Objects/**/*.c',
-    'Python/**/*.h',
+    'Parser/**/*.h',
     'Parser/**/*.c',
     'Python/**/*.h',
-    'Parser/**/*.c',
+    'Python/**/*.c',
 ]
 LEVEL_GLOBS = {
     'stable': 'Include/*.h',

diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py
index 6b55c41c2c71e16d5f48d8f0ff20807682ddab21..e58871cf73dcec2fbeb460761904b4fae455a96e 100644 (file)
--- a/Tools/c-analyzer/cpython/_parser.py
+++ b/Tools/c-analyzer/cpython/_parser.py
@@ -81,6 +81,7 @@ Modules/unicodename_db.h
 Objects/unicodetype_db.h
 
 # generated
+Python/deepfreeze/*.c
 Python/frozen_modules/*.h
 Python/opcode_targets.h
 Python/stdlib_module_names.h
@@ -157,6 +158,7 @@ Modules/cmathmodule.c       Py_BUILD_CORE   1
 Modules/faulthandler.c Py_BUILD_CORE   1
 Modules/gcmodule.c     Py_BUILD_CORE   1
 Modules/getpath.c      Py_BUILD_CORE   1
+Modules/getpath_noop.c Py_BUILD_CORE   1
 Modules/itertoolsmodule.c      Py_BUILD_CORE   1
 Modules/main.c Py_BUILD_CORE   1
 Modules/mathmodule.c   Py_BUILD_CORE   1
@@ -169,9 +171,10 @@ Modules/timemodule.c       Py_BUILD_CORE   1
 Modules/unicodedata.c  Py_BUILD_CORE   1
 Objects/stringlib/codecs.h     Py_BUILD_CORE   1
 Objects/stringlib/unicode_format.h     Py_BUILD_CORE   1
+Parser/string_parser.h Py_BUILD_CORE   1
+Parser/pegen.h Py_BUILD_CORE   1
 Python/ceval_gil.h     Py_BUILD_CORE   1
 Python/condvar.h       Py_BUILD_CORE   1
-Modules/getpath_noop.c Py_BUILD_CORE   1
 
 Modules/_json.c        Py_BUILD_CORE_BUILTIN   1
 Modules/_pickle.c      Py_BUILD_CORE_BUILTIN   1
@@ -293,6 +296,9 @@ MAX_SIZES = {
     _abs('Modules/expat/expat.h'): (10_000, 400),
     _abs('Objects/stringlib/unicode_format.h'): (10_000, 400),
     _abs('Objects/typeobject.c'): (20_000, 200),
+    _abs('Python/compile.c'): (20_000, 500),
+    _abs('Python/pylifecycle.c'): (200_000, 5000),
+    _abs('Python/pystate.c'): (200_000, 5000),
 }
 
 

diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv
index 56904814858313b790f3910e202032ca0234c867..4ef0a6c848b7e452be20858543cf52e7f23c8d2a 100644 (file)
--- a/Tools/c-analyzer/cpython/globals-to-fix.tsv
+++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv
@@ -106,6 +106,28 @@ Objects/unionobject.c      -       _Py_UnionType   -
 Objects/weakrefobject.c        -       _PyWeakref_CallableProxyType    -
 Objects/weakrefobject.c        -       _PyWeakref_ProxyType    -
 Objects/weakrefobject.c        -       _PyWeakref_RefType      -
+Python/bltinmodule.c   -       PyFilter_Type   -
+Python/bltinmodule.c   -       PyMap_Type      -
+Python/bltinmodule.c   -       PyZip_Type      -
+Python/context.c       -       PyContextToken_Type     -
+Python/context.c       -       PyContextVar_Type       -
+Python/context.c       -       PyContext_Type  -
+Python/context.c       -       _PyContextTokenMissing_Type     -
+Python/errors.c        -       UnraisableHookArgsType  -
+Python/hamt.c  -       _PyHamtItems_Type       -
+Python/hamt.c  -       _PyHamtKeys_Type        -
+Python/hamt.c  -       _PyHamtValues_Type      -
+Python/hamt.c  -       _PyHamt_ArrayNode_Type  -
+Python/hamt.c  -       _PyHamt_BitmapNode_Type -
+Python/hamt.c  -       _PyHamt_CollisionNode_Type      -
+Python/hamt.c  -       _PyHamt_Type    -
+Python/symtable.c      -       PySTEntry_Type  -
+Python/sysmodule.c     -       AsyncGenHooksType       -
+Python/sysmodule.c     -       FlagsType       -
+Python/sysmodule.c     -       Hash_InfoType   -
+Python/sysmodule.c     -       VersionInfoType -
+Python/thread.c        -       ThreadInfoType  -
+Python/traceback.c     -       PyTraceBack_Type        -
 
 #-----------------------
 # builtin exception types
@@ -306,12 +328,20 @@ Python/compile.c  compiler_visit_annotations      return_str      -
 Python/import.c        PyImport_Import builtins_str    -
 # XXX This should have been found by the analyzer but wasn't:
 Python/import.c        PyImport_Import import_str      -
+Python/symtable.c      -       __class__       -
+Python/symtable.c      -       _annotation     -
+Python/symtable.c      -       dictcomp        -
+Python/symtable.c      -       genexpr -
+Python/symtable.c      -       lambda  -
+Python/symtable.c      -       listcomp        -
+Python/symtable.c      -       setcomp -
+Python/symtable.c      -       top     -
 # XXX This should have been found by the analyzer but wasn't:
 Python/sysmodule.c     -       whatstrings     -
 # XXX This should have been found by the analyzer but wasn't:
 Python/sysmodule.c     sys_displayhook newline -
 
-# _PyArg_Parser
+# _PyArg_Parser (holds tuple of strings)
 Objects/clinic/bytearrayobject.c.h     bytearray___init__      _parser -
 Objects/clinic/bytearrayobject.c.h     bytearray_decode        _parser -
 Objects/clinic/bytearrayobject.c.h     bytearray_hex   _parser -
@@ -367,6 +397,9 @@ Python/clinic/sysmodule.c.h sys_addaudithook        _parser -
 Python/clinic/sysmodule.c.h    sys_set_coroutine_origin_tracking_depth _parser -
 Python/clinic/traceback.c.h    tb_new  _parser -
 
+# holds strings
+Objects/typeobject.c   -       slotdefs        -
+
 # other
 Objects/typeobject.c   -       method_cache    -
 Objects/unicodeobject.c        -       _string_module  -
@@ -400,29 +433,73 @@ Python/import.c   -       extensions      -
 # global non-objects to fix in core code
 
 #-----------------------
-# initialized once
+# initialized/set once
 
 # pre-allocated buffer
 Modules/getbuildinfo.c Py_GetBuildInfo buildinfo       -
 
 # during init
+Objects/typeobject.c   -       slotdefs_initialized    -
+Objects/unicodeobject.c        -       bloom_linebreak -
 Parser/parser.c        -       Py_DebugFlag    -
-
-# other
-Objects/codeobject.c   PyCode_NewEmpty nulltuple       -
+Python/bootstrap_hash.c        -       _Py_HashSecret_Initialized      -
+Python/bootstrap_hash.c        py_getrandom    getrandom_works -
+Python/pyhash.c        -       _Py_HashSecret  -
+Python/pylifecycle.c   -       runtime_initialized     -
+Python/sysmodule.c     -       _PySys_ImplCacheTag     -
+Python/sysmodule.c     -       _PySys_ImplName -
+Python/sysmodule.c     -       _preinit_warnoptions    -
+Python/sysmodule.c     -       _preinit_xoptions       -
+Python/thread.c        -       thread_debug    -
+Python/thread.c        -       initialized     -
+
+# lazy
+Objects/floatobject.c  -       double_format   -
+Objects/floatobject.c  -       float_format    -
+Objects/floatobject.c  -       detected_double_format  -
+Objects/floatobject.c  -       detected_float_format   -
 Objects/longobject.c   PyLong_FromString       log_base_BASE   -
 Objects/longobject.c   PyLong_FromString       convwidth_base  -
 Objects/longobject.c   PyLong_FromString       convmultmax_base        -
-Objects/typeobject.c   -       slotdefs        -
-Objects/typeobject.c   -       slotdefs_initialized    -
-Objects/unicodeobject.c        -       bloom_linebreak -
 Objects/unicodeobject.c        -       ucnhash_capi    -
 Parser/action_helpers.c        _PyPegen_dummy_name     cache   -
-Parser/pegen.c _PyPegen_dummy_name     cache   -
+Python/dtoa.c  -       p5s     -
+Python/fileutils.c     -       force_ascii     -
+Python/fileutils.c     set_inheritable ioctl_works     -
 # XXX This should have been found by the analyzer but wasn't:
 Python/import.c        -       import_lock     -
 # XXX This should have been found by the analyzer but wasn't:
 Python/import.c        import_find_and_load    header  -
+Python/specialize.c    -       _list_append    -
+
+#-----------------------
+# unlikely to change after init (or main thread)
+
+# through C-API
+Python/frozen.c        -       PyImport_FrozenModules  -
+Python/frozen.c        -       _PyImport_FrozenAliases -
+Python/frozen.c        -       _PyImport_FrozenBootstrap       -
+Python/frozen.c        -       _PyImport_FrozenStdlib  -
+Python/frozen.c        -       _PyImport_FrozenTest    -
+Python/import.c        -       inittab_copy    -
+Python/import.c        -       PyImport_Inittab        -
+Python/preconfig.c     -       Py_FileSystemDefaultEncoding    -
+Python/preconfig.c     -       Py_HasFileSystemDefaultEncoding -
+Python/preconfig.c     -       Py_FileSystemDefaultEncodeErrors        -
+Python/preconfig.c     -       _Py_HasFileSystemDefaultEncodeErrors    -
+
+# REPL
+Parser/myreadline.c    -       _PyOS_ReadlineLock      -
+Parser/myreadline.c    -       _PyOS_ReadlineTState    -
+Parser/myreadline.c    -       PyOS_InputHook  -
+Parser/myreadline.c    -       PyOS_ReadlineFunctionPointer    -
+
+# handling C argv
+Python/getopt.c        -       _PyOS_optarg    -
+Python/getopt.c        -       _PyOS_opterr    -
+Python/getopt.c        -       _PyOS_optind    -
+Python/getopt.c        -       opt_ptr -
+Python/pathconfig.c    -       _Py_path_config -
 
 #-----------------------
 # state
@@ -444,20 +521,22 @@ Objects/obmalloc.c        -       unused_arena_objects    -
 Objects/obmalloc.c     -       usable_arenas   -
 Objects/obmalloc.c     new_arena       debug_stats     -
 
-# REPL
-Parser/myreadline.c    -       _PyOS_ReadlineLock      -
-Parser/myreadline.c    -       _PyOS_ReadlineTState    -
-Parser/myreadline.c    -       PyOS_InputHook  -
-Parser/myreadline.c    -       PyOS_ReadlineFunctionPointer    -
+# pre-allocated memory
+Python/dtoa.c  -       freelist        -
+Python/dtoa.c  -       private_mem     -
+
+# local buffer
+Python/getversion.c    Py_GetVersion   version -
+Python/suggestions.c   levenshtein_distance    buffer  -
+
+# linked list
+Python/dtoa.c  -       pmem_next       -
+Python/getargs.c       -       static_arg_parsers      -
 
 # other
 Objects/dictobject.c   -       _pydict_global_version  -
 Objects/dictobject.c   -       next_dict_keys_version  -
 Objects/dictobject.c   -       pydict_global_version   -
-Objects/floatobject.c  -       double_format   -
-Objects/floatobject.c  -       float_format    -
-Objects/floatobject.c  -       detected_double_format  -
-Objects/floatobject.c  -       detected_float_format   -
 Objects/funcobject.c   -       next_func_version       -
 Objects/moduleobject.c -       max_module_number       -
 Objects/object.c       -       _Py_RefTotal    -
@@ -480,10 +559,14 @@ Python/import.c   -       import_lock_thread      -
 Python/import.c        import_find_and_load    accumulated     -
 # XXX This should have been found by the analyzer but wasn't:
 Python/import.c        import_find_and_load    import_level    -
+Python/modsupport.c    -       _Py_PackageContext      -
+Python/pyfpe.c -       PyFPE_counter   -
+Python/pylifecycle.c   _Py_FatalErrorFormat    reentrant       -
 # XXX This should have been found by the analyzer but wasn't:
 Python/pylifecycle.c   -       _Py_UnhandledKeyboardInterrupt  -
 # XXX This should have been found by the analyzer but wasn't:
 Python/pylifecycle.c   fatal_error     reentrant       -
+Python/specialize.c    -       _Py_QuickenedCount      -
 
 
 ##################################

diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv
index 9b8eff3685a6836cedbfacf7e84572e0ea144418..c945542e29aa5beafdff16baa4e7e425b7fbd641 100644 (file)
--- a/Tools/c-analyzer/cpython/ignored.tsv
+++ b/Tools/c-analyzer/cpython/ignored.tsv
@@ -1,6 +1,12 @@
 filename       funcname        name    reason
 #???   -       somevar ???
 
+##################################
+# ignored by design
+
+Python/pylifecycle.c   -       _PyRuntime      -
+
+
 ##################################
 # forward/extern references
 # XXX The analyzer should have ignored these.
@@ -384,6 +390,13 @@ Objects/interpreteridobject.c      interpid_new    kwlist  -
 Objects/weakrefobject.c        weakref_call    kwlist  -
 Objects/exceptions.c   NameError_init  kwlist  -
 Objects/exceptions.c   AttributeError_init     kwlist  -
+Python/_warnings.c     warnings_warn_explicit  kwd_list        -
+Python/bltinmodule.c   builtin___import__      kwlist  -
+Python/bltinmodule.c   min_max kwlist  -
+Python/bltinmodule.c   zip_new kwlist  -
+Python/context.c       contextvar_tp_new       kwlist  -
+Python/sysmodule.c     sys_getsizeof   kwlist  -
+Python/sysmodule.c     sys_set_asyncgen_hooks  keywords        -
 
 #-----------------------
 # PyModuleDef
@@ -397,6 +410,13 @@ Modules/_threadmodule.c    -       thread_module   -
 Modules/_typingmodule.c        -       typingmodule    -
 Modules/signalmodule.c -       signal_module   -
 Modules/xxlimited_35.c -       xxmodule        -
+Python/Python-ast.c    -       _astmodule      -
+Python/Python-tokenize.c       -       _tokenizemodule -
+Python/_warnings.c     -       warnings_module -
+Python/bltinmodule.c   -       builtinsmodule  -
+Python/import.c        -       imp_module      -
+Python/marshal.c       -       marshalmodule   -
+Python/sysmodule.c     -       sysmodule       -
 
 #-----------------------
 # PyModuleDef_Slot
@@ -478,6 +498,8 @@ Modules/xxmodule.c  -       xx_slots        -
 Modules/xxsubtype.c    -       xxsubtype_slots -
 Modules/zlibmodule.c   -       zlib_slots      -
 Python/Python-ast.c    -       astmodule_slots -
+Python/Python-tokenize.c       -       tokenizemodule_slots    -
+Python/_warnings.c     -       warnings_slots  -
 Python/marshal.c       -       marshalmodule_slots     -
 
 #-----------------------
@@ -798,6 +820,7 @@ Objects/unionobject.c       -       union_methods   -
 Objects/weakrefobject.c        -       proxy_methods   -
 Objects/weakrefobject.c        -       weakref_methods -
 Python/Python-ast.c    -       ast_type_methods        -
+Python/Python-tokenize.c       -       tokenize_methods        -
 Python/_warnings.c     -       warnings_functions      -
 Python/bltinmodule.c   -       builtin_methods -
 Python/bltinmodule.c   -       filter_methods  -
@@ -807,6 +830,7 @@ Python/context.c    -       PyContextTokenType_methods      -
 Python/context.c       -       PyContextVar_methods    -
 Python/context.c       -       PyContext_methods       -
 Python/hamt.c  -       PyHamt_methods  -
+Python/import.c        -       imp_slots       -
 Python/import.c        -       imp_methods     -
 Python/marshal.c       -       marshal_methods -
 Python/sysmodule.c     -       sys_methods     -
@@ -1218,6 +1242,7 @@ Modules/xxlimited_35.c    -       Xxo_Type_slots  -
 Modules/zlibmodule.c   -       Comptype_slots  -
 Modules/zlibmodule.c   -       Decomptype_slots        -
 Python/Python-ast.c    -       AST_type_slots  -
+Python/Python-tokenize.c       -       tokenizeriter_slots     -
 
 # PyType_Spec
 Modules/_abc.c -       _abc_data_type_spec     -
@@ -1317,6 +1342,7 @@ Modules/xxlimited_35.c    -       Xxo_Type_spec   -
 Modules/zlibmodule.c   -       Comptype_spec   -
 Modules/zlibmodule.c   -       Decomptype_spec -
 Python/Python-ast.c    -       AST_type_spec   -
+Python/Python-tokenize.c       -       tokenizeriter_spec      -
 
 #-----------------------
 # for structseq
@@ -1470,4 +1496,24 @@ Objects/unicodeobject.c  unicode_translate_call_errorhandler     argparse        -
 Parser/parser.c        -       reserved_keywords       -
 Parser/parser.c        -       soft_keywords   -
 Parser/tokenizer.c     -       type_comment_prefix     -
+Python/ast_opt.c       fold_unaryop    ops     -
+Python/codecs.c        -       Py_hexdigits    -
+Python/codecs.c        -       ucnhash_capi    -
+Python/codecs.c        _PyCodecRegistry_Init   methods -
+Python/dynload_shlib.c -       _PyImport_DynLoadFiletab        -
+Python/dynload_stub.c  -       _PyImport_DynLoadFiletab        -
+Python/frozen.c        -       aliases -
+Python/frozen.c        -       bootstrap_modules       -
+Python/frozen.c        -       stdlib_modules  -
+Python/frozen.c        -       test_modules    -
+Python/getopt.c        -       longopts        -
+Python/import.c        -       _PyImport_Inittab       -
+Python/import.c        -       _PySys_ImplCacheTag     -
 Python/opcode_targets.h        -       opcode_targets  -
+Python/pyhash.c        -       PyHash_Func     -
+Python/pylifecycle.c   -       _C_LOCALE_WARNING       -
+Python/pylifecycle.c   -       _PyOS_mystrnicmp_hack   -
+Python/pylifecycle.c   -       _TARGET_LOCALES -
+Python/specialize.c    -       adaptive_opcodes        -
+Python/specialize.c    -       cache_requirements      -
+Python/specialize.c    -       compare_masks   -