gh-90110: Fix the c-analyzer Tool (#102483)

Some incompatible changes had gone in, and the "ignore" lists weren't properly undated. This change fixes that. It's necessary prior to enabling test_check_c_globals, which I hope to do soon.

Note that this does include moving last_resort_memory_error to PyInterpreterState.

https://github.com/python/cpython/issues/90110

diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h
index 30c7c4e3bbd067b8ee818336fe12e79cad22dc9f..9957da1fc5f22ad299bc9a5e7077a963955d5bb9 100644 (file)
--- a/Include/internal/pycore_global_objects.h
+++ b/Include/internal/pycore_global_objects.h
@@ -86,6 +86,7 @@ struct _Py_interp_static_objects {
         // hamt_empty is here instead of global because of its weakreflist.
         _PyGC_Head_UNUSED _hamt_empty_gc_not_used;
         PyHamtObject hamt_empty;
+        PyBaseExceptionObject last_resort_memory_error;
     } singletons;
 };
 

diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h
index deac145fff7627e84480de1295df418f35b34497..46a52740eb8a0c3ae8257c3bbadf01461af9e385 100644 (file)
--- a/Include/internal/pycore_intrinsics.h
+++ b/Include/internal/pycore_intrinsics.h
@@ -21,6 +21,6 @@
 typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value);
 typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2);
 
-extern instrinsic_func1 _PyIntrinsics_UnaryFunctions[];
-extern instrinsic_func2 _PyIntrinsics_BinaryFunctions[];
+extern const instrinsic_func1 _PyIntrinsics_UnaryFunctions[];
+extern const instrinsic_func2 _PyIntrinsics_BinaryFunctions[];
 

diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h
index b54adf04761d4ebe92e4055151e9af594e5383e0..a2cc7c87c2f3e29e7ad9cbed60df9a6191fc4e90 100644 (file)
--- a/Include/internal/pycore_runtime_init.h
+++ b/Include/internal/pycore_runtime_init.h
@@ -14,6 +14,9 @@ extern "C" {
 #include "pycore_obmalloc_init.h"
 
 
+extern PyTypeObject _PyExc_MemoryError;
+
+
 /* The static initializers defined here should only be used
    in the runtime init code (in pystate.c and pylifecycle.c). */
 
@@ -120,6 +123,9 @@ extern "C" {
                     .ob_base = _PyObject_IMMORTAL_INIT(&_PyHamt_Type), \
                     .h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \
                 }, \
+                .last_resort_memory_error = { \
+                    _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError), \
+                }, \
             }, \
         }, \
         ._initial_thread = _PyThreadState_INIT, \

diff --git a/Objects/exceptions.c b/Objects/exceptions.c
index 976f84dbf63c933f8093ed5e5def452bc6eb48ba..a473cbdfeda7fc1392ca2393eda30c9ca35c6d5a 100644 (file)
--- a/Objects/exceptions.c
+++ b/Objects/exceptions.c
@@ -3207,8 +3207,6 @@ SimpleExtendsException(PyExc_Exception, ReferenceError,
 
 #define MEMERRORS_SAVE 16
 
-static PyBaseExceptionObject last_resort_memory_error;
-
 static PyObject *
 get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds)
 {
@@ -3216,7 +3214,9 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds)
     struct _Py_exc_state *state = get_exc_state();
     if (state->memerrors_freelist == NULL) {
         if (!allow_allocation) {
-            return Py_NewRef(&last_resort_memory_error);
+            PyInterpreterState *interp = _PyInterpreterState_GET();
+            return Py_NewRef(
+                &_Py_INTERP_SINGLETON(interp, last_resort_memory_error));
         }
         PyObject *result = BaseException_new((PyTypeObject *)PyExc_MemoryError, args, kwds);
         return result;
@@ -3239,8 +3239,6 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds)
     return (PyObject *)self;
 }
 
-static PyBaseExceptionObject last_resort_memory_error;
-
 static PyObject *
 MemoryError_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
@@ -3325,7 +3323,7 @@ free_preallocated_memerrors(struct _Py_exc_state *state)
 }
 
 
-static PyTypeObject _PyExc_MemoryError = {
+PyTypeObject _PyExc_MemoryError = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "MemoryError",
     sizeof(PyBaseExceptionObject),
@@ -3339,9 +3337,6 @@ static PyTypeObject _PyExc_MemoryError = {
 };
 PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError;
 
-static PyBaseExceptionObject last_resort_memory_error = {
-    _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError)
-};
 
 /*
  *    BufferError extends Exception

diff --git a/Python/intrinsics.c b/Python/intrinsics.c
index 9e90ef32130f1d925f9931d195cbde67759a6fd0..cca29d859902a4a7c9e32b0505ec0ab33a3d66c7 100644 (file)
--- a/Python/intrinsics.c
+++ b/Python/intrinsics.c
@@ -199,7 +199,7 @@ list_to_tuple(PyThreadState* unused, PyObject *v)
     return _PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v));
 }
 
-instrinsic_func1
+const instrinsic_func1
 _PyIntrinsics_UnaryFunctions[] = {
     [0] = no_intrinsic,
     [INTRINSIC_PRINT] = print_expr,
@@ -221,7 +221,7 @@ prep_reraise_star(PyThreadState* unused, PyObject *orig, PyObject *excs)
     return _PyExc_PrepReraiseStar(orig, excs);
 }
 
-instrinsic_func2
+const instrinsic_func2
 _PyIntrinsics_BinaryFunctions[] = {
     [INTRINSIC_PREP_RERAISE_STAR] = prep_reraise_star,
 };

diff --git a/Tools/c-analyzer/c_parser/parser/_common.py b/Tools/c-analyzer/c_parser/parser/_common.py
index d468d5442a939f0c2c1d7f70eb1ee24399c9d413..2eacace2c001df9b0755ae41e6b4291d3e4ae95d 100644 (file)
--- a/Tools/c-analyzer/c_parser/parser/_common.py
+++ b/Tools/c-analyzer/c_parser/parser/_common.py
@@ -7,13 +7,25 @@ from ._regexes import (
 )
 
 
-def log_match(group, m):
+def log_match(group, m, depth_before=None, depth_after=None):
     from . import _logger
-    text = m.group(0)
-    if text.startswith(('(', ')')) or text.endswith(('(', ')')):
-        _logger.debug(f'matched <{group}> ({text!r})')
+
+    if m is not None:
+        text = m.group(0)
+        if text.startswith(('(', ')')) or text.endswith(('(', ')')):
+            _logger.debug(f'matched <{group}> ({text!r})')
+        else:
+            _logger.debug(f'matched <{group}> ({text})')
+
+    elif depth_before is not None or depth_after is not None:
+        if depth_before is None:
+            depth_before = '???'
+        elif depth_after is None:
+            depth_after = '???'
+        _logger.log(1, f'depth: %s -> %s', depth_before, depth_after)
+
     else:
-        _logger.debug(f'matched <{group}> ({text})')
+        raise NotImplementedError('this should not have been hit')
 
 
 #############################

diff --git a/Tools/c-analyzer/c_parser/parser/_func_body.py b/Tools/c-analyzer/c_parser/parser/_func_body.py
index 42fd459e111d2ceb035a5be6424480ad0e79a171..25f2f5807ae8279cba963b020336c7c833c3f844 100644 (file)
--- a/Tools/c-analyzer/c_parser/parser/_func_body.py
+++ b/Tools/c-analyzer/c_parser/parser/_func_body.py
@@ -65,11 +65,11 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
          ) = m.groups()
 
         if empty:
-            log_match('', m)
+            log_match('', m, depth)
             resolve(None, None, None, text)
             yield None, text
         elif inline_kind:
-            log_match('', m)
+            log_match('', m, depth)
             kind = inline_kind
             name = inline_name or anon_name('inline-')
             data = []  # members
@@ -92,7 +92,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
             # XXX Should "parent" really be None for inline type decls?
             yield resolve(kind, data, name, text, None), text
         elif block_close:
-            log_match('', m)
+            log_match('', m, depth)
             depth -= 1
             resolve(None, None, None, text)
             # XXX This isn't great.  Calling resolve() should have
@@ -101,13 +101,13 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
             # needs to be fixed.
             yield None, text
         elif compound_bare:
-            log_match('', m)
+            log_match('', m, depth)
             yield resolve('statement', compound_bare, None, text, parent), text
         elif compound_labeled:
-            log_match('', m)
+            log_match('', m, depth)
             yield resolve('statement', compound_labeled, None, text, parent), text
         elif compound_paren:
-            log_match('', m)
+            log_match('', m, depth)
             try:
                 pos = match_paren(text)
             except ValueError:
@@ -132,7 +132,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
                     }
                 yield resolve('statement', data, None, text, parent), text
         elif block_open:
-            log_match('', m)
+            log_match('', m, depth)
             depth += 1
             if block_leading:
                 # An inline block: the last evaluated expression is used
@@ -144,10 +144,10 @@ def parse_function_body(name, text, resolve, source, anon_name, parent):
                 resolve(None, None, None, text)
                 yield None, text
         elif simple_ending:
-            log_match('', m)
+            log_match('', m, depth)
             yield resolve('statement', simple_stmt, None, text, parent), text
         elif var_ending:
-            log_match('', m)
+            log_match('', m, depth)
             kind = 'variable'
             _, name, vartype = parse_var_decl(decl)
             data = {
@@ -220,7 +220,7 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth):
     remainder = srcinfo.text[m.end():]
 
     if inline_kind:
-        log_match('func inline', m)
+        log_match('func inline', m, depth, depth)
         kind = inline_kind
         name = inline_name or anon_name('inline-')
         # Immediately emit a forward declaration.
@@ -249,7 +249,7 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth):
         yield parse_body, depth
 
     elif static_decl:
-        log_match('local variable', m)
+        log_match('local variable', m, depth, depth)
         _, name, data = parse_var_decl(static_decl)
 
         yield srcinfo.resolve('variable', data, name, parent=func), depth
@@ -266,10 +266,13 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth):
     else:
         log_match('func other', m)
         if block_open:
+            log_match('func other', None, depth, depth + 1)
             depth += 1
         elif block_close:
+            log_match('func other', None, depth, depth - 1)
             depth -= 1
         elif stmt_end:
+            log_match('func other', None, depth, depth)
             pass
         else:
             # This should be unreachable.

diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py
index 770802253792d8fda4fca1d87dabcd74589dd117..7ef1a8afc3b135eec9e82d2383d9de7ad2c62492 100644 (file)
--- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py
+++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py
@@ -29,7 +29,7 @@ COMPILER_DIRECTIVE_RE = re.compile(r'''
             [^()]*
          )*
      )  # <args>
-    ( [)] [)] )?  # <closed>
+    ( [)] [)] )  # <closed>
 ''', re.VERBOSE)
 
 POST_ARGS = (
@@ -156,6 +156,7 @@ def _iter_top_include_lines(lines, topfile, cwd,
                 if name != 'pragma':
                     raise Exception(line)
             else:
+                line = re.sub(r'__inline__', 'inline', line)
                 if not raw:
                     line, partial = _strip_directives(line, partial=partial)
                 yield _common.SourceLine(

diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py
index ab1d6257f1b1a9442c6ccd88e9e8a223da284b42..e7764165d36c4c0be82d934c3e0075338256a8fd 100644 (file)
--- a/Tools/c-analyzer/cpython/_parser.py
+++ b/Tools/c-analyzer/cpython/_parser.py
@@ -105,9 +105,14 @@ glob       dirname
 *      ./Include
 *      ./Include/internal
 
+Modules/_decimal/**/*.c        Modules/_decimal/libmpdec
+Modules/_hacl/*.c      Modules/_hacl/include
+Modules/_hacl/*.h      Modules/_hacl/include
 Modules/_tkinter.c     /usr/include/tcl8.6
+Modules/md5module.c    Modules/_hacl/include
+Modules/sha1module.c   Modules/_hacl/include
+Modules/sha2module.c   Modules/_hacl/include
 Modules/tkappinit.c    /usr/include/tcl
-Modules/_decimal/**/*.c        Modules/_decimal/libmpdec
 Objects/stringlib/*.h  Objects
 
 # @end=tsv@
@@ -173,6 +178,7 @@ Modules/_datetimemodule.c   Py_BUILD_CORE   1
 Modules/_functoolsmodule.c     Py_BUILD_CORE   1
 Modules/_heapqmodule.c Py_BUILD_CORE   1
 Modules/_io/*.c        Py_BUILD_CORE   1
+Modules/_io/*.h        Py_BUILD_CORE   1
 Modules/_localemodule.c        Py_BUILD_CORE   1
 Modules/_operator.c    Py_BUILD_CORE   1
 Modules/_posixsubprocess.c     Py_BUILD_CORE   1
@@ -296,6 +302,7 @@ MAX_SIZES = {
     # First match wins.
     _abs('Modules/_ctypes/ctypes.h'): (5_000, 500),
     _abs('Modules/_datetimemodule.c'): (20_000, 300),
+    _abs('Modules/_hacl/*.c'): (200_000, 500),
     _abs('Modules/posixmodule.c'): (20_000, 500),
     _abs('Modules/termios.c'): (10_000, 800),
     _abs('Modules/_testcapimodule.c'): (20_000, 400),

diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv
index 2e28c50c6ff69ae48835424d4a4d6d50e574cafd..57b8542fb46482d95bd291a7c584444389285e74 100644 (file)
--- a/Tools/c-analyzer/cpython/globals-to-fix.tsv
+++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv
@@ -539,7 +539,7 @@ Modules/_tkinter.c  -       command_mutex   -
 Modules/_tkinter.c     -       HeadFHCD        -
 Modules/_tkinter.c     -       stdin_ready     -
 Modules/_tkinter.c     -       event_tstate    -
-Modules/_xxsubinterpretersmodule.c     -       _globals        -
+Modules/_xxinterpchannelsmodule.c      -       _globals        -
 Modules/readline.c     -       completer_word_break_characters -
 Modules/readline.c     -       _history_length -
 Modules/readline.c     -       should_auto_add_history -

diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv
index 849e20a1b0f4ebec130ceffa2afd99f9d1018e2f..700ddf2851839e1099bc27289cf551a67ac825dc 100644 (file)
--- a/Tools/c-analyzer/cpython/ignored.tsv
+++ b/Tools/c-analyzer/cpython/ignored.tsv
@@ -206,6 +206,10 @@ Modules/_decimal/_decimal.c        -       invalid_signals_err     -
 Modules/_decimal/_decimal.c    -       signal_map      -
 Modules/_decimal/_decimal.c    -       ssize_constants -
 Modules/_elementtree.c -       ExpatMemoryHandler      -
+Modules/_hashopenssl.c -       py_hashes       -
+Modules/_hacl/Hacl_Hash_SHA1.c -       _h0     -
+Modules/_hacl/Hacl_Hash_MD5.c  -       _h0     -
+Modules/_hacl/Hacl_Hash_MD5.c  -       _t      -
 Modules/_io/_iomodule.c        -       static_types    -
 Modules/_io/textio.c   -       encodefuncs     -
 Modules/_io/winconsoleio.c     -       _PyWindowsConsoleIO_Type        -
@@ -218,9 +222,10 @@ Modules/_sre/sre_targets.h -       sre_targets     -
 Modules/_sre.c pattern_repr    flag_names      -
 Modules/_struct.c      -       bigendian_table -
 Modules/_struct.c      -       lilendian_table -
+Modules/_struct.c      -       native_table    -
 Modules/_tkinter.c     -       state_key       -
-Modules/_xxsubinterpretersmodule.c     -       _channelid_end_recv     -
-Modules/_xxsubinterpretersmodule.c     -       _channelid_end_send     -
+Modules/_xxinterpchannelsmodule.c      -       _channelid_end_recv     -
+Modules/_xxinterpchannelsmodule.c      -       _channelid_end_send     -
 Modules/_zoneinfo.c    -       DAYS_BEFORE_MONTH       -
 Modules/_zoneinfo.c    -       DAYS_IN_MONTH   -
 Modules/arraymodule.c  -       descriptors     -
@@ -332,12 +337,15 @@ Python/frozen.c   -       _PyImport_FrozenTest    -
 Python/getopt.c        -       longopts        -
 Python/import.c        -       _PyImport_Inittab       -
 Python/import.c        -       _PySys_ImplCacheTag     -
+Python/intrinsics.c    -       _PyIntrinsics_UnaryFunctions    -
+Python/intrinsics.c    -       _PyIntrinsics_BinaryFunctions   -
 Python/opcode_targets.h        -       opcode_targets  -
 Python/perf_trampoline.c       -       _Py_perfmap_callbacks   -
 Python/pyhash.c        -       PyHash_Func     -
 Python/pylifecycle.c   -       _C_LOCALE_WARNING       -
 Python/pylifecycle.c   -       _PyOS_mystrnicmp_hack   -
 Python/pylifecycle.c   -       _TARGET_LOCALES -
+Python/pylifecycle.c   -       INTERPRETER_TRAMPOLINE_CODEDEF  -
 Python/pystate.c       -       initial -
 Python/specialize.c    -       adaptive_opcodes        -
 Python/specialize.c    -       cache_requirements      -
@@ -392,8 +400,23 @@ Modules/_testbuffer.c      ndarray_memoryview_from_buffer  strides -
 Modules/_testbuffer.c  ndarray_memoryview_from_buffer  suboffsets      -
 Modules/_testbuffer.c  ndarray_push    kwlist  -
 Modules/_testbuffer.c  staticarray_init        kwlist  -
+Modules/_testcapi/code.c       get_code_extra_index    key     -
+Modules/_testcapi/datetime.c   -       test_run_counter        -
+Modules/_testcapi/exceptions.c -       PyRecursingInfinitelyError_Type -
 Modules/_testcapi/heaptype.c   -       _testcapimodule -
+Modules/_testcapi/mem.c        -       FmData  -
+Modules/_testcapi/mem.c        -       FmHook  -
+Modules/_testcapi/structmember.c       -       test_structmembersType_OldAPI   -
 Modules/_testcapi/unicode.c    -       _testcapimodule -
+Modules/_testcapi/watchers.c   -       g_dict_watch_events     -
+Modules/_testcapi/watchers.c   -       g_dict_watchers_installed       -
+Modules/_testcapi/watchers.c   -       g_type_modified_events  -
+Modules/_testcapi/watchers.c   -       g_type_watchers_installed       -
+Modules/_testcapi/watchers.c   -       num_code_object_created_events  -
+Modules/_testcapi/watchers.c   -       num_code_object_destroyed_events        -
+Modules/_testcapi/watchers.c   -       pyfunc_watchers -
+Modules/_testcapi/watchers.c   -       func_watcher_ids        -
+Modules/_testcapi/watchers.c   -       func_watcher_callbacks  -
 Modules/_testcapimodule.c      -       ContainerNoGC_members   -
 Modules/_testcapimodule.c      -       ContainerNoGC_type      -
 Modules/_testcapimodule.c      -       FmData  -
@@ -460,11 +483,13 @@ Modules/_testcapimodule.c -       meth_static_methods     -
 Modules/_testcapimodule.c      -       ml      -
 Modules/_testcapimodule.c      -       str1    -
 Modules/_testcapimodule.c      -       str2    -
+Modules/_testcapimodule.c      -       test_c_thread   -
 Modules/_testcapimodule.c      -       test_members    -
 Modules/_testcapimodule.c      -       test_run_counter        -
 Modules/_testcapimodule.c      -       test_structmembersType  -
 Modules/_testcapimodule.c      -       thread_done     -
 Modules/_testcapimodule.c      -       x       -
+Modules/_testcapimodule.c      -       wait_done       -
 Modules/_testcapimodule.c      getargs_keyword_only    keywords        -
 Modules/_testcapimodule.c      getargs_keywords        keywords        -
 Modules/_testcapimodule.c      getargs_positional_only_and_keywords    keywords        -
@@ -526,6 +551,7 @@ Modules/_testmultiphase.c   -       slots_exec_unreported_exception -
 Modules/_testmultiphase.c      -       slots_nonmodule_with_exec_slots -
 Modules/_testmultiphase.c      -       testexport_methods      -
 Modules/_testmultiphase.c      -       uninitialized_def       -
+Modules/_testsinglephase.c     -       global_state    -
 Modules/_xxtestfuzz/_xxtestfuzz.c      -       _fuzzmodule     -
 Modules/_xxtestfuzz/_xxtestfuzz.c      -       module_methods  -
 Modules/_xxtestfuzz/fuzzer.c   -       SRE_FLAG_DEBUG  -