]> git.ipfire.org Git - thirdparty/Python/cpython.git/commitdiff
gh-133228: c-analyzer clang preprocessor (GH-133229)
authordgpb <3577712+dg-pb@users.noreply.github.com>
Thu, 27 Nov 2025 22:22:21 +0000 (00:22 +0200)
committerGitHub <noreply@github.com>
Thu, 27 Nov 2025 22:22:21 +0000 (22:22 +0000)
* impl
* included 2 failures to tsvs next to similar entries
* added fix/hack for curses.h fails
* fix leftover from debug

Tools/c-analyzer/c_parser/preprocessor/__init__.py
Tools/c-analyzer/c_parser/preprocessor/clang.py [new file with mode: 0644]
Tools/c-analyzer/c_parser/preprocessor/gcc.py
Tools/c-analyzer/cpython/_parser.py
Tools/c-analyzer/cpython/globals-to-fix.tsv
Tools/c-analyzer/cpython/ignored.tsv

index 30a86cbd7dc494e32ac141a591b85086730102af..f8d2f805cb1b192b13ab34dda203d91e361b167f 100644 (file)
@@ -16,6 +16,7 @@ from . import errors as _errors
 from . import (
     pure as _pure,
     gcc as _gcc,
+    clang as _clang,
 )
 
 
@@ -234,7 +235,7 @@ _COMPILERS = {
     'bcpp': None,
     # aliases/extras:
     'gcc': _gcc.preprocess,
-    'clang': None,
+    'clang': _clang.preprocess,
 }
 
 
diff --git a/Tools/c-analyzer/c_parser/preprocessor/clang.py b/Tools/c-analyzer/c_parser/preprocessor/clang.py
new file mode 100644 (file)
index 0000000..574a23f
--- /dev/null
@@ -0,0 +1,104 @@
+import os.path
+import re, sys
+
+from . import common as _common
+from . import gcc as _gcc
+
+_normpath = _gcc._normpath
+
+TOOL = 'clang'
+
+META_FILES = {
+    '<built-in>',
+    '<command line>',
+}
+
+
+def preprocess(filename,
+               incldirs=None,
+               includes=None,
+               macros=None,
+               samefiles=None,
+               cwd=None,
+               ):
+    if not cwd or not os.path.isabs(cwd):
+        cwd = os.path.abspath(cwd or '.')
+    filename = _normpath(filename, cwd)
+
+    postargs = _gcc.POST_ARGS
+    basename = os.path.basename(filename)
+    dirname = os.path.basename(os.path.dirname(filename))
+    if (basename not in _gcc.FILES_WITHOUT_INTERNAL_CAPI
+       and dirname not in _gcc.DIRS_WITHOUT_INTERNAL_CAPI):
+        postargs += ('-DPy_BUILD_CORE=1',)
+
+    text = _common.preprocess(
+        TOOL,
+        filename,
+        incldirs=incldirs,
+        includes=includes,
+        macros=macros,
+        #preargs=PRE_ARGS,
+        postargs=postargs,
+        executable=['clang'],
+        compiler='unix',
+        cwd=cwd,
+    )
+    return _iter_lines(text, filename, samefiles, cwd)
+
+
+EXIT_MARKERS = {'# 2 "<built-in>" 2', '# 3 "<built-in>" 2', '# 4 "<built-in>" 2'}
+
+
+def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
+    lines = iter(text.splitlines())
+
+    # The first line is special.
+    # The subsequent lines are consistent.
+    firstlines = [
+        f'# 1 "{reqfile}"',
+        '# 1 "<built-in>" 1',
+        '# 1 "<built-in>" 3',
+        '# 370 "<built-in>" 3',
+        '# 1 "<command line>" 1',
+        '# 1 "<built-in>" 2',
+    ]
+    for expected in firstlines:
+        line = next(lines)
+        if line != expected:
+            raise NotImplementedError((line, expected))
+
+    # Do all the CLI-provided includes.
+    filter_reqfile = (lambda f: _gcc._filter_reqfile(f, reqfile, samefiles))
+    make_info = (lambda lno: _common.FileInfo(reqfile, lno))
+    last = None
+    for line in lines:
+        assert last != reqfile, (last,)
+        # NOTE:condition is clang specific
+        if not line:
+            continue
+        lno, included, flags = _gcc._parse_marker_line(line, reqfile)
+        if not included:
+            raise NotImplementedError((line,))
+        if included == reqfile:
+            # This will be the last one.
+            assert 2 in flags, (line, flags)
+        else:
+            # NOTE:first condition is specific to clang
+            if _normpath(included, cwd) == reqfile:
+                assert 1 in flags or 2 in flags, (line, flags, included, reqfile)
+            else:
+                assert 1 in flags, (line, flags, included, reqfile)
+        yield from _gcc._iter_top_include_lines(
+            lines,
+            _normpath(included, cwd),
+            cwd,
+            filter_reqfile,
+            make_info,
+            raw,
+            EXIT_MARKERS
+        )
+        last = included
+    # The last one is always the requested file.
+    # NOTE:_normpath is clang specific
+    assert _normpath(included, cwd) == reqfile, (line,)
index d20cd19f6e6d5e84947c07735a7a5d557e3598ac..4a55a1a24ee1bedd6af82cec7c05ec04eb10e26f 100644 (file)
@@ -65,6 +65,8 @@ POST_ARGS = (
     '-E',
 )
 
+EXIT_MARKERS = {'# 0 "<command-line>" 2', '# 1 "<command-line>" 2'}
+
 
 def preprocess(filename,
                incldirs=None,
@@ -138,6 +140,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
             filter_reqfile,
             make_info,
             raw,
+            EXIT_MARKERS
         )
         last = included
     # The last one is always the requested file.
@@ -146,7 +149,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
 
 def _iter_top_include_lines(lines, topfile, cwd,
                             filter_reqfile, make_info,
-                            raw):
+                            raw, exit_markers):
     partial = 0  # depth
     files = [topfile]
     # We start at 1 in case there are source lines (including blank ones)
@@ -154,12 +157,20 @@ def _iter_top_include_lines(lines, topfile, cwd,
     # _parse_marker_line() that the preprocessor reported lno as 1.
     lno = 1
     for line in lines:
-        if line == '# 0 "<command-line>" 2' or line == '# 1 "<command-line>" 2':
+        if line in exit_markers:
             # We're done with this top-level include.
             return
 
         _lno, included, flags = _parse_marker_line(line)
         if included:
+            # HACK:
+            # Mixes curses.h and ncurses.h marker lines
+            # gcc silently passes this, while clang fails
+            # See: /Include/py_curses.h #if-elif directives
+            # And compare with preprocessor output
+            if os.path.basename(included) == 'curses.h':
+                included = os.path.join(os.path.dirname(included), 'ncurses.h')
+
             lno = _lno
             included = _normpath(included, cwd)
             # We hit a marker line.
index fd198d7d06c96fe8ae7516d8d9627107aa67e39b..d348a99fff7a110ef68902ee119dc1982f53f87d 100644 (file)
@@ -340,6 +340,10 @@ MAX_SIZES = {
 
     # Catch-alls:
     _abs('Include/**/*.h'): (5_000, 500),
+
+    # Specific to clang
+    _abs('Modules/selectmodule.c'): (40_000, 3000),
+    _abs('Modules/_testcapi/pyatomic.c'): (30_000, 1000),
 }
 
 
index 3c3cb2f9c86f16beba37ede9a1a06e986a6a2e11..301784f773d31f3f89734ebc9d48c5627c04265c 100644 (file)
@@ -400,6 +400,7 @@ Modules/_tkinter.c  -       tcl_lock        -
 Modules/_tkinter.c     -       excInCmd        -
 Modules/_tkinter.c     -       valInCmd        -
 Modules/_tkinter.c     -       trbInCmd        -
+Modules/socketmodule.c         -       netdb_lock      -
 
 
 ##################################
index bd4a8cf0d3e65cbfe4b7a94bd77f896a9c1067b0..adb183000deeffc2b893b0c589efa02799d9db24 100644 (file)
@@ -16,6 +16,7 @@ filename      funcname        name    reason
 ## indicators for resource availability/capability
 # (set during first init)
 Python/bootstrap_hash.c        py_getrandom    getrandom_works -
+Python/bootstrap_hash.c        py_getentropy   getentropy_works        -
 Python/fileutils.c     -       _Py_open_cloexec_works  -
 Python/fileutils.c     set_inheritable ioctl_works     -
 # (set lazily, *after* first init)