[3.14] gh-143750: Compile OpenSSL with TSan for TSan CI (#153316) (#153355)

Compile OpenSSL with TSan in the TSan CI job so that data races between
Python code and OpenSSL internals are visible to the sanitizer.

Also skip test_ssl_in_multiple_threads under TSan: concurrent calls to
SSLContext.load_cert_chain on the same context race on the SSL_CTX
default password callback. The race is fixed in 3.15+ (GH-143818), but
with TSan-instrumented OpenSSL it is now reported on 3.14.

(cherry picked from commit fc19ad7b8b08c43667c9087d89f32f08830544ce)

Co-authored-by: Sam Gross <colesbury@gmail.com>

diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml
index afe0100c085e5d223d0c9d22633c4dfd79aef431..317763c65f3c2532784884c6087eb56ac7b050ae 100644 (file)
--- a/.github/workflows/reusable-san.yml
+++ b/.github/workflows/reusable-san.yml
@@ -17,6 +17,7 @@ permissions:
 
 env:
   FORCE_COLOR: 1
+  OPENSSL_VER: 3.5.7
 
 jobs:
   build-san-reusable:
@@ -45,28 +46,53 @@ jobs:
         sudo update-alternatives --set clang /usr/bin/clang-20
         sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-20 100
         sudo update-alternatives --set clang++ /usr/bin/clang++-20
-
-        if [ "${SANITIZER}" = "TSan" ]; then
-          # Reduce ASLR to avoid TSan crashing
-          sudo sysctl -w vm.mmap_rnd_bits=28
-        fi
-
-    - name: Sanitizer option setup
-      run: |
-        if [ "${SANITIZER}" = "TSan" ]; then
-          echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{
-              fromJSON(inputs.free-threading)
-              && '_free_threading'
-              || ''
-            }}.txt handle_segv=0" >> "$GITHUB_ENV"
-        else
-          echo "UBSAN_OPTIONS=${SAN_LOG_OPTION}" >> "$GITHUB_ENV"
-        fi
         echo "CC=clang" >> "$GITHUB_ENV"
         echo "CXX=clang++" >> "$GITHUB_ENV"
+    - name: TSan option setup
+      if: inputs.sanitizer == 'TSan'
+      run: |
+        sudo sysctl -w vm.mmap_rnd_bits=28  # Reduce ASLR to avoid TSan crashing
+
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV"
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV"
+        echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${SUPPRESSIONS_SUFFIX}.txt handle_segv=0" >> "$GITHUB_ENV"
       env:
-        SANITIZER: ${{ inputs.sanitizer }}
         SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log
+        SUPPRESSIONS_SUFFIX: >-
+          ${{
+            fromJSON(inputs.free-threading)
+            && '_free_threading'
+            || ''
+          }}
+    - name: UBSan option setup
+      if: inputs.sanitizer != 'TSan'
+      run: >-
+        echo
+        "UBSAN_OPTIONS=${SAN_LOG_OPTION}"
+        >> "$GITHUB_ENV"
+      env:
+        SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
+    - name: 'Restore OpenSSL build (TSan)'
+      id: cache-openssl
+      if: inputs.sanitizer == 'TSan'
+      uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+      with:
+        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+        key: ${{ env.IMAGE_OS_VERSION }}-multissl-openssl-tsan-${{ env.OPENSSL_VER }}
+    - name: Install OpenSSL (TSan)
+      if: >-
+        inputs.sanitizer == 'TSan'
+        && steps.cache-openssl.outputs.cache-hit != 'true'
+      run: >-
+        python3 Tools/ssl/multissltests.py
+        --steps=library
+        --base-directory="${MULTISSL_DIR}"
+        --openssl="${OPENSSL_VER}"
+        --system=Linux
+        --tsan
     - name: Configure CPython
       run: >-
         ./configure
@@ -77,6 +103,7 @@ jobs:
           || '--with-undefined-behavior-sanitizer'
         }}
         --with-pydebug
+        ${{ inputs.sanitizer == 'TSan' && '--with-openssl="$OPENSSL_DIR" --with-openssl-rpath=auto' || '' }}
         ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
     - name: Build CPython
       run: make -j4

diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index ef45d5be69301877b0f3c342108544abe1120b3e..91bfb408bb9965d50a6b48d166387a757dd66a77 100644 (file)
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -1311,6 +1311,10 @@ class ContextTests(unittest.TestCase):
         # Make sure the password function isn't called if it isn't needed
         ctx.load_cert_chain(CERTFILE, password=getpass_exception)
 
+    @support.skip_if_sanitizer("gh-143756: data race in "
+                               "SSLContext.load_cert_chain when called "
+                               "concurrently on the same context",
+                               thread=True)
     @threading_helper.requires_working_threading()
     def test_load_cert_chain_thread_safety(self):
         # gh-134698: _ssl detaches the thread state (and as such,
@@ -3059,6 +3063,10 @@ class ThreadedTests(unittest.TestCase):
                 'Cannot create a client socket with a PROTOCOL_TLS_SERVER context',
                 str(e.exception))
 
+    @support.skip_if_sanitizer("gh-143756: data race in "
+                               "SSLContext.load_cert_chain when called "
+                               "concurrently on the same context",
+                               thread=True)
     @unittest.skipUnless(support.Py_GIL_DISABLED, "test is only useful if the GIL is disabled")
     def test_ssl_in_multiple_threads(self):
         # See GH-124984: OpenSSL is not thread safe.

diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py
index 68bd6e0860deb6f2b74e9352a5a88fdabacc89ff..cb349236a034c400dfb0ecea1bcb9601d2728ce6 100755 (executable)
--- a/Tools/ssl/multissltests.py
+++ b/Tools/ssl/multissltests.py
@@ -147,6 +147,12 @@ parser.add_argument(
     dest='keep_sources',
     help="Keep original sources for debugging."
 )
+parser.add_argument(
+    '--tsan',
+    action='store_true',
+    dest='tsan',
+    help="Build with thread sanitizer. (Disables fips in OpenSSL 3.x)."
+)
 
 
 class AbstractBuilder(object):
@@ -301,6 +307,8 @@ class AbstractBuilder(object):
         """Now build openssl"""
         log.info("Running build in {}".format(self.build_dir))
         cwd = self.build_dir
+        if self.args.tsan:
+            config_args += ("-fsanitize=thread",)
         cmd = [
             "./config", *config_args,
             "shared", "--debug",