[3.12] gh-109793: Allow Switching Interpreters During Finalization (gh-109794) (gh...

Essentially, we should check the thread ID rather than the thread state pointer.

diff --git a/Doc/data/python3.12.abi b/Doc/data/python3.12.abi
index ca82d5a3107d70a78e63609bd2b8b14657dcbbb8..6abf43ca6a458bea2451dc94a44a36cf4981b0dc 100644 (file)
--- a/Doc/data/python3.12.abi
+++ b/Doc/data/python3.12.abi
@@ -1707,7 +1707,7 @@
     <elf-symbol name='_PyNotImplemented_Type' size='416' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>\r
     <elf-symbol name='_PyOS_ReadlineTState' size='8' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>\r
     <elf-symbol name='_PyParser_TokenNames' size='552' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>\r
-    <elf-symbol name='_PyRuntime' size='459904' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>\r
+    <elf-symbol name='_PyRuntime' size='459920' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>\r
     <elf-symbol name='_PySet_Dummy' size='8' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>\r
     <elf-symbol name='_PyWeakref_CallableProxyType' size='416' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>\r
     <elf-symbol name='_PyWeakref_ProxyType' size='416' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>\r
@@ -2534,14 +2534,14 @@
       <parameter type-id='type-id-20'/>\r
       <return type-id='type-id-54'/>\r
     </function-decl>\r
-    <function-decl name='_PyInterpreterState_DeleteExceptMain' filepath='./Include/internal/pycore_pystate.h' line='147' column='1' visibility='default' binding='global' size-in-bits='64'>\r
+    <function-decl name='_PyInterpreterState_DeleteExceptMain' filepath='./Include/internal/pycore_pystate.h' line='151' column='1' visibility='default' binding='global' size-in-bits='64'>\r
       <parameter type-id='type-id-178'/>\r
       <return type-id='type-id-54'/>\r
     </function-decl>\r
-    <function-decl name='_PySignal_AfterFork' filepath='./Include/internal/pycore_pystate.h' line='148' column='1' visibility='default' binding='global' size-in-bits='64'>\r
+    <function-decl name='_PySignal_AfterFork' filepath='./Include/internal/pycore_pystate.h' line='152' column='1' visibility='default' binding='global' size-in-bits='64'>\r
       <return type-id='type-id-46'/>\r
     </function-decl>\r
-    <function-decl name='_PyRuntimeState_ReInitThreads' filepath='./Include/internal/pycore_runtime.h' line='198' column='1' visibility='default' binding='global' size-in-bits='64'>\r
+    <function-decl name='_PyRuntimeState_ReInitThreads' filepath='./Include/internal/pycore_runtime.h' line='201' column='1' visibility='default' binding='global' size-in-bits='64'>\r
       <parameter type-id='type-id-178'/>\r
       <return type-id='type-id-54'/>\r
     </function-decl>\r
@@ -7563,19 +7563,19 @@
   </abi-instr>\r
   <abi-instr address-size='64' path='Objects/interpreteridobject.c' comp-dir-path='/home/runner/work/cpython/cpython' language='LANG_C11'>\r
     <var-decl name='_PyInterpreterID_Type' type-id='type-id-256' mangled-name='_PyInterpreterID_Type' visibility='default' filepath='./Include/cpython/interpreteridobject.h' line='7' column='1' elf-symbol-id='_PyInterpreterID_Type'/>\r
-    <function-decl name='_PyInterpreterState_LookUpID' mangled-name='_PyInterpreterState_LookUpID' filepath='./Include/internal/pycore_interp.h' line='233' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyInterpreterState_LookUpID'>\r
+    <function-decl name='_PyInterpreterState_LookUpID' mangled-name='_PyInterpreterState_LookUpID' filepath='./Include/internal/pycore_interp.h' line='251' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyInterpreterState_LookUpID'>\r
       <parameter type-id='type-id-377'/>\r
       <return type-id='type-id-20'/>\r
     </function-decl>\r
-    <function-decl name='_PyInterpreterState_IDInitref' mangled-name='_PyInterpreterState_IDInitref' filepath='./Include/internal/pycore_interp.h' line='235' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyInterpreterState_IDInitref'>\r
+    <function-decl name='_PyInterpreterState_IDInitref' mangled-name='_PyInterpreterState_IDInitref' filepath='./Include/internal/pycore_interp.h' line='253' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyInterpreterState_IDInitref'>\r
       <parameter type-id='type-id-20'/>\r
       <return type-id='type-id-8'/>\r
     </function-decl>\r
-    <function-decl name='_PyInterpreterState_IDIncref' mangled-name='_PyInterpreterState_IDIncref' filepath='./Include/internal/pycore_interp.h' line='236' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyInterpreterState_IDIncref'>\r
+    <function-decl name='_PyInterpreterState_IDIncref' mangled-name='_PyInterpreterState_IDIncref' filepath='./Include/internal/pycore_interp.h' line='254' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyInterpreterState_IDIncref'>\r
       <parameter type-id='type-id-20'/>\r
       <return type-id='type-id-8'/>\r
     </function-decl>\r
-    <function-decl name='_PyInterpreterState_IDDecref' mangled-name='_PyInterpreterState_IDDecref' filepath='./Include/internal/pycore_interp.h' line='237' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyInterpreterState_IDDecref'>\r
+    <function-decl name='_PyInterpreterState_IDDecref' mangled-name='_PyInterpreterState_IDDecref' filepath='./Include/internal/pycore_interp.h' line='255' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyInterpreterState_IDDecref'>\r
       <parameter type-id='type-id-20'/>\r
       <return type-id='type-id-46'/>\r
     </function-decl>\r
@@ -16067,7 +16067,7 @@
         <var-decl name='max_str_digits' type-id='type-id-8' visibility='default' filepath='./Include/internal/pycore_interp.h' line='39' column='1'/>\r
       </data-member>\r
     </class-decl>\r
-    <class-decl name='_is' size-in-bits='3068160' is-struct='yes' visibility='default' filepath='./Include/internal/pycore_interp.h' line='49' column='1' id='type-id-935'>\r
+    <class-decl name='_is' size-in-bits='3068224' is-struct='yes' visibility='default' filepath='./Include/internal/pycore_interp.h' line='49' column='1' id='type-id-935'>\r
       <data-member access='public' layout-offset-in-bits='0'>\r
         <var-decl name='next' type-id='type-id-20' visibility='default' filepath='./Include/internal/pycore_interp.h' line='51' column='1'/>\r
       </data-member>\r
@@ -16276,7 +16276,10 @@
         <var-decl name='static_objects' type-id='type-id-875' visibility='default' filepath='./Include/internal/pycore_interp.h' line='195' column='1'/>\r
       </data-member>\r
       <data-member access='public' layout-offset-in-bits='3065856'>\r
-        <var-decl name='_initial_thread' type-id='type-id-945' visibility='default' filepath='./Include/internal/pycore_interp.h' line='198' column='1'/>\r
+        <var-decl name='_finalizing_id' type-id='type-id-819' visibility='default' filepath='./Include/internal/pycore_interp.h' line='199' column='1'/>\r
+      </data-member>\r
+      <data-member access='public' layout-offset-in-bits='3065920'>\r
+        <var-decl name='_initial_thread' type-id='type-id-945' visibility='default' filepath='./Include/internal/pycore_interp.h' line='202' column='1'/>\r
       </data-member>\r
     </class-decl>\r
     <class-decl name='pythreads' size-in-bits='256' is-struct='yes' visibility='default' filepath='./Include/internal/pycore_interp.h' line='67' column='1' id='type-id-936'>\r
@@ -16293,18 +16296,18 @@
         <var-decl name='stacksize' type-id='type-id-19' visibility='default' filepath='./Include/internal/pycore_interp.h' line='77' column='1'/>\r
       </data-member>\r
     </class-decl>\r
-    <class-decl name='_xidregitem' size-in-bits='256' is-struct='yes' visibility='default' filepath='./Include/internal/pycore_interp.h' line='226' column='1' id='type-id-946'>\r
+    <class-decl name='_xidregitem' size-in-bits='256' is-struct='yes' visibility='default' filepath='./Include/internal/pycore_interp.h' line='244' column='1' id='type-id-946'>\r
       <data-member access='public' layout-offset-in-bits='0'>\r
-        <var-decl name='prev' type-id='type-id-947' visibility='default' filepath='./Include/internal/pycore_interp.h' line='227' column='1'/>\r
+        <var-decl name='prev' type-id='type-id-947' visibility='default' filepath='./Include/internal/pycore_interp.h' line='245' column='1'/>\r
       </data-member>\r
       <data-member access='public' layout-offset-in-bits='64'>\r
-        <var-decl name='next' type-id='type-id-947' visibility='default' filepath='./Include/internal/pycore_interp.h' line='228' column='1'/>\r
+        <var-decl name='next' type-id='type-id-947' visibility='default' filepath='./Include/internal/pycore_interp.h' line='246' column='1'/>\r
       </data-member>\r
       <data-member access='public' layout-offset-in-bits='128'>\r
-        <var-decl name='cls' type-id='type-id-2' visibility='default' filepath='./Include/internal/pycore_interp.h' line='229' column='1'/>\r
+        <var-decl name='cls' type-id='type-id-2' visibility='default' filepath='./Include/internal/pycore_interp.h' line='247' column='1'/>\r
       </data-member>\r
       <data-member access='public' layout-offset-in-bits='192'>\r
-        <var-decl name='getdata' type-id='type-id-796' visibility='default' filepath='./Include/internal/pycore_interp.h' line='230' column='1'/>\r
+        <var-decl name='getdata' type-id='type-id-796' visibility='default' filepath='./Include/internal/pycore_interp.h' line='248' column='1'/>\r
       </data-member>\r
     </class-decl>\r
     <class-decl name='_Py_list_state' size-in-bits='5184' is-struct='yes' visibility='default' filepath='./Include/internal/pycore_list.h' line='31' column='1' id='type-id-943'>\r
@@ -16587,7 +16590,7 @@
       </data-member>\r
     </class-decl>\r
     <typedef-decl name='_Py_AuditHookEntry' type-id='type-id-983' filepath='./Include/internal/pycore_runtime.h' line='54' column='1' id='type-id-985'/>\r
-    <class-decl name='pyruntimestate' size-in-bits='3679232' is-struct='yes' visibility='default' filepath='./Include/internal/pycore_runtime.h' line='61' column='1' id='type-id-986'>\r
+    <class-decl name='pyruntimestate' size-in-bits='3679360' is-struct='yes' visibility='default' filepath='./Include/internal/pycore_runtime.h' line='61' column='1' id='type-id-986'>\r
       <data-member access='public' layout-offset-in-bits='0'>\r
         <var-decl name='_initialized' type-id='type-id-8' visibility='default' filepath='./Include/internal/pycore_runtime.h' line='66' column='1'/>\r
       </data-member>\r
@@ -16700,10 +16703,13 @@
         <var-decl name='cached_objects' type-id='type-id-869' visibility='default' filepath='./Include/internal/pycore_runtime.h' line='164' column='1'/>\r
       </data-member>\r
       <data-member access='public' layout-offset-in-bits='611008'>\r
-        <var-decl name='sys_path_0' type-id='type-id-52' visibility='default' filepath='./Include/internal/pycore_runtime.h' line='169' column='1'/>\r
+        <var-decl name='_finalizing_id' type-id='type-id-819' visibility='default' filepath='./Include/internal/pycore_runtime.h' line='168' column='1'/>\r
       </data-member>\r
       <data-member access='public' layout-offset-in-bits='611072'>\r
-        <var-decl name='_main_interpreter' type-id='type-id-995' visibility='default' filepath='./Include/internal/pycore_runtime.h' line='186' column='1'/>\r
+        <var-decl name='sys_path_0' type-id='type-id-52' visibility='default' filepath='./Include/internal/pycore_runtime.h' line='172' column='1'/>\r
+      </data-member>\r
+      <data-member access='public' layout-offset-in-bits='611136'>\r
+        <var-decl name='_main_interpreter' type-id='type-id-995' visibility='default' filepath='./Include/internal/pycore_runtime.h' line='189' column='1'/>\r
       </data-member>\r
     </class-decl>\r
     <class-decl name='pyinterpreters' size-in-bits='256' is-struct='yes' visibility='default' filepath='./Include/internal/pycore_runtime.h' line='87' column='1' id='type-id-987'>\r
@@ -17463,7 +17469,7 @@
     </function-decl>\r
     <var-decl name='_PyOS_ReadlineTState' type-id='type-id-177' mangled-name='_PyOS_ReadlineTState' visibility='default' filepath='./Include/cpython/pythonrun.h' line='120' column='1' elf-symbol-id='_PyOS_ReadlineTState'/>\r
     <var-decl name='PyOS_ReadlineFunctionPointer' type-id='type-id-1068' mangled-name='PyOS_ReadlineFunctionPointer' visibility='default' filepath='./Include/cpython/pythonrun.h' line='121' column='1' elf-symbol-id='PyOS_ReadlineFunctionPointer'/>\r
-    <function-decl name='_PyOS_InterruptOccurred' mangled-name='_PyOS_InterruptOccurred' filepath='./Include/internal/pycore_pystate.h' line='160' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyOS_InterruptOccurred'>\r
+    <function-decl name='_PyOS_InterruptOccurred' mangled-name='_PyOS_InterruptOccurred' filepath='./Include/internal/pycore_pystate.h' line='164' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyOS_InterruptOccurred'>\r
       <parameter type-id='type-id-177'/>\r
       <return type-id='type-id-8'/>\r
     </function-decl>\r
@@ -22067,11 +22073,11 @@
       <parameter type-id='type-id-177'/>\r
       <return type-id='type-id-46'/>\r
     </function-decl>\r
-    <function-decl name='_PyThreadState_MustExit' filepath='./Include/internal/pycore_pystate.h' line='75' column='1' visibility='default' binding='global' size-in-bits='64'>\r
+    <function-decl name='_PyThreadState_MustExit' filepath='./Include/internal/pycore_pystate.h' line='79' column='1' visibility='default' binding='global' size-in-bits='64'>\r
       <parameter type-id='type-id-177'/>\r
       <return type-id='type-id-8'/>\r
     </function-decl>\r
-    <function-decl name='_PyThreadState_DeleteExcept' mangled-name='_PyThreadState_DeleteExcept' filepath='./Include/internal/pycore_pystate.h' line='135' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyThreadState_DeleteExcept'>\r
+    <function-decl name='_PyThreadState_DeleteExcept' mangled-name='_PyThreadState_DeleteExcept' filepath='./Include/internal/pycore_pystate.h' line='139' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyThreadState_DeleteExcept'>\r
       <parameter type-id='type-id-177'/>\r
       <return type-id='type-id-46'/>\r
     </function-decl>\r
@@ -23504,7 +23510,7 @@
       <parameter type-id='type-id-12'/>\r
       <return type-id='type-id-8'/>\r
     </function-decl>\r
-    <function-decl name='_PyRuntime_Initialize' mangled-name='_PyRuntime_Initialize' filepath='./Include/internal/pycore_runtime.h' line='203' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyRuntime_Initialize'>\r
+    <function-decl name='_PyRuntime_Initialize' mangled-name='_PyRuntime_Initialize' filepath='./Include/internal/pycore_runtime.h' line='206' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyRuntime_Initialize'>\r
       <return type-id='type-id-54'/>\r
     </function-decl>\r
     <function-decl name='Py_FinalizeEx' mangled-name='Py_FinalizeEx' filepath='./Include/pylifecycle.h' line='16' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='Py_FinalizeEx'>\r
@@ -24753,7 +24759,7 @@
       </data-member>\r
     </class-decl>\r
     <typedef-decl name='_PyShimCodeDef' type-id='type-id-1486' filepath='./Include/internal/pycore_code.h' line='458' column='1' id='type-id-1487'/>\r
-    <typedef-decl name='_PyRuntimeState' type-id='type-id-986' filepath='./Include/internal/pycore_runtime.h' line='187' column='1' id='type-id-1488'/>\r
+    <typedef-decl name='_PyRuntimeState' type-id='type-id-986' filepath='./Include/internal/pycore_runtime.h' line='190' column='1' id='type-id-1488'/>\r
     <typedef-decl name='PyOS_sighandler_t' type-id='type-id-1020' filepath='./Include/pylifecycle.h' line='61' column='1' id='type-id-1489'/>\r
     <typedef-decl name='nl_item' type-id='type-id-8' filepath='/usr/include/nl_types.h' line='36' column='1' id='type-id-1490'/>\r
     <typedef-decl name='sigset_t' type-id='type-id-30' filepath='/usr/include/x86_64-linux-gnu/bits/types/sigset_t.h' line='7' column='1' id='type-id-73'/>\r
@@ -24945,7 +24951,7 @@
       <parameter type-id='type-id-937'/>\r
       <return type-id='type-id-54'/>\r
     </function-decl>\r
-    <function-decl name='_PyInterpreterState_Clear' filepath='./Include/internal/pycore_interp.h' line='204' column='1' visibility='default' binding='global' size-in-bits='64'>\r
+    <function-decl name='_PyInterpreterState_Clear' filepath='./Include/internal/pycore_interp.h' line='208' column='1' visibility='default' binding='global' size-in-bits='64'>\r
       <parameter type-id='type-id-177'/>\r
       <return type-id='type-id-46'/>\r
     </function-decl>\r
@@ -25134,24 +25140,24 @@
       <parameter type-id='type-id-19'/>\r
       <return type-id='type-id-46'/>\r
     </function-decl>\r
-    <function-decl name='_PyThreadState_New' mangled-name='_PyThreadState_New' filepath='./Include/internal/pycore_pystate.h' line='130' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyThreadState_New'>\r
+    <function-decl name='_PyThreadState_New' mangled-name='_PyThreadState_New' filepath='./Include/internal/pycore_pystate.h' line='134' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyThreadState_New'>\r
       <parameter type-id='type-id-20'/>\r
       <return type-id='type-id-177'/>\r
     </function-decl>\r
-    <function-decl name='_PyThreadState_Bind' mangled-name='_PyThreadState_Bind' filepath='./Include/internal/pycore_pystate.h' line='131' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyThreadState_Bind'>\r
+    <function-decl name='_PyThreadState_Bind' mangled-name='_PyThreadState_Bind' filepath='./Include/internal/pycore_pystate.h' line='135' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyThreadState_Bind'>\r
       <parameter type-id='type-id-177'/>\r
       <return type-id='type-id-46'/>\r
     </function-decl>\r
-    <function-decl name='_PyInterpreterState_Enable' mangled-name='_PyInterpreterState_Enable' filepath='./Include/internal/pycore_pystate.h' line='144' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyInterpreterState_Enable'>\r
+    <function-decl name='_PyInterpreterState_Enable' mangled-name='_PyInterpreterState_Enable' filepath='./Include/internal/pycore_pystate.h' line='148' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyInterpreterState_Enable'>\r
       <parameter type-id='type-id-178'/>\r
       <return type-id='type-id-54'/>\r
     </function-decl>\r
-    <var-decl name='_PyRuntime' type-id='type-id-1488' mangled-name='_PyRuntime' visibility='default' filepath='./Include/internal/pycore_runtime.h' line='192' column='1' elf-symbol-id='_PyRuntime'/>\r
-    <function-decl name='_PyRuntimeState_Init' mangled-name='_PyRuntimeState_Init' filepath='./Include/internal/pycore_runtime.h' line='194' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyRuntimeState_Init'>\r
+    <var-decl name='_PyRuntime' type-id='type-id-1488' mangled-name='_PyRuntime' visibility='default' filepath='./Include/internal/pycore_runtime.h' line='195' column='1' elf-symbol-id='_PyRuntime'/>\r
+    <function-decl name='_PyRuntimeState_Init' mangled-name='_PyRuntimeState_Init' filepath='./Include/internal/pycore_runtime.h' line='197' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyRuntimeState_Init'>\r
       <parameter type-id='type-id-178'/>\r
       <return type-id='type-id-54'/>\r
     </function-decl>\r
-    <function-decl name='_PyRuntimeState_Fini' mangled-name='_PyRuntimeState_Fini' filepath='./Include/internal/pycore_runtime.h' line='195' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyRuntimeState_Fini'>\r
+    <function-decl name='_PyRuntimeState_Fini' mangled-name='_PyRuntimeState_Fini' filepath='./Include/internal/pycore_runtime.h' line='198' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='_PyRuntimeState_Fini'>\r
       <parameter type-id='type-id-178'/>\r
       <return type-id='type-id-46'/>\r
     </function-decl>\r
@@ -25489,7 +25495,7 @@
       <parameter type-id='type-id-20'/>\r
       <return type-id='type-id-46'/>\r
     </function-decl>\r
-    <var-decl name='_Py_tss_tstate' type-id='type-id-177' visibility='default' filepath='./Include/internal/pycore_pystate.h' line='67' column='1'/>\r
+    <var-decl name='_Py_tss_tstate' type-id='type-id-177' visibility='default' filepath='./Include/internal/pycore_pystate.h' line='71' column='1'/>\r
     <function-decl name='PyThread_get_thread_native_id' mangled-name='PyThread_get_thread_native_id' filepath='./Include/pythread.h' line='27' column='1' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='PyThread_get_thread_native_id'>\r
       <return type-id='type-id-28'/>\r
     </function-decl>\r

diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
index 1db23145a539cba0e685c44e6fb38f3a7914a1b8..2fbb9f1aec6154b5c97649ec1b627f2bbce4360c 100644 (file)
--- a/Include/internal/pycore_interp.h
+++ b/Include/internal/pycore_interp.h
@@ -194,6 +194,10 @@ struct _is {
     struct _Py_interp_cached_objects cached_objects;
     struct _Py_interp_static_objects static_objects;
 
+    /* The ID of the OS thread in which we are finalizing.
+       We use _Py_atomic_address instead of adding a new _Py_atomic_ulong. */
+    _Py_atomic_address _finalizing_id;
+
    /* the initial PyInterpreterState.threads.head */
     PyThreadState _initial_thread;
 };
@@ -209,9 +213,23 @@ _PyInterpreterState_GetFinalizing(PyInterpreterState *interp) {
     return (PyThreadState*)_Py_atomic_load_relaxed(&interp->_finalizing);
 }
 
+static inline unsigned long
+_PyInterpreterState_GetFinalizingID(PyInterpreterState *interp) {
+    return (unsigned long)_Py_atomic_load_relaxed(&interp->_finalizing_id);
+}
+
 static inline void
 _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tstate) {
     _Py_atomic_store_relaxed(&interp->_finalizing, (uintptr_t)tstate);
+    if (tstate == NULL) {
+        _Py_atomic_store_relaxed(&interp->_finalizing_id, 0);
+    }
+    else {
+        // XXX Re-enable this assert once gh-109860 is fixed.
+        //assert(tstate->thread_id == PyThread_get_thread_ident());
+        _Py_atomic_store_relaxed(&interp->_finalizing_id,
+                                 (uintptr_t)tstate->thread_id);
+    }
 }
 
 

diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h
index 4209d090b77df29169be8876ea19438f94b868fd..e4186b3a9214283e027409023c9b7da013d5522b 100644 (file)
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -36,8 +36,12 @@ _Py_IsMainInterpreter(PyInterpreterState *interp)
 static inline int
 _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp)
 {
-    return (_PyRuntimeState_GetFinalizing(interp->runtime) != NULL &&
-            interp == &interp->runtime->_main_interpreter);
+    /* bpo-39877: Access _PyRuntime directly rather than using
+       tstate->interp->runtime to support calls from Python daemon threads.
+       After Py_Finalize() has been called, tstate can be a dangling pointer:
+       point to PyThreadState freed memory. */
+    return (_PyRuntimeState_GetFinalizing(&_PyRuntime) != NULL &&
+            interp == &_PyRuntime._main_interpreter);
 }
 
 

diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h
index 9b5e12356018fd5690c7c780b6c9e40eb78f6d65..bc972783fc94f184f65ecaabcd2222d897d1d9af 100644 (file)
--- a/Include/internal/pycore_runtime.h
+++ b/Include/internal/pycore_runtime.h
@@ -163,6 +163,9 @@ typedef struct pyruntimestate {
     struct _Py_static_objects static_objects;
     struct _Py_cached_objects cached_objects;
 
+    /* The ID of the OS thread in which we are finalizing.
+       We use _Py_atomic_address instead of adding a new _Py_atomic_ulong. */
+    _Py_atomic_address _finalizing_id;
     /* The value to use for sys.path[0] in new subinterpreters.
        Normally this would be part of the PyConfig struct.  However,
        we cannot add it there in 3.12 since that's an ABI change. */
@@ -210,9 +213,23 @@ _PyRuntimeState_GetFinalizing(_PyRuntimeState *runtime) {
     return (PyThreadState*)_Py_atomic_load_relaxed(&runtime->_finalizing);
 }
 
+static inline unsigned long
+_PyRuntimeState_GetFinalizingID(_PyRuntimeState *runtime) {
+    return (unsigned long)_Py_atomic_load_relaxed(&runtime->_finalizing_id);
+}
+
 static inline void
 _PyRuntimeState_SetFinalizing(_PyRuntimeState *runtime, PyThreadState *tstate) {
     _Py_atomic_store_relaxed(&runtime->_finalizing, (uintptr_t)tstate);
+    if (tstate == NULL) {
+        _Py_atomic_store_relaxed(&runtime->_finalizing_id, 0);
+    }
+    else {
+        // XXX Re-enable this assert once gh-109860 is fixed.
+        //assert(tstate->thread_id == PyThread_get_thread_ident());
+        _Py_atomic_store_relaxed(&runtime->_finalizing_id,
+                                 (uintptr_t)tstate->thread_id);
+    }
 }
 
 #ifdef __cplusplus

diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py
index b426fa12f3e70e7079d38e9959ca96354b6f6942..38cf4b6fe737dda41265d9622e9519f0eba9a05d 100644 (file)
--- a/Lib/test/test_interpreters.py
+++ b/Lib/test/test_interpreters.py
@@ -639,6 +639,26 @@ class StartupTests(TestBase):
                 self.assertEqual(sp0_sub, expected)
 
 
+class FinalizationTests(TestBase):
+
+    def test_gh_109793(self):
+        import subprocess
+        argv = [sys.executable, '-c', '''if True:
+            import _xxsubinterpreters as _interpreters
+            interpid = _interpreters.create()
+            raise Exception
+            ''']
+        proc = subprocess.run(argv, capture_output=True, text=True)
+        self.assertIn('Traceback', proc.stderr)
+        if proc.returncode == 0 and support.verbose:
+            print()
+            print("--- cmd unexpected succeeded ---")
+            print(f"stdout:\n{proc.stdout}")
+            print(f"stderr:\n{proc.stderr}")
+            print("------")
+        self.assertEqual(proc.returncode, 1)
+
+
 class TestIsShareable(TestBase):
 
     def test_default_shareables(self):

diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-25-09-24-10.gh-issue-109793.zFQBkv.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-25-09-24-10.gh-issue-109793.zFQBkv.rst
new file mode 100644 (file)
index 0000000..d2dc4c8
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-25-09-24-10.gh-issue-109793.zFQBkv.rst
@@ -0,0 +1,4 @@
+The main thread no longer exits prematurely when a subinterpreter
+is cleaned up during runtime finalization.  The bug was a problem
+particularly because, when triggered, the Python process would
+always return with a 0 exitcode, even if it failed.

diff --git a/Python/pystate.c b/Python/pystate.c
index e789eb9ce39086fac934ec34ea3b6b219ee90a2c..0430454432be7bfa0292580952aea7e493bea6c2 100644 (file)
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -2916,11 +2916,26 @@ _PyThreadState_MustExit(PyThreadState *tstate)
        tstate->interp->runtime to support calls from Python daemon threads.
        After Py_Finalize() has been called, tstate can be a dangling pointer:
        point to PyThreadState freed memory. */
+    unsigned long finalizing_id = _PyRuntimeState_GetFinalizingID(&_PyRuntime);
     PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(&_PyRuntime);
     if (finalizing == NULL) {
+        // XXX This isn't completely safe from daemon thraeds,
+        // since tstate might be a dangling pointer.
         finalizing = _PyInterpreterState_GetFinalizing(tstate->interp);
+        finalizing_id = _PyInterpreterState_GetFinalizingID(tstate->interp);
     }
-    return (finalizing != NULL && finalizing != tstate);
+    // XXX else check &_PyRuntime._main_interpreter._initial_thread
+    if (finalizing == NULL) {
+        return 0;
+    }
+    else if (finalizing == tstate) {
+        return 0;
+    }
+    else if (finalizing_id == PyThread_get_thread_ident()) {
+        /* gh-109793: we must have switched interpreters. */
+        return 0;
+    }
+    return 1;
 }