gh-115168: Add pystats counter for invalidated executors (GH-115169)

diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h
index fe54d1ddfe61293d50dd387e0a87f272ca22499a..8fc9fb62aebdb415ea4fd1bd77dfbd0ece2f1b83 100644 (file)
--- a/Include/cpython/optimizer.h
+++ b/Include/cpython/optimizer.h
@@ -100,8 +100,8 @@ void _Py_ExecutorClear(_PyExecutorObject *);
 void _Py_BloomFilter_Init(_PyBloomFilter *);
 void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj);
 PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj);
-PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj);
-extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp);
+PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation);
+extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation);
 
 /* For testing */
 PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void);

diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h
index db9aaedec950e45c7f967b7187c6366b6518ed64..887fbbedf88502b4ccbfe8515637e825f4fcc507 100644 (file)
--- a/Include/cpython/pystats.h
+++ b/Include/cpython/pystats.h
@@ -115,6 +115,7 @@ typedef struct _optimization_stats {
     uint64_t inner_loop;
     uint64_t recursive_call;
     uint64_t low_confidence;
+    uint64_t executors_invalidated;
     UOpStats opcode[512];
     uint64_t unsupported_opcode[256];
     uint64_t trace_length_hist[_Py_UOP_HIST_SIZE];

diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index 0d23b1899f22e44246041c76d6cb1d7d12ce9b5b..5b714ca3f3dc33e6416a0a8f42fe6aa4b8cbe0f1 100644 (file)
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -1035,7 +1035,7 @@ static PyObject *
 invalidate_executors(PyObject *self, PyObject *obj)
 {
     PyInterpreterState *interp = PyInterpreterState_Get();
-    _Py_Executors_InvalidateDependency(interp, obj);
+    _Py_Executors_InvalidateDependency(interp, obj, 1);
     Py_RETURN_NONE;
 }
 

diff --git a/Python/instrumentation.c b/Python/instrumentation.c
index 878d19f0552bf5982503dc74b151d7f7f6db54d8..6f1bc2e0a107df53de3a2f39802a3954898bf1e6 100644 (file)
--- a/Python/instrumentation.c
+++ b/Python/instrumentation.c
@@ -1599,7 +1599,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp)
     if (code->co_executors != NULL) {
         _PyCode_Clear_Executors(code);
     }
-    _Py_Executors_InvalidateDependency(interp, code);
+    _Py_Executors_InvalidateDependency(interp, code, 1);
     int code_len = (int)Py_SIZE(code);
     /* Exit early to avoid creating instrumentation
      * data for potential statically allocated code
@@ -1820,7 +1820,7 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events)
         return -1;
     }
     set_global_version(tstate, new_version);
-    _Py_Executors_InvalidateAll(interp);
+    _Py_Executors_InvalidateAll(interp, 1);
     return instrument_all_executing_code_objects(interp);
 }
 
@@ -1850,7 +1850,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent
         /* Force instrumentation update */
         code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT;
     }
-    _Py_Executors_InvalidateDependency(interp, code);
+    _Py_Executors_InvalidateDependency(interp, code, 1);
     if (_Py_Instrument(code, interp)) {
         return -1;
     }

diff --git a/Python/optimizer.c b/Python/optimizer.c
index 6b2ba3addefc95d799fd009455b1c827f920aea1..c04ee17ee2171d6911673e9aadd3eeceacd4a4e8 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -1348,7 +1348,7 @@ _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj)
  * May cause other executors to be invalidated as well
  */
 void
-_Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj)
+_Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation)
 {
     _PyBloomFilter obj_filter;
     _Py_BloomFilter_Init(&obj_filter);
@@ -1360,6 +1360,9 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj)
         _PyExecutorObject *next = exec->vm_data.links.next;
         if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) {
             _Py_ExecutorClear(exec);
+            if (is_invalidation) {
+                OPT_STAT_INC(executors_invalidated);
+            }
         }
         exec = next;
     }
@@ -1367,7 +1370,7 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj)
 
 /* Invalidate all executors */
 void
-_Py_Executors_InvalidateAll(PyInterpreterState *interp)
+_Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation)
 {
     while (interp->executor_list_head) {
         _PyExecutorObject *executor = interp->executor_list_head;
@@ -1378,5 +1381,8 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp)
         else {
             _Py_ExecutorClear(executor);
         }
+        if (is_invalidation) {
+            OPT_STAT_INC(executors_invalidated);
+        }
     }
 }

diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c
index 1a9e82903ffbc00451ae31bc130480f2c29383d5..e9751291e2fd5aeb489320511cbd1d3ed9bbf304 100644 (file)
--- a/Python/optimizer_analysis.c
+++ b/Python/optimizer_analysis.c
@@ -405,7 +405,7 @@ globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict,
 {
     RARE_EVENT_STAT_INC(watched_globals_modification);
     assert(get_mutations(dict) < _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS);
-    _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict);
+    _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), dict, 1);
     increment_mutations(dict);
     PyDict_Unwatch(GLOBALS_WATCHER_ID, dict);
     return 0;

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 04487345f7ec053d0186aad9b09a437001d937d7..3a2c0a450ac9d97d371e0f7103c8f8a554346d39 100644 (file)
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -612,7 +612,7 @@ builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, Py
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
     if (interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) {
-        _Py_Executors_InvalidateAll(interp);
+        _Py_Executors_InvalidateAll(interp, 1);
     }
     RARE_EVENT_INTERP_INC(interp, builtin_dict);
     return 0;
@@ -1628,7 +1628,7 @@ finalize_modules(PyThreadState *tstate)
     PyInterpreterState *interp = tstate->interp;
 
     // Invalidate all executors and turn off tier 2 optimizer
-    _Py_Executors_InvalidateAll(interp);
+    _Py_Executors_InvalidateAll(interp, 0);
     _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL);
     Py_XDECREF(old);
 

diff --git a/Python/pystate.c b/Python/pystate.c
index bb8e24c1dbe12fe1d0e9a1f0973e264ee17c67ac..a80c1b7fb9c8665bfb57e87c855a8b79f3451ad4 100644 (file)
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -2666,7 +2666,7 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp,
         return;
     }
     if (eval_frame != NULL) {
-        _Py_Executors_InvalidateAll(interp);
+        _Py_Executors_InvalidateAll(interp, 1);
     }
     RARE_EVENT_INC(set_eval_frame_func);
     interp->eval_frame = eval_frame;

diff --git a/Python/specialize.c b/Python/specialize.c
index 871979d92298b6ec416c2b912cdd034528c9de15..f83d8a9ceb0282dc26c2ff072fab5efc308587cf 100644 (file)
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -236,6 +236,7 @@ print_optimization_stats(FILE *out, OptimizationStats *stats)
     fprintf(out, "Optimization inner loop: %" PRIu64 "\n", stats->inner_loop);
     fprintf(out, "Optimization recursive call: %" PRIu64 "\n", stats->recursive_call);
     fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence);
+    fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated);
 
     print_histogram(out, "Trace length", stats->trace_length_hist);
     print_histogram(out, "Trace run length", stats->trace_run_length_hist);

diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 69b6d886ccc3e90a958c73210f54a273334466b8..1bfd031fdd26b21877f3b12e628dee8d983866a7 100644 (file)
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -2138,7 +2138,7 @@ sys__clear_internal_caches_impl(PyObject *module)
 /*[clinic end generated code: output=0ee128670a4966d6 input=253e741ca744f6e8]*/
 {
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    _Py_Executors_InvalidateAll(interp);
+    _Py_Executors_InvalidateAll(interp, 0);
     PyType_ClearCache();
     Py_RETURN_NONE;
 }

diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py
index 6b60b59b3b0e797a1bc89a53b6d9fe923d5c81b6..2925e096f4d95ef91b188eeae1fe99881f1183d5 100644 (file)
--- a/Tools/scripts/summarize_stats.py
+++ b/Tools/scripts/summarize_stats.py
@@ -451,6 +451,7 @@ class Stats:
         inner_loop = self._data["Optimization inner loop"]
         recursive_call = self._data["Optimization recursive call"]
         low_confidence = self._data["Optimization low confidence"]
+        executors_invalidated = self._data["Executors invalidated"]
 
         return {
             Doc(
@@ -493,11 +494,19 @@ class Stats:
                 "A trace is abandoned because the likelihood of the jump to top being taken "
                 "is too low.",
             ): (low_confidence, attempts),
+            Doc(
+                "Executors invalidated",
+                "The number of executors that were invalidated due to watched "
+                "dictionary changes.",
+            ): (executors_invalidated, created),
             Doc("Traces executed", "The number of traces that were executed"): (
                 executed,
                 None,
             ),
-            Doc("Uops executed", "The total number of uops (micro-operations) that were executed"): (
+            Doc(
+                "Uops executed",
+                "The total number of uops (micro-operations) that were executed",
+            ): (
                 uops,
                 executed,
             ),