Improve object stats (#92845)

* Add incref/decref stats

* Show ratios for allocation in summary

diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index e11d1f05129c67b0c5e346241db0539e6a10af66..96cc9dcae11a71ff9321d4d529a84e16d415bfc3 100644 (file)
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -265,53 +265,6 @@ extern int _PyStaticCode_InternStrings(PyCodeObject *co);
 
 #ifdef Py_STATS
 
-#define SPECIALIZATION_FAILURE_KINDS 30
-
-typedef struct _specialization_stats {
-    uint64_t success;
-    uint64_t failure;
-    uint64_t hit;
-    uint64_t deferred;
-    uint64_t miss;
-    uint64_t deopt;
-    uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
-} SpecializationStats;
-
-typedef struct _opcode_stats {
-    SpecializationStats specialization;
-    uint64_t execution_count;
-    uint64_t pair_count[256];
-} OpcodeStats;
-
-typedef struct _call_stats {
-    uint64_t inlined_py_calls;
-    uint64_t pyeval_calls;
-    uint64_t frames_pushed;
-    uint64_t frame_objects_created;
-} CallStats;
-
-typedef struct _object_stats {
-    uint64_t allocations;
-    uint64_t allocations512;
-    uint64_t allocations4k;
-    uint64_t allocations_big;
-    uint64_t frees;
-    uint64_t to_freelist;
-    uint64_t from_freelist;
-    uint64_t new_values;
-    uint64_t dict_materialized_on_request;
-    uint64_t dict_materialized_new_key;
-    uint64_t dict_materialized_too_big;
-    uint64_t dict_materialized_str_subclass;
-} ObjectStats;
-
-typedef struct _stats {
-    OpcodeStats opcode_stats[256];
-    CallStats call_stats;
-    ObjectStats object_stats;
-} PyStats;
-
-extern PyStats _py_stats;
 
 #define STAT_INC(opname, name) _py_stats.opcode_stats[opname].specialization.name++
 #define STAT_DEC(opname, name) _py_stats.opcode_stats[opname].specialization.name--
@@ -321,8 +274,6 @@ extern PyStats _py_stats;
 #define OBJECT_STAT_INC_COND(name, cond) \
     do { if (cond) _py_stats.object_stats.name++; } while (0)
 
-extern void _Py_PrintSpecializationStats(int to_file);
-
 // Used by the _opcode extension which is built as a shared library
 PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
 

diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
index f022f8246989d1509a00a343a841752fdcf320f0..cc50418e2ef40f0b4eb97bfa7913ba52958cfe40 100644 (file)
--- a/Include/internal/pycore_object.h
+++ b/Include/internal/pycore_object.h
@@ -34,6 +34,7 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
 static inline void
 _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
 {
+    _Py_DECREF_STAT_INC();
 #ifdef Py_REF_DEBUG
     _Py_RefTotal--;
 #endif
@@ -51,6 +52,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
 static inline void
 _Py_DECREF_NO_DEALLOC(PyObject *op)
 {
+    _Py_DECREF_STAT_INC();
 #ifdef Py_REF_DEBUG
     _Py_RefTotal--;
 #endif

diff --git a/Include/object.h b/Include/object.h
index f2af428e2bb9730e4dd0e610d21b44c7610d3d2f..f01b9fa86d01488e91c67e28f26475b06a4e9db8 100644 (file)
--- a/Include/object.h
+++ b/Include/object.h
@@ -51,6 +51,8 @@ A standard interface exists for objects that contain an array of items
 whose size is determined when the object is allocated.
 */
 
+#include "pystats.h"
+
 /* Py_DEBUG implies Py_REF_DEBUG. */
 #if defined(Py_DEBUG) && !defined(Py_REF_DEBUG)
 #  define Py_REF_DEBUG
@@ -490,6 +492,7 @@ PyAPI_FUNC(void) _Py_DecRef(PyObject *);
 
 static inline void Py_INCREF(PyObject *op)
 {
+    _Py_INCREF_STAT_INC();
 #if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000
     // Stable ABI for Python 3.10 built in debug mode.
     _Py_IncRef(op);
@@ -506,7 +509,6 @@ static inline void Py_INCREF(PyObject *op)
 #  define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op))
 #endif
 
-
 #if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000
 // Stable ABI for limited C API version 3.10 of Python debug build
 static inline void Py_DECREF(PyObject *op) {
@@ -517,6 +519,7 @@ static inline void Py_DECREF(PyObject *op) {
 #elif defined(Py_REF_DEBUG)
 static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
 {
+    _Py_DECREF_STAT_INC();
     _Py_RefTotal--;
     if (--op->ob_refcnt != 0) {
         if (op->ob_refcnt < 0) {
@@ -532,6 +535,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
 #else
 static inline void Py_DECREF(PyObject *op)
 {
+    _Py_DECREF_STAT_INC();
     // Non-limited C API and limited C API for Python 3.9 and older access
     // directly PyObject.ob_refcnt.
     if (--op->ob_refcnt == 0) {

diff --git a/Include/pystats.h b/Include/pystats.h
new file mode 100644 (file)
index 0000000..bc05dd8
--- /dev/null
+++ b/Include/pystats.h
@@ -0,0 +1,77 @@
+
+
+#ifndef Py_PYSTATS_H
+#define Py_PYSTATS_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef Py_STATS
+
+#define SPECIALIZATION_FAILURE_KINDS 32
+
+typedef struct _specialization_stats {
+    uint64_t success;
+    uint64_t failure;
+    uint64_t hit;
+    uint64_t deferred;
+    uint64_t miss;
+    uint64_t deopt;
+    uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
+} SpecializationStats;
+
+typedef struct _opcode_stats {
+    SpecializationStats specialization;
+    uint64_t execution_count;
+    uint64_t pair_count[256];
+} OpcodeStats;
+
+typedef struct _call_stats {
+    uint64_t inlined_py_calls;
+    uint64_t pyeval_calls;
+    uint64_t frames_pushed;
+    uint64_t frame_objects_created;
+} CallStats;
+
+typedef struct _object_stats {
+    uint64_t increfs;
+    uint64_t decrefs;
+    uint64_t allocations;
+    uint64_t allocations512;
+    uint64_t allocations4k;
+    uint64_t allocations_big;
+    uint64_t frees;
+    uint64_t to_freelist;
+    uint64_t from_freelist;
+    uint64_t new_values;
+    uint64_t dict_materialized_on_request;
+    uint64_t dict_materialized_new_key;
+    uint64_t dict_materialized_too_big;
+    uint64_t dict_materialized_str_subclass;
+} ObjectStats;
+
+typedef struct _stats {
+    OpcodeStats opcode_stats[256];
+    CallStats call_stats;
+    ObjectStats object_stats;
+} PyStats;
+
+PyAPI_DATA(PyStats) _py_stats;
+
+extern void _Py_PrintSpecializationStats(int to_file);
+
+
+#define _Py_INCREF_STAT_INC() _py_stats.object_stats.increfs++
+#define _Py_DECREF_STAT_INC()  _py_stats.object_stats.decrefs++
+
+#else
+
+#define _Py_INCREF_STAT_INC() ((void)0)
+#define _Py_DECREF_STAT_INC() ((void)0)
+
+#endif  // !Py_STATS
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PYSTATs_H */

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 515c18cc216664bb93c60de70a03f5f139b11797..a178dd0a65c734d3708e23f74c3065f229aabbde 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1509,6 +1509,7 @@ PYTHON_HEADERS= \
                $(srcdir)/Include/pymem.h \
                $(srcdir)/Include/pyport.h \
                $(srcdir)/Include/pystate.h \
+               $(srcdir)/Include/pystats.h \
                $(srcdir)/Include/pystrcmp.h \
                $(srcdir)/Include/pystrtod.h \
                $(srcdir)/Include/pythonrun.h \

diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index a35884b3c358879b252923a794260bbcb0afe5fb..be76f1bc55859a579a9c0d2e9d501d76c60d8785 100644 (file)
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -280,6 +280,7 @@
     <ClInclude Include="..\Include\pymem.h" />
     <ClInclude Include="..\Include\pyport.h" />
     <ClInclude Include="..\Include\pystate.h" />
+    <ClInclude Include="..\Include\pystats.h" />
     <ClInclude Include="..\Include\pystrcmp.h" />
     <ClInclude Include="..\Include\pystrtod.h" />
     <ClInclude Include="..\Include\pythonrun.h" />

diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
index ff42cc92c4bd23060c516b6174924b08b2bafc9c..5573e0020491a6b9d06cbf4d2ecf7486fd8e0019 100644 (file)
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -174,6 +174,9 @@
     <ClInclude Include="..\Include\pystate.h">
       <Filter>Include</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\pystats.h">
+      <Filter>Include</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\pystrcmp.h">
       <Filter>Include</Filter>
     </ClInclude>

diff --git a/Python/ceval.c b/Python/ceval.c
index c73218fcf307efc2f63aa19a9197cbc61543c100..c81d0efff9b9b35ba7aa22346acd557ee2921e59 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -55,6 +55,7 @@
 #undef Py_DECREF
 #define Py_DECREF(arg) \
     do { \
+        _Py_DECREF_STAT_INC(); \
         PyObject *op = _PyObject_CAST(arg); \
         if (--op->ob_refcnt == 0) { \
             destructor dealloc = Py_TYPE(op)->tp_dealloc; \
@@ -78,6 +79,7 @@
 #undef _Py_DECREF_SPECIALIZED
 #define _Py_DECREF_SPECIALIZED(arg, dealloc) \
     do { \
+        _Py_DECREF_STAT_INC(); \
         PyObject *op = _PyObject_CAST(arg); \
         if (--op->ob_refcnt == 0) { \
             destructor d = (destructor)(dealloc); \

diff --git a/Python/specialize.c b/Python/specialize.c
index fa42993606fd639c99064cd9bf3328e86b555b8c..6a91389f85610610e1146711bdf7aceed6d47a26 100644 (file)
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -191,6 +191,8 @@ print_object_stats(FILE *out, ObjectStats *stats)
     fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);
     fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
     fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
+    fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs);
+    fprintf(out, "Object decrefs: %" PRIu64 "\n", stats->decrefs);
     fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);
     fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
     fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);

diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py
index bc528ca316f9315f18b37e9bd83bae26ac7836c5..91b190114008eade8a4dbc4f455e3ebccf5032c8 100644 (file)
--- a/Tools/scripts/summarize_stats.py
+++ b/Tools/scripts/summarize_stats.py
@@ -8,6 +8,7 @@ import opcode
 from datetime import date
 import itertools
 import argparse
+import sys
 
 if os.name == "nt":
     DEFAULT_DIR = "c:\\temp\\py_stats\\"
@@ -88,7 +89,11 @@ def gather_stats():
     for filename in os.listdir(DEFAULT_DIR):
         with open(os.path.join(DEFAULT_DIR, filename)) as fd:
             for line in fd:
-                key, value = line.split(":")
+                try:
+                    key, value = line.split(":")
+                except ValueError:
+                    print (f"Unparsable line: '{line.strip()}' in  {filename}", file=sys.stderr)
+                    continue
                 key = key.strip()
                 value = int(value)
                 stats[key] += value
@@ -265,17 +270,20 @@ def emit_call_stats(stats):
 
 def emit_object_stats(stats):
     with Section("Object stats", summary="allocations, frees and dict materializatons"):
-        total = stats.get("Object new values")
+        total_materializations = stats.get("Object new values")
+        total_allocations = stats.get("Object allocations")
         rows = []
         for key, value in stats.items():
             if key.startswith("Object"):
                 if "materialize" in key:
-                    materialize = f"{100*value/total:0.1f}%"
+                    ratio = f"{100*value/total_materializations:0.1f}%"
+                elif "allocations" in key:
+                    ratio = f"{100*value/total_allocations:0.1f}%"
                 else:
-                    materialize = ""
+                    ratio = ""
                 label = key[6:].strip()
                 label = label[0].upper() + label[1:]
-                rows.append((label, value, materialize))
+                rows.append((label, value, ratio))
         emit_table(("",  "Count:", "Ratio:"), rows)
 
 def get_total(opcode_stats):