Allow cumulative statistics to read/write auxiliary data from/to disk

Cumulative stats kinds gain the capability to write additional per-entry
data when flushing the stats at shutdown, and read this data when
loading back the stats at startup.  This can be fit for example in the
case of variable-length data (like normalized query strings), so as it
becomes possible to link the shared memory stats entries to data that is
stored in a different area, like a DSA segment.

Three new optional callbacks are added to PgStat_KindInfo, available to
variable-numbered stats kinds:
* to_serialized_data: writes auxiliary data for an entry.
* from_serialized_data: reads auxiliary data for an entry.
* finish: performs actions after read/write/discard operations.  This is
invoked after processing all the entries of a kind, allowing extensions
to close file handles and clean up resources.

Stats kinds have the option to store this data in the existing pgstats
file, but can as well store it in one or more additional files whose
names can be built upon the entry keys.  The new serialized callbacks
are called once an entry key is read or written from the main stats
file.  A file descriptor to the main pgstats file is available in the
arguments of the callbacks.

Author: Sami Imseih <samimseih@gmail.com>
Co-authored-by: Michael Paquier <michael@paquier.xyz>
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/CAA5RZ0s9SDOu+Z6veoJCHWk+kDeTktAtC-KY9fQ9Z6BJdDUirQ@mail.gmail.com

diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c
index 8713c7a048340f03945af2733d3d80f7874094e9..f317c6e8e908cbc6ee8efad23808596820ee372a 100644 (file)
--- a/src/backend/utils/activity/pgstat.c
+++ b/src/backend/utils/activity/pgstat.c
@@ -523,6 +523,7 @@ pgstat_discard_stats(void)
 
        /* NB: this needs to be done even in single user mode */
 
+       /* First, cleanup the main pgstats file */
        ret = unlink(PGSTAT_STAT_PERMANENT_FILENAME);
        if (ret != 0)
        {
@@ -544,6 +545,15 @@ pgstat_discard_stats(void)
                                                                 PGSTAT_STAT_PERMANENT_FILENAME)));
        }
 
+       /* Finish callbacks, if required */
+       for (PgStat_Kind kind = PGSTAT_KIND_MIN; kind <= PGSTAT_KIND_MAX; kind++)
+       {
+               const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind);
+
+               if (kind_info && kind_info->finish)
+                       kind_info->finish(STATS_DISCARD);
+       }
+
        /*
         * Reset stats contents. This will set reset timestamps of fixed-numbered
         * stats to the current time (no variable stats exist).
@@ -1702,6 +1712,10 @@ pgstat_write_statsfile(void)
                pgstat_write_chunk(fpout,
                                                   pgstat_get_entry_data(ps->key.kind, shstats),
                                                   pgstat_get_entry_len(ps->key.kind));
+
+               /* Write more data for the entry, if required */
+               if (kind_info->to_serialized_data)
+                       kind_info->to_serialized_data(&ps->key, shstats, fpout);
        }
        dshash_seq_term(&hstat);
 
@@ -1734,6 +1748,15 @@ pgstat_write_statsfile(void)
                /* durable_rename already emitted log message */
                unlink(tmpfile);
        }
+
+       /* Finish callbacks, if required */
+       for (PgStat_Kind kind = PGSTAT_KIND_MIN; kind <= PGSTAT_KIND_MAX; kind++)
+       {
+               const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind);
+
+               if (kind_info && kind_info->finish)
+                       kind_info->finish(STATS_WRITE);
+       }
 }
 
 /* helper for pgstat_read_statsfile() */
@@ -1871,6 +1894,7 @@ pgstat_read_statsfile(void)
                                        PgStat_HashKey key;
                                        PgStatShared_HashEntry *p;
                                        PgStatShared_Common *header;
+                                       const PgStat_KindInfo *kind_info = NULL;
 
                                        CHECK_FOR_INTERRUPTS();
 
@@ -1891,7 +1915,8 @@ pgstat_read_statsfile(void)
                                                        goto error;
                                                }
 
-                                               if (!pgstat_get_kind_info(key.kind))
+                                               kind_info = pgstat_get_kind_info(key.kind);
+                                               if (!kind_info)
                                                {
                                                        elog(WARNING, "could not find information of kind for entry %u/%u/%" PRIu64 " of type %c",
                                                                 key.kind, key.dboid,
@@ -1902,7 +1927,6 @@ pgstat_read_statsfile(void)
                                        else
                                        {
                                                /* stats entry identified by name on disk (e.g. slots) */
-                                               const PgStat_KindInfo *kind_info = NULL;
                                                PgStat_Kind kind;
                                                NameData        name;
 
@@ -1996,6 +2020,18 @@ pgstat_read_statsfile(void)
                                                goto error;
                                        }
 
+                                       /* read more data for the entry, if required */
+                                       if (kind_info->from_serialized_data)
+                                       {
+                                               if (!kind_info->from_serialized_data(&key, header, fpin))
+                                               {
+                                                       elog(WARNING, "could not read auxiliary data for entry %u/%u/%" PRIu64 " of type %c",
+                                                                key.kind, key.dboid,
+                                                                key.objid, t);
+                                                       goto error;
+                                               }
+                                       }
+
                                        break;
                                }
                        case PGSTAT_FILE_ENTRY_END:
@@ -2019,11 +2055,21 @@ pgstat_read_statsfile(void)
        }
 
 done:
+       /* First, cleanup the main stats file */
        FreeFile(fpin);
 
        elog(DEBUG2, "removing permanent stats file \"%s\"", statfile);
        unlink(statfile);
 
+       /* Finish callbacks, if required */
+       for (PgStat_Kind kind = PGSTAT_KIND_MIN; kind <= PGSTAT_KIND_MAX; kind++)
+       {
+               const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind);
+
+               if (kind_info && kind_info->finish)
+                       kind_info->finish(STATS_READ);
+       }
+
        return;
 
 error:

diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h
index ca1ba6420ca19d1ba8f3a3b33bf9ffb838f2aeae..5c1ce4d3d6af00176af4f5848934cf26ee2f0296 100644 (file)
--- a/src/include/utils/pgstat_internal.h
+++ b/src/include/utils/pgstat_internal.h
@@ -63,6 +63,20 @@ typedef struct PgStat_HashKey
                                                                 * identifier. */
 } PgStat_HashKey;
 
+/*
+ * Tracks if the stats file is being read, written or discarded, used in
+ * combination with the finish callback.
+ *
+ * These states allow plugins that create auxiliary data files to determine
+ * the current operation and perform any necessary file cleanup.
+ */
+typedef enum PgStat_StatsFileOp
+{
+       STATS_WRITE,
+       STATS_READ,
+       STATS_DISCARD,
+} PgStat_StatsFileOp;
+
 /*
  * PgStat_HashKey should not have any padding.  Checking that the structure
  * size matches with the sum of each field is a check simple enough to
@@ -303,6 +317,38 @@ typedef struct PgStat_KindInfo
                                                                           const PgStatShared_Common *header, NameData *name);
        bool            (*from_serialized_name) (const NameData *name, PgStat_HashKey *key);
 
+       /*
+        * For variable-numbered stats: read or write additional data related to
+        * an entry, in the stats file or optionally in a different file.
+        * Optional.
+        *
+        * to_serialized_data: write auxiliary data for an entry.
+        *
+        * from_serialized_data: read auxiliary data for an entry.  Returns true
+        * on success, false on read error.
+        *
+        * "statfile" is a pointer to the on-disk stats file, named
+        * PGSTAT_STAT_PERMANENT_FILENAME.  "key" is the hash key of the entry
+        * just written or read.  "header" is a pointer to the stats data.
+        */
+       void            (*to_serialized_data) (const PgStat_HashKey *key,
+                                                                          const PgStatShared_Common *header,
+                                                                          FILE *statfile);
+       bool            (*from_serialized_data) (const PgStat_HashKey *key,
+                                                                                const PgStatShared_Common *header,
+                                                                                FILE *statfile);
+
+       /*
+        * For fixed-numbered or variable-numbered statistics.
+        *
+        * Perform custom actions when done processing the on-disk stats file
+        * after all the stats entries have been processed.  Optional.
+        *
+        * "status" tracks the operation done for the on-disk stats file (read,
+        * write, discard).
+        */
+       void            (*finish) (PgStat_StatsFileOp status);
+
        /*
         * For fixed-numbered statistics: Initialize shared memory state.
         *

diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index d7459af2ba3e1cf5e9f72c8f0402e72e5ae2d1b8..3451538565e8efbff36ef96a271768d0365e51b9 100644 (file)
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2279,6 +2279,7 @@ PgStat_StatFuncEntry
 PgStat_StatReplSlotEntry
 PgStat_StatSubEntry
 PgStat_StatTabEntry
+PgStat_StatsFileOp
 PgStat_SubXactStatus
 PgStat_TableCounts
 PgStat_TableStatus