Add wal_fpi_bytes to pg_stat_wal and pg_stat_get_backend_wal()

This new counter, called "wal_fpi_bytes", tracks the total amount in
bytes of full page images (FPIs) generated in WAL.  This data becomes
available globally via pg_stat_wal, and for backend statistics via
pg_stat_get_backend_wal().

Previously, this information could only be retrieved with pg_waldump or
pg_walinspect, which may not be available depending on the environment,
and are expensive to execute.  It offers hints about how much FPIs
impact the WAL generated, which could be a large percentage for some
workloads, as well as the effects of wal_compression or page holes.

Bump catalog version.
Bump PGSTAT_FILE_FORMAT_ID, due to the addition of wal_fpi_bytes in
PgStat_WalCounters.

Author: Shinya Kato <shinya11.kato@gmail.com>
Reviewed-by: Michael Paquier <michael@paquier.xyz>
Discussion: https://postgr.es/m/CAOzEurQtZEAfg6P0kU3Wa-f9BWQOi0RzJEMPN56wNTOmJLmfaQ@mail.gmail.com

diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index d5f0fb7ba7c95c4fb769cdd3af997692182a6696..f3bf527d5b4b92fb1f4699f6bbb52775788af3c1 100644 (file)
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -3323,6 +3323,15 @@ description | Waiting for a newly initialized WAL file to reach durable storage
       </para></entry>
      </row>
 
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>wal_fpi_bytes</structfield> <type>numeric</type>
+      </para>
+      <para>
+       Total amount of WAL full page images in bytes
+      </para></entry>
+     </row>
+
      <row>
       <entry role="catalog_table_entry"><para role="column_definition">
        <structfield>wal_buffers_full</structfield> <type>bigint</type>

diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 496e0fa4ac67b647ac8235956466f0e24cd679fe..b3abf386f801c2f5ec6b53e14894342792543611 100644 (file)
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -33,12 +33,14 @@
 #include "access/xloginsert.h"
 #include "catalog/pg_control.h"
 #include "common/pg_lzcompress.h"
+#include "executor/instrument.h"
 #include "miscadmin.h"
 #include "pg_trace.h"
 #include "replication/origin.h"
 #include "storage/bufmgr.h"
 #include "storage/proc.h"
 #include "utils/memutils.h"
+#include "utils/pgstat_internal.h"
 
 /*
  * Guess the maximum buffer size required to store a compressed version of
@@ -796,6 +798,10 @@ XLogRecordAssemble(RmgrId rmid, uint8 info,
                        }
 
                        total_len += bimg.length;
+
+                       /* Track the WAL full page images in bytes */
+                       pgWalUsage.wal_fpi_bytes += bimg.length;
+                       pgstat_report_fixed = true;
                }
 
                if (needs_data)

diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 823776c14989ba30ddbd02219371d6ced5e70d59..dec8df4f8ee67373d4eee51b711d372fcf6100d1 100644 (file)
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1221,6 +1221,7 @@ CREATE VIEW pg_stat_wal AS
         w.wal_records,
         w.wal_fpi,
         w.wal_bytes,
+        w.wal_fpi_bytes,
         w.wal_buffers_full,
         w.stats_reset
     FROM pg_stat_get_wal() w;

diff --git a/src/backend/executor/instrument.c b/src/backend/executor/instrument.c
index 56e635f47000dd8e74e3f32b55235b8f2a036100..9e11c662a7c1d23f707a46d36457a278bd425539 100644 (file)
--- a/src/backend/executor/instrument.c
+++ b/src/backend/executor/instrument.c
@@ -280,6 +280,7 @@ WalUsageAdd(WalUsage *dst, WalUsage *add)
        dst->wal_bytes += add->wal_bytes;
        dst->wal_records += add->wal_records;
        dst->wal_fpi += add->wal_fpi;
+       dst->wal_fpi_bytes += add->wal_fpi_bytes;
        dst->wal_buffers_full += add->wal_buffers_full;
 }
 
@@ -289,5 +290,6 @@ WalUsageAccumDiff(WalUsage *dst, const WalUsage *add, const WalUsage *sub)
        dst->wal_bytes += add->wal_bytes - sub->wal_bytes;
        dst->wal_records += add->wal_records - sub->wal_records;
        dst->wal_fpi += add->wal_fpi - sub->wal_fpi;
+       dst->wal_fpi_bytes += add->wal_fpi_bytes - sub->wal_fpi_bytes;
        dst->wal_buffers_full += add->wal_buffers_full - sub->wal_buffers_full;
 }

diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c
index a864ae8e6a6087112f104dd591b103289a70ef69..199ba2cc17a789b953379be15d63ba0858098386 100644 (file)
--- a/src/backend/utils/activity/pgstat_backend.c
+++ b/src/backend/utils/activity/pgstat_backend.c
@@ -252,6 +252,7 @@ pgstat_flush_backend_entry_wal(PgStat_EntryRef *entry_ref)
        WALSTAT_ACC(wal_records, wal_usage_diff);
        WALSTAT_ACC(wal_fpi, wal_usage_diff);
        WALSTAT_ACC(wal_bytes, wal_usage_diff);
+       WALSTAT_ACC(wal_fpi_bytes, wal_usage_diff);
 #undef WALSTAT_ACC
 
        /*

diff --git a/src/backend/utils/activity/pgstat_wal.c b/src/backend/utils/activity/pgstat_wal.c
index 0d04480d2f6d0f1450d9f8bad2cb004b75953187..d4edb8b57335db61cfbbeee3582cc7ea819b0aad 100644 (file)
--- a/src/backend/utils/activity/pgstat_wal.c
+++ b/src/backend/utils/activity/pgstat_wal.c
@@ -121,6 +121,7 @@ pgstat_wal_flush_cb(bool nowait)
        WALSTAT_ACC(wal_records, wal_usage_diff);
        WALSTAT_ACC(wal_fpi, wal_usage_diff);
        WALSTAT_ACC(wal_bytes, wal_usage_diff);
+       WALSTAT_ACC(wal_fpi_bytes, wal_usage_diff);
        WALSTAT_ACC(wal_buffers_full, wal_usage_diff);
 #undef WALSTAT_ACC
 

diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 1fe33df2756ec87d99ebf124e576f943aa9aa912..a710508979e47c3b3fa3dc29494fe109abfe3cab 100644 (file)
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -1637,7 +1637,7 @@ static Datum
 pg_stat_wal_build_tuple(PgStat_WalCounters wal_counters,
                                                TimestampTz stat_reset_timestamp)
 {
-#define PG_STAT_WAL_COLS       5
+#define PG_STAT_WAL_COLS       6
        TupleDesc       tupdesc;
        Datum           values[PG_STAT_WAL_COLS] = {0};
        bool            nulls[PG_STAT_WAL_COLS] = {0};
@@ -1651,9 +1651,11 @@ pg_stat_wal_build_tuple(PgStat_WalCounters wal_counters,
                                           INT8OID, -1, 0);
        TupleDescInitEntry(tupdesc, (AttrNumber) 3, "wal_bytes",
                                           NUMERICOID, -1, 0);
-       TupleDescInitEntry(tupdesc, (AttrNumber) 4, "wal_buffers_full",
+       TupleDescInitEntry(tupdesc, (AttrNumber) 4, "wal_fpi_bytes",
+                                          NUMERICOID, -1, 0);
+       TupleDescInitEntry(tupdesc, (AttrNumber) 5, "wal_buffers_full",
                                           INT8OID, -1, 0);
-       TupleDescInitEntry(tupdesc, (AttrNumber) 5, "stats_reset",
+       TupleDescInitEntry(tupdesc, (AttrNumber) 6, "stats_reset",
                                           TIMESTAMPTZOID, -1, 0);
 
        BlessTupleDesc(tupdesc);
@@ -1669,12 +1671,18 @@ pg_stat_wal_build_tuple(PgStat_WalCounters wal_counters,
                                                                        ObjectIdGetDatum(0),
                                                                        Int32GetDatum(-1));
 
-       values[3] = Int64GetDatum(wal_counters.wal_buffers_full);
+       snprintf(buf, sizeof buf, UINT64_FORMAT, wal_counters.wal_fpi_bytes);
+       values[3] = DirectFunctionCall3(numeric_in,
+                                                                       CStringGetDatum(buf),
+                                                                       ObjectIdGetDatum(0),
+                                                                       Int32GetDatum(-1));
+
+       values[4] = Int64GetDatum(wal_counters.wal_buffers_full);
 
        if (stat_reset_timestamp != 0)
-               values[4] = TimestampTzGetDatum(stat_reset_timestamp);
+               values[5] = TimestampTzGetDatum(stat_reset_timestamp);
        else
-               nulls[4] = true;
+               nulls[5] = true;
 
        /* Returns the record as Datum */
        PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls)));

diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h
index 1b0b16a343f5b4aa4b3cf951905a12842258bebd..18e95179ab6762a7079b1ec1f64df33fb61e4823 100644 (file)
--- a/src/include/catalog/catversion.h
+++ b/src/include/catalog/catversion.h
@@ -57,6 +57,6 @@
  */
 
 /*                                                     yyyymmddN */
-#define CATALOG_VERSION_NO     202510221
+#define CATALOG_VERSION_NO     202510281
 
 #endif

diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index eecb43ec6f0fb6b57412bd9f0430878f5a255495..9121a382f76b477f08c835a66c5602bb16f1fadc 100644 (file)
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -6029,16 +6029,16 @@
 { oid => '1136', descr => 'statistics: information about WAL activity',
   proname => 'pg_stat_get_wal', proisstrict => 'f', provolatile => 's',
   proparallel => 'r', prorettype => 'record', proargtypes => '',
-  proallargtypes => '{int8,int8,numeric,int8,timestamptz}',
-  proargmodes => '{o,o,o,o,o}',
-  proargnames => '{wal_records,wal_fpi,wal_bytes,wal_buffers_full,stats_reset}',
+  proallargtypes => '{int8,int8,numeric,numeric,int8,timestamptz}',
+  proargmodes => '{o,o,o,o,o,o}',
+  proargnames => '{wal_records,wal_fpi,wal_bytes,wal_fpi_bytes,wal_buffers_full,stats_reset}',
   prosrc => 'pg_stat_get_wal' },
 { oid => '6313', descr => 'statistics: backend WAL activity',
   proname => 'pg_stat_get_backend_wal', provolatile => 'v', proparallel => 'r',
   prorettype => 'record', proargtypes => 'int4',
-  proallargtypes => '{int4,int8,int8,numeric,int8,timestamptz}',
-  proargmodes => '{i,o,o,o,o,o}',
-  proargnames => '{backend_pid,wal_records,wal_fpi,wal_bytes,wal_buffers_full,stats_reset}',
+  proallargtypes => '{int4,int8,int8,numeric,numeric,int8,timestamptz}',
+  proargmodes => '{i,o,o,o,o,o,o}',
+  proargnames => '{backend_pid,wal_records,wal_fpi,wal_bytes,wal_fpi_bytes,wal_buffers_full,stats_reset}',
   prosrc => 'pg_stat_get_backend_wal' },
 { oid => '6248', descr => 'statistics: information about WAL prefetching',
   proname => 'pg_stat_get_recovery_prefetch', prorows => '1', proretset => 't',

diff --git a/src/include/executor/instrument.h b/src/include/executor/instrument.h
index 03653ab6c6cde6f7e17434e212327d9271f71659..ffe470f2b84c921b370b161160a83d9fafa99db6 100644 (file)
--- a/src/include/executor/instrument.h
+++ b/src/include/executor/instrument.h
@@ -53,6 +53,7 @@ typedef struct WalUsage
        int64           wal_records;    /* # of WAL records produced */
        int64           wal_fpi;                /* # of WAL full page images produced */
        uint64          wal_bytes;              /* size of WAL records produced */
+       uint64          wal_fpi_bytes;  /* size of WAL full page images produced */
        int64           wal_buffers_full;       /* # of times the WAL buffers became full */
 } WalUsage;
 

diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index bc8077cbae6364f4fdf42023a22e226c70b95fd5..7ae503e71a27a418a7c2a5b5f42af0761bb190d6 100644 (file)
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -212,7 +212,7 @@ typedef struct PgStat_TableXactStatus
  * ------------------------------------------------------------
  */
 
-#define PGSTAT_FILE_FORMAT_ID  0x01A5BCB9
+#define PGSTAT_FILE_FORMAT_ID  0x01A5BCBA
 
 typedef struct PgStat_ArchiverStats
 {
@@ -473,6 +473,7 @@ typedef struct PgStat_WalCounters
        PgStat_Counter wal_records;
        PgStat_Counter wal_fpi;
        uint64          wal_bytes;
+       uint64          wal_fpi_bytes;
        PgStat_Counter wal_buffers_full;
 } PgStat_WalCounters;
 

diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 16753b2e4c0b004766ac0be078629bf08dbd4026..77e25ca029e5c1c0d5b9d4a93f26ac0f4cfb0377 100644 (file)
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -2306,9 +2306,10 @@ pg_stat_user_tables| SELECT relid,
 pg_stat_wal| SELECT wal_records,
     wal_fpi,
     wal_bytes,
+    wal_fpi_bytes,
     wal_buffers_full,
     stats_reset
-   FROM pg_stat_get_wal() w(wal_records, wal_fpi, wal_bytes, wal_buffers_full, stats_reset);
+   FROM pg_stat_get_wal() w(wal_records, wal_fpi, wal_bytes, wal_fpi_bytes, wal_buffers_full, stats_reset);
 pg_stat_wal_receiver| SELECT pid,
     status,
     receive_start_lsn,