bool historic;
XLogRecPtr read_upto;
bool end_of_wal;
+ int num_descendant_tlis;
+ TimeLineID *descendant_tlis;
} SummarizerReadLocalXLogPrivate;
/* Pointer to shared memory state. */
static void WalSummarizerShutdown(int code, Datum arg);
static XLogRecPtr GetLatestLSN(TimeLineID *tli);
+static XLogRecPtr WalSummarizerSwitchPoint(TimeLineID current_tli, List *tles,
+ int *num_descendant_tlis,
+ TimeLineID **descendant_tlis);
static void ProcessWalSummarizerInterrupts(void);
static XLogRecPtr SummarizeWAL(TimeLineID tli, XLogRecPtr start_lsn,
bool exact, XLogRecPtr switch_lsn,
- XLogRecPtr maximum_lsn);
+ XLogRecPtr maximum_lsn,
+ int num_descendant_tlis, TimeLineID *descendant_tlis);
static void SummarizeDbaseRecord(XLogReaderState *xlogreader,
BlockRefTable *brtab);
static void SummarizeSmgrRecord(XLogReaderState *xlogreader,
BlockRefTable *brtab);
static bool SummarizeXlogRecord(XLogReaderState *xlogreader,
bool *new_fast_forward);
+static void summarizer_wal_segment_open(XLogReaderState *state,
+ XLogSegNo nextSegNo,
+ TimeLineID *tli_p);
static int summarizer_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen,
* true if 'current_lsn' is known to be the start of a WAL record or WAL
* segment, and false if it might be in the middle of a record someplace.
*
- * 'switch_lsn' and 'switch_tli', if set, are the LSN at which we need to
- * switch to a new timeline and the timeline to which we need to switch.
- * If not set, we either haven't figured out the answers yet or we're
- * already on the latest timeline.
+ * 'switch_lsn', is the LSN at which we need to switch to a new timeline.
+ * If not set, we either haven't figured out the answer yet or we're
+ * already on the latest timeline. 'descendant_tlis' stores an array of
+ * future timeline IDs to which we know we'll need to switch, and
+ * 'num_descendant_tlis' is the length of that array. The first element of
+ * the array is the first timeline to which we will need to switch.
*/
XLogRecPtr current_lsn;
TimeLineID current_tli;
bool exact;
XLogRecPtr switch_lsn = InvalidXLogRecPtr;
- TimeLineID switch_tli = 0;
+ int num_descendant_tlis = 0;
+ TimeLineID *descendant_tlis = NULL;
Assert(startup_data_len == 0);
if (current_tli != latest_tli && !XLogRecPtrIsValid(switch_lsn))
{
List *tles = readTimeLineHistory(latest_tli);
+ int new_num_descendant_tlis;
+ TimeLineID *new_descendant_tlis;
- switch_lsn = tliSwitchPoint(current_tli, tles, &switch_tli);
+ /*
+ * Make sure that the array of descendant TLIs get stored into
+ * TopMemoryContext.
+ */
+ MemoryContextSwitchTo(TopMemoryContext);
+ switch_lsn = WalSummarizerSwitchPoint(current_tli, tles,
+ &new_num_descendant_tlis,
+ &new_descendant_tlis);
+ MemoryContextSwitchTo(context);
+
+ /*
+ * Free any old array of descendant TLIs and install the new
+ * values.
+ */
+ if (descendant_tlis != NULL)
+ pfree(descendant_tlis);
+ num_descendant_tlis = new_num_descendant_tlis;
+ descendant_tlis = new_descendant_tlis;
+
+ /* Debug message. */
ereport(DEBUG1,
errmsg_internal("switch point from TLI %u to TLI %u is at %X/%08X",
- current_tli, switch_tli, LSN_FORMAT_ARGS(switch_lsn)));
+ current_tli, descendant_tlis[0],
+ LSN_FORMAT_ARGS(switch_lsn)));
}
/*
if (XLogRecPtrIsValid(switch_lsn) && current_lsn >= switch_lsn)
{
/* Restart summarization from switch point. */
- current_tli = switch_tli;
+ Assert(num_descendant_tlis > 0);
+ current_tli = descendant_tlis[0];
current_lsn = switch_lsn;
- /* Next timeline and switch point, if any, not yet known. */
+ /* Switch point, if any, and future TLIs, not yet known. */
switch_lsn = InvalidXLogRecPtr;
- switch_tli = 0;
+ num_descendant_tlis = 0;
+ pfree(descendant_tlis);
+ descendant_tlis = NULL;
/* Update (really, rewind, if needed) state in shared memory. */
LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
maximum_lsn = XLogRecPtrIsValid(switch_lsn) ? switch_lsn : latest_lsn;
end_of_summary_lsn = SummarizeWAL(current_tli,
current_lsn, exact,
- switch_lsn, maximum_lsn);
+ switch_lsn, maximum_lsn,
+ num_descendant_tlis, descendant_tlis);
Assert(XLogRecPtrIsValid(end_of_summary_lsn));
Assert(end_of_summary_lsn >= current_lsn);
}
}
+/*
+ * Compute the LSN at which we switched from current_tli to some later timeline.
+ * 'tles' must be the timeline history of the latest timeline.
+ *
+ * As a side effect, we set *num_descendant_tlis to the number of later TLIs that
+ * appear in the timeline history, and *descendant_tlis to an array of those TLIs,
+ * starting with immediate successor of current_tli.
+ */
+static XLogRecPtr
+WalSummarizerSwitchPoint(TimeLineID current_tli, List *tles,
+ int *num_descendant_tlis, TimeLineID **descendant_tlis)
+{
+ XLogRecPtr switch_lsn = InvalidXLogRecPtr;
+ int count = 0;
+
+ /*
+ * Find the switch point and, at the same time, count the number of TLIs
+ * in this history that are descendants of that TLI.
+ */
+ foreach_ptr(TimeLineHistoryEntry, tle, tles)
+ {
+ if (tle->tli == current_tli)
+ {
+ switch_lsn = tle->end;
+ break;
+ }
+ ++count;
+ }
+
+ /* Sanity checks. */
+ if (!XLogRecPtrIsValid(switch_lsn))
+ ereport(ERROR,
+ (errmsg("requested timeline %u is not in this server's history",
+ current_tli)));
+ if (count == 0)
+ elog(ERROR, "cannot compute switch point for current TLI %u", current_tli);
+
+ /*
+ * Generate an array of TLIs that are part of this history and descendants
+ * of current_tli. The TLE list starts with the newest timeline and works
+ * backward toward older timelines; we want the opposite ordering.
+ */
+ *num_descendant_tlis = count;
+ *descendant_tlis = palloc_array(TimeLineID, count);
+ for (int i = 0; i < count; ++i)
+ {
+ TimeLineHistoryEntry *tle;
+
+ tle = (TimeLineHistoryEntry *) list_nth(tles, count - i - 1);
+ (*descendant_tlis)[i] = tle->tli;
+ }
+
+ /* Return value is the switchpoint. */
+ return switch_lsn;
+}
+
/*
* Interrupt handler for main loop of WAL summarizer process.
*/
*/
static XLogRecPtr
SummarizeWAL(TimeLineID tli, XLogRecPtr start_lsn, bool exact,
- XLogRecPtr switch_lsn, XLogRecPtr maximum_lsn)
+ XLogRecPtr switch_lsn, XLogRecPtr maximum_lsn,
+ int num_descendant_tlis, TimeLineID *descendant_tlis)
{
SummarizerReadLocalXLogPrivate *private_data;
XLogReaderState *xlogreader;
private_data->tli = tli;
private_data->historic = XLogRecPtrIsValid(switch_lsn);
private_data->read_upto = maximum_lsn;
+ private_data->num_descendant_tlis = num_descendant_tlis;
+ private_data->descendant_tlis = descendant_tlis;
/* Create xlogreader. */
xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
XL_ROUTINE(.page_read = &summarizer_read_local_xlog_page,
- .segment_open = &wal_segment_open,
+ .segment_open = &summarizer_wal_segment_open,
.segment_close = &wal_segment_close),
private_data);
if (xlogreader == NULL)
return true;
}
+/*
+ * Similar to wal_segment_open, but checks for a file on any descendant timelines
+ * known to us if no file is found on the requested timeline.
+ */
+static void
+summarizer_wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
+ TimeLineID *tli_p)
+{
+ SummarizerReadLocalXLogPrivate *private_data = state->private_data;
+ int count = 0;
+ TimeLineID tli = *tli_p;
+ char path[MAXPGPATH];
+
+ for (;;)
+ {
+ XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
+ state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
+ if (state->seg.ws_file >= 0)
+ {
+ *tli_p = tli;
+ return;
+ }
+
+ /*
+ * If the error is anything other than file-not-found, complain at
+ * once.
+ */
+ if (errno != ENOENT)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not open file \"%s\": %m",
+ path)));
+
+ /* Try other timelines, if any remain. */
+ if (count >= private_data->num_descendant_tlis)
+ break;
+ tli = private_data->descendant_tlis[count];
+ ++count;
+ }
+
+ /* Complain about the originally requested filename. */
+ XLogFilePath(path, *tli_p, nextSegNo, state->segcxt.ws_segsize);
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("requested WAL segment %s has already been removed",
+ path)));
+}
+
/*
* Similar to read_local_xlog_page, but limited to read from one particular
* timeline. If the end of WAL is reached, it will wait for more if reading
--- /dev/null
+# Copyright (c) 2021-2026, PostgreSQL Global Development Group
+#
+# In the original version of the WAL summarizer code, we were only willing
+# to read WAL for a given TLI from a file with that exact TLI encoded into
+# the filename. This could result in WAL summarization running on an archiving
+# standby getting stuck.
+#
+# The reason for the problem is that when a new primary is promoted, the
+# partial file that ends the old timeline is renamed, giving it a ".partial"
+# suffix, meaning that it will be ignored by both recovery and by the WAL
+# summarizer. The bytes that appear at the start of that segment will be copied
+# into the first segment on the new timeline, and recovery was able to read
+# them from there and work as expected. However, the WAL summarizer was
+# unwilling to do the same thing, so it got stuck. This test aims to validate
+# that this bug has been fixed.
+
+use strict;
+use warnings FATAL => 'all';
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Set up node1 as primary.
+my $node1 = PostgreSQL::Test::Cluster->new('node1');
+$node1->init(allows_streaming => 1);
+$node1->append_conf('postgresql.conf', <<EOM);
+autovacuum = off
+EOM
+$node1->start;
+
+# Set up node2 as a standby for node1. Use archive_mode=always, to make sure it
+# archives both before and after promotion.
+$node1->backup('backup1');
+my $node2 = PostgreSQL::Test::Cluster->new('node2');
+$node2->init_from_backup($node1, 'backup1', has_streaming => 1);
+$node2->enable_archiving();
+$node2->append_conf('postgresql.conf', <<EOM);
+archive_mode = always
+EOM
+$node2->start;
+
+# Wait for node2 to catch up.
+$node1->wait_for_replay_catchup($node2);
+
+# Set up node3 as a standby for node2. We want it to fetch WAL only from the
+# archive, so we clear primary_conninfo. We don't want long delays during the
+# test, so we reduce wal_retrieve_retry_interval. We also don't want it to try
+# to archive anything to node2's archive, but at the same time, we don't want
+# it to remove WAL before we enable WAL summarization. To accomplish that, we
+# set archive_command to the empty string.
+$node2->backup('backup2');
+my $node3 = PostgreSQL::Test::Cluster->new('node3');
+$node3->init_from_backup($node2, 'backup2', has_restoring => 1);
+$node3->append_conf('postgresql.conf', <<EOM);
+primary_conninfo = ''
+wal_retrieve_retry_interval = '500ms'
+archive_command = ''
+EOM
+$node3->start;
+
+# Create a new, partially-filled WAL segment on node1.
+$node1->safe_psql('postgres', <<EOM);
+SELECT pg_switch_wal();
+CREATE TABLE dummy ();
+EOM
+$node1->wait_for_replay_catchup($node2);
+
+# Record the WAL insert LSN on node1, so we can later verify that summarization
+# on node3 advances past this point.
+my $node1_final_lsn = $node1->safe_psql('postgres',
+ 'SELECT pg_current_wal_insert_lsn()');
+
+# Promote node2. This creates a timeline switch that node3 must follow.
+$node2->promote;
+$node2->poll_query_until('postgres', "SELECT pg_is_in_recovery() = 'f';");
+
+# Cause the partial segment to get archived on the *new* timeline.
+#
+# In more detail: the WAL segment that contains the current insert LSN exists
+# on timeline 1, but since all we did is CREATE TABLE dummy (), it wasn't full.
+# We're now running on timeline 2, and pg_switch_wal() fills up the rest of the
+# segment. So the full segment should get archived on timeline 2, but not on
+# timeline 1. We do a CHECKPOINT here to make sure that the summarizer tries
+# to progress.
+$node2->safe_psql('postgres', <<EOM);
+SELECT pg_switch_wal();
+CHECKPOINT;
+EOM
+
+# Wait until replay has reached TLI 2 on node3, and then start the WAL
+# summarizer. If node3 is started with the summarizer already enabled, then
+# it may try to fetch the partial segment from timeline 1 before it learns
+# about timeline 2. If that happens, it will error out, wait 10 seconds, and
+# retry, slowing down the test. This avoids that.
+#
+# We set log_min_messages=debug1 at the same time we enable WAL summarization
+# so that we get useful debug messages if there's any problem.
+$node3->poll_query_until('postgres',
+ "SELECT replay_end_tli = 2 FROM pg_stat_get_recovery()")
+ or die "TLI 2 not reached on node3";
+$node3->append_conf('postgresql.conf', <<EOM);
+summarize_wal = on
+log_min_messages = debug1
+EOM
+$node3->reload;
+
+# Wait for WAL summarization on node3 to advance past the pre-promotion LSN.
+# If the bug is present, the summarizer gets stuck trying to open the old
+# timeline's segment file.
+my $result = $node3->poll_query_until('postgres', <<EOM);
+SELECT EXISTS (SELECT * FROM pg_available_wal_summaries() WHERE tli = 2)
+EOM
+ok($result, "WAL summarization on node3 advanced past timeline switch");
+
+# Verify the absence of summaries on timeline 2 starting before the final LSN
+# from timeline 1.
+my $too_early_summaries = $node3->safe_psql('postgres', <<EOM);
+SELECT start_lsn, end_lsn FROM pg_available_wal_summaries()
+WHERE tli = 2 AND start_lsn < '$node1_final_lsn' ORDER BY start_lsn
+EOM
+is($too_early_summaries, '', "no summaries from before LSN $node1_final_lsn");
+
+# Verify the presence of summaries on timeline 2 starting at or after the final
+# LSN from timeline 1.
+my $summaries = $node3->safe_psql('postgres', <<EOM);
+SELECT tli, start_lsn, end_lsn FROM pg_available_wal_summaries()
+WHERE tli = 2 AND start_lsn >= '$node1_final_lsn' ORDER BY start_lsn
+EOM
+my @summary_lines = split(/\n/, $summaries);
+ok(@summary_lines > 0, "at least one summary from LSN $node1_final_lsn or later");
+
+# We expect the new summaries to be empty, because we have not actually touched
+# any block data (and we disabled autovacuum from the start).
+for my $line (@summary_lines)
+{
+ my ($tli, $start_lsn, $end_lsn) = split(/\|/, $line);
+ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
+ $node3->data_dir, $tli,
+ split(m@/@, $start_lsn),
+ split(m@/@, $end_lsn);
+ my ($stdout, $stderr) = run_command([ 'pg_walsummary', $filename ]);
+ is($stdout, '', "pg_walsummary TLI $tli $start_lsn-$end_lsn: no blocks");
+ is($stderr, '', "pg_walsummary TLI $tli $start_lsn-$end_lsn: no error");
+}
+
+done_testing();
projects / thirdparty / postgresql.git / commitdiff
