Skip WAL recycling and preallocation during archive recovery.

The previous commit addressed the chief consequences of a race condition
between InstallXLogFileSegment() and KeepFileRestoredFromArchive().  Fix
three lesser consequences.  A spurious durable_rename_excl() LOG message
remained possible.  KeepFileRestoredFromArchive() wasted the proceeds of
WAL recycling and preallocation.  Finally, XLogFileInitInternal() could
return a descriptor for a file that KeepFileRestoredFromArchive() had
already unlinked.  That felt like a recipe for future bugs.

This commit has been applied as of cc2c7d65fc27 in v15 and newer
versions.  This is required on stable branches of v13 and v14 to fix a
regression reported by Noah Misch, introduced by 1f95181b44c8, causing
spurious failures in archive recovery (neither streaming nor archive
recovery) with concurrent restartpoints.  The backpatched versions of
the patches have been aligned on these branches by me, Noah Misch is the
author.  Tests have been conducted by the both of us.

Note that this commit is known to have introduced a regression of its
own.  This is fixed by the commit following this one, and not grouped in
a single commit to keep the commit history consistent across all
branches.

Reported-by: Arun Thirupathi
Author: Noah Misch <noah@leadboat.com>
Discussion: https://postgr.es/m/20210202151416.GB3304930@rfd.leadboat.com
Discussion: https://postgr.es/m/20250306193013.36.nmisch@google.com
Backpatch-through: 13

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 73a828f075ad77172071aad0e58398987d336cc7..c5fe7e6fa4cccabdb330636f5db8e0c22c3d87f7 100644 (file)
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -673,6 +673,16 @@ typedef struct XLogCtlData
         */
        bool            SharedHotStandbyActive;
 
+       /*
+        * InstallXLogFileSegmentActive indicates whether the checkpointer should
+        * arrange for future segments by recycling and/or PreallocXlogFiles().
+        * Protected by ControlFileLock.  Only the startup process changes it.  If
+        * true, anyone can use InstallXLogFileSegment().  If false, the startup
+        * process owns the exclusive right to install segments, by reading from
+        * the archive and possibly replacing existing files.
+        */
+       bool            InstallXLogFileSegmentActive;
+
        /*
         * SharedPromoteIsTriggered indicates if a standby promotion has been
         * triggered.  Protected by info_lck.
@@ -935,6 +945,7 @@ static int  XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
                                                 int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
 static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
                                                                                bool fetching_ckpt, XLogRecPtr tliRecPtr);
+static void XLogShutdownWalRcv(void);
 static int     emode_for_corrupt_record(int emode, XLogRecPtr RecPtr);
 static void XLogFileClose(void);
 static void PreallocXlogFiles(XLogRecPtr endptr);
@@ -3653,8 +3664,8 @@ XLogFileCopy(XLogSegNo destsegno, TimeLineID srcTLI, XLogSegNo srcsegno,
  * is false.)
  *
  * Returns true if the file was installed successfully.  false indicates that
- * max_segno limit was exceeded, or an error occurred while renaming the
- * file into place.
+ * max_segno limit was exceeded, the startup process has disabled this
+ * function for now, or an error occurred while renaming the file into place.
  */
 static bool
 InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
@@ -3666,6 +3677,11 @@ InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
        XLogFilePath(path, ThisTimeLineID, *segno, wal_segment_size);
 
        LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
+       if (!XLogCtl->InstallXLogFileSegmentActive)
+       {
+               LWLockRelease(ControlFileLock);
+               return false;
+       }
 
        if (!find_free)
        {
@@ -3770,6 +3786,7 @@ XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
         */
        if (source == XLOG_FROM_ARCHIVE)
        {
+               Assert(!XLogCtl->InstallXLogFileSegmentActive);
                KeepFileRestoredFromArchive(path, xlogfname);
 
                /*
@@ -3971,6 +3988,9 @@ PreallocXlogFiles(XLogRecPtr endptr)
        char            path[MAXPGPATH];
        uint64          offset;
 
+       if (!XLogCtl->InstallXLogFileSegmentActive)
+               return;                                 /* unlocked check says no */
+
        XLByteToPrevSeg(endptr, _logSegNo, wal_segment_size);
        offset = XLogSegmentOffset(endptr - 1, wal_segment_size);
        if (offset >= (uint32) (0.75 * wal_segment_size))
@@ -4252,6 +4272,7 @@ RemoveXlogFile(const char *segname, XLogSegNo recycleSegNo,
         */
        if (wal_recycle &&
                *endlogSegNo <= recycleSegNo &&
+               XLogCtl->InstallXLogFileSegmentActive &&        /* callee rechecks this */
                lstat(path, &statbuf) == 0 && S_ISREG(statbuf.st_mode) &&
                InstallXLogFileSegment(endlogSegNo, path,
                                                           true, recycleSegNo))
@@ -4265,7 +4286,7 @@ RemoveXlogFile(const char *segname, XLogSegNo recycleSegNo,
        }
        else
        {
-               /* No need for any more future segments... */
+               /* No need for any more future segments, or recycling failed ... */
                int                     rc;
 
                ereport(DEBUG2,
@@ -5270,6 +5291,7 @@ XLOGShmemInit(void)
        XLogCtl->XLogCacheBlck = XLOGbuffers - 1;
        XLogCtl->SharedRecoveryState = RECOVERY_STATE_CRASH;
        XLogCtl->SharedHotStandbyActive = false;
+       XLogCtl->InstallXLogFileSegmentActive = false;
        XLogCtl->SharedPromoteIsTriggered = false;
        XLogCtl->WalWriterSleeping = false;
 
@@ -5297,6 +5319,11 @@ BootStrapXLOG(void)
        struct timeval tv;
        pg_crc32c       crc;
 
+       /* allow ordinary WAL segment creation, like StartupXLOG() would */
+       LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
+       XLogCtl->InstallXLogFileSegmentActive = true;
+       LWLockRelease(ControlFileLock);
+
        /*
         * Select a hopefully-unique system identifier code for this installation.
         * We use the result of gettimeofday(), including the fractional seconds
@@ -7685,7 +7712,7 @@ StartupXLOG(void)
         * over these records and subsequent ones if it's still alive when we
         * start writing WAL.
         */
-       ShutdownWalRcv();
+       XLogShutdownWalRcv();
 
        /*
         * Reset unlogged relations to the contents of their INIT fork. This is
@@ -7710,7 +7737,7 @@ StartupXLOG(void)
         * recovery, e.g., timeline history file) from archive or pg_wal.
         *
         * Note that standby mode must be turned off after killing WAL receiver,
-        * i.e., calling ShutdownWalRcv().
+        * i.e., calling XLogShutdownWalRcv().
         */
        Assert(!WalRcvStreaming());
        StandbyMode = false;
@@ -7779,6 +7806,14 @@ StartupXLOG(void)
         */
        oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
 
+       /*
+        * Allow ordinary WAL segment creation before any exitArchiveRecovery(),
+        * which sometimes creates a segment, and after the last ReadRecord().
+        */
+       LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
+       XLogCtl->InstallXLogFileSegmentActive = true;
+       LWLockRelease(ControlFileLock);
+
        /*
         * Consider whether we need to assign a new timeline ID.
         *
@@ -12717,7 +12752,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
                                         */
                                        if (StandbyMode && CheckForStandbyTrigger())
                                        {
-                                               ShutdownWalRcv();
+                                               XLogShutdownWalRcv();
                                                return false;
                                        }
 
@@ -12765,7 +12800,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
                                         * WAL that we restore from archive.
                                         */
                                        if (WalRcvStreaming())
-                                               ShutdownWalRcv();
+                                               XLogShutdownWalRcv();
 
                                        /*
                                         * Before we sleep, re-scan for possible new timelines if
@@ -12895,7 +12930,7 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
                                         */
                                        if (pendingWalRcvRestart && !startWalReceiver)
                                        {
-                                               ShutdownWalRcv();
+                                               XLogShutdownWalRcv();
 
                                                /*
                                                 * Re-scan for possible new timelines if we were
@@ -12945,6 +12980,9 @@ WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
                                                                         tli, curFileTLI);
                                                }
                                                curFileTLI = tli;
+                                               LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
+                                               XLogCtl->InstallXLogFileSegmentActive = true;
+                                               LWLockRelease(ControlFileLock);
                                                RequestXLogStreaming(tli, ptr, PrimaryConnInfo,
                                                                                         PrimarySlotName,
                                                                                         wal_receiver_create_temp_slot);
@@ -13115,6 +13153,17 @@ StartupRequestWalReceiverRestart(void)
        }
 }
 
+/* Thin wrapper around ShutdownWalRcv(). */
+static void
+XLogShutdownWalRcv(void)
+{
+       ShutdownWalRcv();
+
+       LWLockAcquire(ControlFileLock, LW_EXCLUSIVE);
+       XLogCtl->InstallXLogFileSegmentActive = false;
+       LWLockRelease(ControlFileLock);
+}
+
 /*
  * Determine what log level should be used to report a corrupt WAL record
  * in the current WAL page, previously read by XLogPageRead().