From: dan <Dan Kennedy>
Date: Sat, 6 Nov 2021 16:36:30 +0000 (+0000)
Subject: Merge change [6979efbf07d93e7a] from trunk to this branch.
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f54bc90ca3113b1ceedd2b7968c2a61cb69ab266;p=thirdparty%2Fsqlite.git

Merge change [6979efbf07d93e7a] from trunk to this branch.

FossilOrigin-Name: 2bb2448d6042b8c1597aab53b2c1c1aa0cdf9b36f15ef5c44730558f213297da
---

f54bc90ca3113b1ceedd2b7968c2a61cb69ab266
diff --cc manifest
index 3ee3a4230d,7abb13825f..629222092a
--- a/manifest
+++ b/manifest
@@@ -1,5 -1,5 +1,5 @@@
- C Merge\schanges\sfrom\strunk\sinto\sthis\sbranch.
- D 2021-11-06T16:10:10.217
 -C Ensure\sthat\sthe\sWAL\scode\scorrectly\shandles\sall\spossible\soutcomes\sfrom\sthe\nwalIndexPage()\sroutine.
 -D 2021-10-28T00:09:31.896
++C Merge\schange\s[6979efbf07d93e7a]\sfrom\strunk\sto\sthis\sbranch.
++D 2021-11-06T16:36:30.666
  F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
  F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
  F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@@ -635,8 -633,8 +635,8 @@@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1
  F src/vdbevtab.c f99b275366c5fc5e2d99f734729880994ab9500bdafde7fae3b02d562b9d323c
  F src/vtab.c d07cc24dd84b0b51bf05adb187b0d2e6b0cac56cfbc0197995a26d4f8fa5c7e2
  F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
- F src/wal.c cf63450f720845d309b4c6050b59861c423c71fbc86c6dd853dfb5ce274ffa9e
 -F src/wal.c 6ae14d4797f31c67fc2be659d24fbc6e1a6f5f423bdfb5ef831ea171bce42320
 -F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
++F src/wal.c b4b8fa75897fe5229cc30725345db656e122b65c394dfc5d279bf49e422481ed
 +F src/wal.h d01234e828943e002040c22a7e017642962f9fd9b2dc142fa599769ae4e459e9
  F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
  F src/where.c ecabef93b0f14442a73eca205fc960428984d75fbdc7d60226b7fc9cac127187
  F src/whereInt.h 83877a75a1bce056ea44aff02f1dfa958ad1d6038c213ddadb8652003b45151d
@@@ -1941,7 -1929,8 +1941,7 @@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9
  F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
  F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
  F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
- P 7e2bc836f6aedfd69588f5723f8797f11ee6437d3b63ffc43c88d40e3baadb1c 7238d58051bfdcea8f7a4aeab89145849d0659c987df9063aacafe97be6657fe
- R 958432f8e903781491ddd95858915d61
 -P 7238d58051bfdcea8f7a4aeab89145849d0659c987df9063aacafe97be6657fe 12715c6b234a04627ca27e94bfa8bd456998360a9f0117480e0038f4747818d6
 -R fe9d125153f206ef4cb83374e184623d
 -T +closed 12715c6b234a04627ca27e94bfa8bd456998360a9f0117480e0038f4747818d6
 -U drh
 -Z b79ac0d17a59f6e7c71e03a206543e27
++P 52667bce485354ee4fee87f19015845baef12adf2674127f8c6f1bac1ccf3b7d 6979efbf07d93e7afad508165df684dcc6fe33b91ca772397c8afa00d16d1a0d
++R d08ac1bd1a069bc290b94d8879d538c8
 +U dan
- Z b88f4550c8f1a7eb32fef38df24556c5
++Z cdeeb26f2bf3f011e17f433f6b261bdc
diff --cc manifest.uuid
index a37dc999d0,11f72edd9c..396fa91e54
--- a/manifest.uuid
+++ b/manifest.uuid
@@@ -1,1 -1,1 +1,1 @@@
- 52667bce485354ee4fee87f19015845baef12adf2674127f8c6f1bac1ccf3b7d
 -6979efbf07d93e7afad508165df684dcc6fe33b91ca772397c8afa00d16d1a0d
++2bb2448d6042b8c1597aab53b2c1c1aa0cdf9b36f15ef5c44730558f213297da
diff --cc src/wal.c
index 76594b86ae,4c37560798..1e0f60e71b
--- a/src/wal.c
+++ b/src/wal.c
@@@ -1418,9 -1102,11 +1425,10 @@@ static void walCleanupHash(Wal *pWal)
    }
    
    /* Zero the entries in the aPgno array that correspond to frames with
 -  ** frame numbers greater than pWal->hdr.mxFrame. 
 -  */
 +  ** frame numbers greater than pWal->hdr.mxFrame.  */
-   nByte = (int)((char *)sLoc.aHash - (char *)&sLoc.aPgno[iLimit+1]);
-   memset((void *)&sLoc.aPgno[iLimit+1], 0, nByte);
+   nByte = (int)((char *)sLoc.aHash - (char *)&sLoc.aPgno[iLimit]);
+   assert( nByte>=0 );
+   memset((void *)&sLoc.aPgno[iLimit], 0, nByte);
  
  #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT
    /* Verify that the every entry in the mapping region is still reachable
@@@ -1529,212 -1208,6 +1537,212 @@@ static int walIndexAppend(Wal *pWal, in
    return rc;
  }
  
 +/*
 +** Recover a single wal file - *-wal if iWal==0, or *-wal2 if iWal==1.
 +*/
 +static int walIndexRecoverOne(Wal *pWal, int iWal, u32 *pnCkpt, int *pbZero){
 +  i64 nSize;                      /* Size of log file */
 +  u32 aFrameCksum[2] = {0, 0};
 +  int rc;
 +  sqlite3_file *pWalFd = pWal->apWalFd[iWal];
 +
 +  assert( iWal==0 || iWal==1 );
 +
 +  memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
 +  sqlite3_randomness(8, pWal->hdr.aSalt);
 +
 +  rc = sqlite3OsFileSize(pWalFd, &nSize);
 +  if( rc==SQLITE_OK ){
 +    if( nSize>WAL_HDRSIZE ){
 +      u8 aBuf[WAL_HDRSIZE];         /* Buffer to load WAL header into */
 +      u32 *aPrivate = 0;            /* Heap copy of *-shm pg being populated */
 +      u8 *aFrame = 0;               /* Malloc'd buffer to load entire frame */
 +      int szFrame;                  /* Number of bytes in buffer aFrame[] */
 +      u8 *aData;                    /* Pointer to data part of aFrame buffer */
 +      int szPage;                   /* Page size according to the log */
 +      u32 magic;                    /* Magic value read from WAL header */
 +      u32 version;                  /* Magic value read from WAL header */
 +      int isValid;                  /* True if this frame is valid */
 +      int iPg;                      /* Current 32KB wal-index page */
 +      int iLastFrame;               /* Last frame in wal, based on size alone */
 +      int iLastPg;                  /* Last shm page used by this wal */
 +  
 +      /* Read in the WAL header. */
 +      rc = sqlite3OsRead(pWalFd, aBuf, WAL_HDRSIZE, 0);
 +      if( rc!=SQLITE_OK ){
 +        return rc;
 +      }
 +  
 +      /* If the database page size is not a power of two, or is greater than
 +      ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid 
 +      ** data. Similarly, if the 'magic' value is invalid, ignore the whole
 +      ** WAL file.
 +      */
 +      magic = sqlite3Get4byte(&aBuf[0]);
 +      szPage = sqlite3Get4byte(&aBuf[8]);
 +      if( (magic&0xFFFFFFFE)!=WAL_MAGIC 
 +       || szPage&(szPage-1) 
 +       || szPage>SQLITE_MAX_PAGE_SIZE 
 +       || szPage<512 
 +      ){
 +        return SQLITE_OK;
 +      }
 +      pWal->hdr.bigEndCksum = (u8)(magic&0x00000001);
 +      pWal->szPage = szPage;
 +  
 +      /* Verify that the WAL header checksum is correct */
 +      walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, 
 +          aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum
 +      );
 +      if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24])
 +       || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28])
 +      ){
 +        return SQLITE_OK;
 +      }
 +  
 +      memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);
 +      *pnCkpt = sqlite3Get4byte(&aBuf[12]);
 +  
 +      /* Verify that the version number on the WAL format is one that
 +      ** are able to understand */
 +      version = sqlite3Get4byte(&aBuf[4]);
 +      if( version!=WAL_VERSION1 && version!=WAL_VERSION2 ){
 +        return SQLITE_CANTOPEN_BKPT;
 +      }
 +      pWal->hdr.iVersion = version;
 +  
 +      /* Malloc a buffer to read frames into. */
 +      szFrame = szPage + WAL_FRAME_HDRSIZE;
 +      aFrame = (u8 *)sqlite3_malloc64(szFrame + WALINDEX_PGSZ);
 +      if( !aFrame ){
 +        return SQLITE_NOMEM_BKPT;
 +      }
 +      aData = &aFrame[WAL_FRAME_HDRSIZE];
 +      aPrivate = (u32*)&aData[szPage];
 +  
 +      /* Read all frames from the log file. */
 +      iLastFrame = (nSize - WAL_HDRSIZE) / szFrame;
 +      if( version==WAL_VERSION2 ){
 +        iLastPg = walFramePage2(iWal, iLastFrame);
 +      }else{
 +        iLastPg = walFramePage(iLastFrame);
 +      }
 +      for(iPg=iWal; iPg<=iLastPg; iPg+=(version==WAL_VERSION2 ? 2 : 1)){
 +        u32 *aShare;
 +        int iFrame;                 /* Index of last frame read */
 +        int iLast;
 +        int iFirst;
 +        int nHdr, nHdr32;
 +
 +        rc = walIndexPage(pWal, iPg, (volatile u32**)&aShare);
-         if( rc ) break;
++        assert( aShare!=0 || rc!=SQLITE_OK );
++        if( aShare==0 ) break;
 +        pWal->apWiData[iPg] = aPrivate;
 +
 +        if( iWal ){
 +          assert( version==WAL_VERSION2 );
 +          iFirst = 1 + (iPg/2)*HASHTABLE_NPAGE;
 +          iLast = iFirst + HASHTABLE_NPAGE - 1;
 +        }else{
 +          int i2 = (version==WAL_VERSION2) ? (iPg/2) : iPg;
 +          iLast = HASHTABLE_NPAGE_ONE+i2*HASHTABLE_NPAGE;
 +          iFirst = 1 + (i2==0?0:HASHTABLE_NPAGE_ONE+(i2-1)*HASHTABLE_NPAGE);
 +        }
 +        iLast = MIN(iLast, iLastFrame);
 +
 +        for(iFrame=iFirst; iFrame<=iLast; iFrame++){
 +          i64 iOffset = walFrameOffset(iFrame, szPage);
 +          u32 pgno;                 /* Database page number for frame */
 +          u32 nTruncate;            /* dbsize field from frame header */
 +
 +          /* Read and decode the next log frame. */
 +          rc = sqlite3OsRead(pWalFd, aFrame, szFrame, iOffset);
 +          if( rc!=SQLITE_OK ) break;
 +          isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame);
 +          if( !isValid ) break;
 +          rc = walIndexAppend(pWal, iWal, iFrame, pgno);
 +          if( NEVER(rc!=SQLITE_OK) ) break;
 +  
 +          /* If nTruncate is non-zero, this is a commit record. */
 +          if( nTruncate ){
 +            pWal->hdr.mxFrame = iFrame;
 +            pWal->hdr.nPage = nTruncate;
 +            pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16));
 +            testcase( szPage<=32768 );
 +            testcase( szPage>=65536 );
 +            aFrameCksum[0] = pWal->hdr.aFrameCksum[0];
 +            aFrameCksum[1] = pWal->hdr.aFrameCksum[1];
 +          }
 +        }
 +        pWal->apWiData[iPg] = aShare;
 +        nHdr = (iPg==0 ? WALINDEX_HDR_SIZE : 0);
 +        nHdr32 = nHdr / sizeof(u32);
 +#ifndef SQLITE_SAFER_WALINDEX_RECOVERY
 +        /* Memcpy() should work fine here, on all reasonable implementations.
 +        ** Technically, memcpy() might change the destination to some
 +        ** intermediate value before setting to the final value, and that might
 +        ** cause a concurrent reader to malfunction.  Memcpy() is allowed to
 +        ** do that, according to the spec, but no memcpy() implementation that
 +        ** we know of actually does that, which is why we say that memcpy()
 +        ** is safe for this.  Memcpy() is certainly a lot faster.
 +        */
 +        memcpy(&aShare[nHdr32], &aPrivate[nHdr32], WALINDEX_PGSZ-nHdr);
 +#else
 +        /* In the event that some platform is found for which memcpy()
 +        ** changes the destination to some intermediate value before
 +        ** setting the final value, this alternative copy routine is
 +        ** provided.
 +        */
 +        {
 +          int i;
 +          for(i=nHdr32; i<WALINDEX_PGSZ/sizeof(u32); i++){
 +            if( aShare[i]!=aPrivate[i] ){
 +              /* Atomic memory operations are not required here because if
 +              ** the value needs to be changed, that means it is not being
 +              ** accessed concurrently. */
 +              aShare[i] = aPrivate[i];
 +            }
 +          }
 +        }
 +#endif
 +        if( iFrame<=iLast ) break;
 +      }
 +  
 +      sqlite3_free(aFrame);
 +    }else if( pbZero ){
 +      *pbZero = 1;
 +    }
 +  }
 +
 +  pWal->hdr.aFrameCksum[0] = aFrameCksum[0];
 +  pWal->hdr.aFrameCksum[1] = aFrameCksum[1];
 +
 +  return rc;
 +}
 +
 +static int walOpenWal2(Wal *pWal){
 +  int rc = SQLITE_OK;
 +  if( !isOpen(pWal->apWalFd[1]) ){
 +    int f = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL);
 +    rc = sqlite3OsOpen(pWal->pVfs, pWal->zWalName2, pWal->apWalFd[1], f, &f);
 +  }
 +  return rc;
 +}
 +
 +static int walTruncateWal2(Wal *pWal){
 +  int bIs;
 +  int rc;
 +  assert( !isOpen(pWal->apWalFd[1]) );
 +  rc = sqlite3OsAccess(pWal->pVfs, pWal->zWalName2, SQLITE_ACCESS_EXISTS, &bIs);
 +  if( rc==SQLITE_OK && bIs ){
 +    rc = walOpenWal2(pWal);
 +    if( rc==SQLITE_OK ){
 +      rc = sqlite3OsTruncate(pWal->apWalFd[1], 0);
 +      sqlite3OsClose(pWal->apWalFd[1]);
 +    }
 +  }
 +  return rc;
 +}
  
  /*
  ** Recover the wal-index by reading the write-ahead log file. 
@@@ -2315,19 -1822,9 +2323,18 @@@ static int walIteratorInit
        int j;                      /* Counter variable */
        int nEntry;                 /* Number of entries in this segment */
        ht_slot *aIndex;            /* Sorted index for this segment */
 +      u32 iZero;
  
 -      if( (i+1)==nSegment ){
 -        nEntry = (int)(iLast - sLoc.iZero);
 +      if( iMode==2 ){
 +        walExternalDecode(sLoc.iZero+1, &iZero);
 +        iZero--;
 +        assert( iZero==0 || i>=2 );
 +      }else{
 +        iZero = sLoc.iZero;
 +      }
 +
-       sLoc.aPgno++;
 +      if( i==iLastSeg ){
 +        nEntry = (int)(iLast - iZero);
        }else{
          nEntry = (int)((u32*)sLoc.aHash - (u32*)sLoc.aPgno);
        }
@@@ -3763,94 -3116,10 +3771,94 @@@ int sqlite3WalBeginReadTransaction(Wal 
  */
  void sqlite3WalEndReadTransaction(Wal *pWal){
    sqlite3WalEndWriteTransaction(pWal);
 -  if( pWal->readLock>=0 ){
 +  if( pWal->readLock!=WAL_LOCK_NONE ){
      walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
 -    pWal->readLock = -1;
 +    pWal->readLock = WAL_LOCK_NONE;
 +  }
 +}
 +
 +/* Search hash table iHash for an entry matching page number
 +** pgno. Each call to this function searches a single hash table
 +** (each hash table indexes up to HASHTABLE_NPAGE frames).
 +**
 +** This code might run concurrently to the code in walIndexAppend()
 +** that adds entries to the wal-index (and possibly to this hash 
 +** table). This means the value just read from the hash 
 +** slot (aHash[iKey]) may have been added before or after the 
 +** current read transaction was opened. Values added after the
 +** read transaction was opened may have been written incorrectly -
 +** i.e. these slots may contain garbage data. However, we assume
 +** that any slots written before the current read transaction was
 +** opened remain unmodified.
 +**
 +** For the reasons above, the if(...) condition featured in the inner
 +** loop of the following block is more stringent that would be required 
 +** if we had exclusive access to the hash-table:
 +**
 +**   (aPgno[iFrame]==pgno): 
 +**     This condition filters out normal hash-table collisions.
 +**
 +**   (iFrame<=iLast): 
 +**     This condition filters out entries that were added to the hash
 +**     table after the current read-transaction had started.
 +*/
 +static int walSearchHash(
 +  Wal *pWal, 
 +  u32 iLast,
 +  int iHash, 
 +  Pgno pgno, 
 +  u32 *piRead
 +){
 +  WalHashLoc sLoc;                /* Hash table location */
 +  int iKey;                       /* Hash slot index */
 +  int nCollide;                   /* Number of hash collisions remaining */
 +  int rc;                         /* Error code */
 +
 +  rc = walHashGet(pWal, iHash, &sLoc);
 +  if( rc!=SQLITE_OK ){
 +    return rc;
    }
 +  nCollide = HASHTABLE_NSLOT;
 +  for(iKey=walHash(pgno); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){
 +    u32 iFrame = sLoc.aHash[iKey] + sLoc.iZero;
 +    if( iFrame<=iLast 
 +     && iFrame>=pWal->minFrame 
-      && sLoc.aPgno[sLoc.aHash[iKey]]==pgno 
++     && sLoc.aPgno[sLoc.aHash[iKey]-1]==pgno 
 +    ){
 +      assert( iFrame>*piRead || CORRUPT_DB );
 +      *piRead = iFrame;
 +    }
 +    if( (nCollide--)==0 ){
 +      return SQLITE_CORRUPT_BKPT;
 +    }
 +  }
 +
 +  return SQLITE_OK;
 +}
 +
 +static int walSearchWal(
 +  Wal *pWal, 
 +  int iWal, 
 +  Pgno pgno, 
 +  u32 *piRead
 +){
 +  int rc = SQLITE_OK;
 +  int bWal2 = isWalMode2(pWal);
 +  u32 iLast = walidxGetMxFrame(&pWal->hdr, iWal);
 +  if( iLast ){
 +    int iHash;
 +    int iMinHash = walFramePage(pWal->minFrame);
 +    u32 iExternal = bWal2 ? walExternalEncode(iWal, iLast) : iLast;
 +    assert( bWal2==0 || pWal->minFrame==0 );
 +    for(iHash=walFramePage(iExternal); 
 +        iHash>=iMinHash && *piRead==0; 
 +        iHash-=(1+bWal2)
 +    ){
 +      rc = walSearchHash(pWal, iExternal, iHash, pgno, piRead);
 +      if( rc!=SQLITE_OK ) break;
 +    }
 +  }
 +  return rc;
  }
  
  /*