Merge the wal2 feature into this branch.

FossilOrigin-Name: a2c618c0a45e1b1d271a4a981ee1e3b368dfcce94eda5c2d7d08f794784046d9

diff --cc manifest
index 735edc7a3f6b17cce17eedd2357874e921f52dff,4bc575194a060923d2bbc7da8767814776d7c736..fecfd96dc2ae174657fc88bd5a6a9ddbd43a2aa0
--- 1/manifest
--- 2/manifest
+++ b/manifest
@@@ -1,5 -1,5 +1,5 @@@
- C Merge\slatest\sbegin-concurrent\schanges\sinto\sthis\sbranch.
- D 2018-12-18T17:47:53.703
 -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch.
 -D 2018-12-18T17:32:08.747
++C Merge\sthe\swal2\sfeature\sinto\sthis\sbranch.
++D 2018-12-18T18:01:17.673
  F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
  F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
  F Makefile.in d8b254f8bb81bab43c340d70d17dc3babab40fcc8a348c8255881f780a45fee6
@@@ -504,14 -505,14 +505,14 @@@ F src/parse.y 8206217fe7fa96652aa1b8a79
  F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee
  F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586
  F src/pcache1.c ddc9fc7d9861cf3a1f30660264b76b1ae9e1dce5dbba085cf001d5cb6b41cf8c
- F src/pragma.c 96ce7dce4dc9cb2b7aa0e1b2ce7536870bdc00b10becc278245e775489447ea0
+ F src/pragma.c 3b23e9eb390888a4d2c71c86be6c710c34179657f4f039fae019ba660be00ca9
 -F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13
 +F src/pragma.h 7003ea8e45e5da0a7cd6d35846214f9ae9ecf5be66b268415ceea5855324af11
  F src/prepare.c 0e8fc0deaf36da104e08d07ce7d97bc09ab57d078b399381532fec3fa1d3f2bb
  F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381
 -F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 +F src/random.c f27af4099afaea7284ade5c206224dcfdb2334cfd119d018b470d46356b3f27d
  F src/resolve.c 72fe8cae7326b979e7258ab4c531956951e1a5f3fe8644c646abaec1b2eb6d95
  F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
 -F src/select.c 8c7317d5ee920516a56b8b4ca79fbfca70a1f8b52d67e884c808ea3a016c04e3
 +F src/select.c ca760497a7a0263b29ca4001760eac53de7c05b0b9edb12adeae1a1c44466779
  F src/shell.c.in 207da30342db0b6fac8b2487abd60b059a5ea80cc9494bd1db76a1dd4aae7cca
  F src/sqlite.h.in 846968c2880c2223e0d65c544698fedcf45bbddc52693a9b020519725690e1fd
  F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@@ -593,8 -594,8 +594,8 @@@ F src/vdbesort.c 90aad5a92608f2dd771c96
  F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392
  F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa
  F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
- F src/wal.c 59b20f8fd0d7420e8896a1ef7cbd5db7352f383235ec25db08d9c22b55f4b603
- F src/wal.h f325a5856b669f5ba449157485915816103857c8574efc746ac55eba3335c5e0
 -F src/wal.c 2640020e19d6d2f14ac02fdc468ea965388d956880221d4c8088f20077a85598
 -F src/wal.h c398e0269e8f37495cedb63b5e288c2aac6f6d103d05fb55f4affec21311615d
++F src/wal.c dca50fc1279f726751947aaa9334ad946094f55f6b0a850e70e58e41037f9433
++F src/wal.h 4ef31f6defdd12877eff7832eb5f56c1f898bc074ac5aab67a539f9c59cb8118
  F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66
  F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e
  F src/whereInt.h f125f29fca80890768e0b2caa14f95db74b2dacd3a122a168f97aa7b64d6968f
@@@ -1195,7 -1196,7 +1196,7 @@@ F test/pagesize.test 5769fc62d8c890a83a
  F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
  F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
  F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
- F test/permutations.test 4a991666f8840fe1c7cbf20d29ea75cdc76f4a6fc6b647a760924f8eb2f4e88b
 -F test/permutations.test 2a639a69a3ddee195b36cb5118e7a6ac2a1a0a320b08fdbb32239bf88189ec30
++F test/permutations.test d0f878173c36562b50898ecccc1a021f06dbb17c3b41ee5b6e620eb4c872a73c
  F test/pg_common.tcl 301ac19c1a52fd55166d26db929b3b89165c634d52b5f8ad76ea8cb06960db30
  F test/pragma.test c267bf02742c823a191960895b3d52933cebd7beee26757d1ed694f213fcd867
  F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
@@@ -1800,7 -1812,7 +1812,10 @@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9
  F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
  F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
  F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
- P f2083ee410b7504e8478f7373e76ded913e3a6a92cb7345b8c1ac27554f6edf8 123cbb3312917ad5b3c32556547c5b7e8ba4e2d2def8651ff80f9fc1bdc1875c
- R 121415d4c182782d4168d3f43c3faca4
 -P fa46fa3bfcd4b6d1d219db4179ce69d0edb0786a521acaa034fdb74b312274ff d64f248da3ce7762fe2c17fbc83f7bea9ffca73723bb3ad0982a85320839da90
 -R 8beca6502a060f919ffe486db5f7e63b
++P a93ca38b432e0c9fb2e00499cfc88d09f59f27f908f377d8ae99b6717e548a60 b3a163b46cad19a2e6a90cdd5b4c2e94d66e80b7ad2a0c9857580fe74748a604
++R 8df706f43d7960b329e6ae5fae0ee12e
++T *branch * begin-concurrent-pnu-wal2
++T *sym-begin-concurrent-pnu-wal2 *
++T -sym-begin-concurrent-pnu *
  U dan
- Z c5e975ad0db2f2825cbe9c283ff86d0a
 -Z a47178a1de728bf0e9a383c129165be6
++Z ca2e4b85418d60538bab17e132700aa3

diff --cc manifest.uuid
index fbd978a0c4a65197cf6ae2130af59fe526db0680,df45b7c9cea15279d9cf394b5526911e4b94ea3f..811cd2f0763e44828f341cfde01323733513cb14
--- 1/manifest.uuid
--- 2/manifest.uuid
+++ b/manifest.uuid
@@@ -1,1 -1,1 +1,1 @@@
- a93ca38b432e0c9fb2e00499cfc88d09f59f27f908f377d8ae99b6717e548a60
 -b3a163b46cad19a2e6a90cdd5b4c2e94d66e80b7ad2a0c9857580fe74748a604
++a2c618c0a45e1b1d271a4a981ee1e3b368dfcce94eda5c2d7d08f794784046d9

diff --cc src/wal.c
index 4b22ba42ef347634f990bf7d5e49ea5aebe3c4f5,b7e2391041cf22d54876d5fbe5ef6d41f05c8a5c..5e9ca3691fafbd05541f0cef1d4dc4b767cae4f6
--- 1/src/wal.c
--- 2/src/wal.c
+++ b/src/wal.c
@@@ -473,8 -707,8 +707,9 @@@ struct Wal 
    u32 iReCksum;              /* On commit, recalculate checksums from here */
    u32 nPriorFrame;           /* For sqlite3WalInfo() */
    const char *zWalName;      /* Name of WAL file */
+   char *zWalName2;           /* Name of second WAL file */
    u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
 +  FastPrng sPrng;            /* Random number generator */
  #ifdef SQLITE_DEBUG
    u8 lockError;              /* True if a locking error has occurred */
  #endif
@@@ -1122,6 -1425,148 +1426,148 @@@ static int walIndexAppend(Wal *pWal, in
    return rc;
  }
  
 -  sqlite3_randomness(8, pWal->hdr.aSalt);
+ /*
+ ** Recover a single wal file - *-wal if iWal==0, or *-wal2 if iWal==1.
+ */
+ static int walIndexRecoverOne(Wal *pWal, int iWal, u32 *pnCkpt, int *pbZero){
+   i64 nSize;                      /* Size of log file */
+   u32 aFrameCksum[2] = {0, 0};
+   int rc;
+   sqlite3_file *pWalFd = pWal->apWalFd[iWal];
+ 
+   assert( iWal==0 || iWal==1 );
+ 
+   memset(&pWal->hdr, 0, sizeof(WalIndexHdr));
++  sqlite3FastRandomness(&pWal->sPrng, 8, pWal->hdr.aSalt);
+ 
+   rc = sqlite3OsFileSize(pWalFd, &nSize);
+   if( rc==SQLITE_OK ){
+     if( nSize>WAL_HDRSIZE ){
+       u8 aBuf[WAL_HDRSIZE];         /* Buffer to load WAL header into */
+       u8 *aFrame = 0;               /* Malloc'd buffer to load entire frame */
+       int szFrame;                  /* Number of bytes in buffer aFrame[] */
+       u8 *aData;                    /* Pointer to data part of aFrame buffer */
+       int iFrame;                   /* Index of last frame read */
+       i64 iOffset;                  /* Next offset to read from log file */
+       int szPage;                   /* Page size according to the log */
+       u32 magic;                    /* Magic value read from WAL header */
+       u32 version;                  /* Magic value read from WAL header */
+       int isValid;                  /* True if this frame is valid */
+   
+       /* Read in the WAL header. */
+       rc = sqlite3OsRead(pWalFd, aBuf, WAL_HDRSIZE, 0);
+       if( rc!=SQLITE_OK ){
+         return rc;
+       }
+   
+       /* If the database page size is not a power of two, or is greater than
+       ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid 
+       ** data. Similarly, if the 'magic' value is invalid, ignore the whole
+       ** WAL file.
+       */
+       magic = sqlite3Get4byte(&aBuf[0]);
+       szPage = sqlite3Get4byte(&aBuf[8]);
+       if( (magic&0xFFFFFFFE)!=WAL_MAGIC 
+        || szPage&(szPage-1) 
+        || szPage>SQLITE_MAX_PAGE_SIZE 
+        || szPage<512 
+       ){
+         return SQLITE_OK;
+       }
+       pWal->hdr.bigEndCksum = (u8)(magic&0x00000001);
+       pWal->szPage = szPage;
+   
+       /* Verify that the WAL header checksum is correct */
+       walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, 
+           aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum
+       );
+       if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24])
+        || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28])
+       ){
+         return SQLITE_OK;
+       }
+   
+       memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);
+       *pnCkpt = sqlite3Get4byte(&aBuf[12]);
+   
+       /* Verify that the version number on the WAL format is one that
+       ** are able to understand */
+       version = sqlite3Get4byte(&aBuf[4]);
+       if( version!=WAL_VERSION1 && version!=WAL_VERSION2 ){
+         return SQLITE_CANTOPEN_BKPT;
+       }
+       pWal->hdr.iVersion = version;
+   
+       /* Malloc a buffer to read frames into. */
+       szFrame = szPage + WAL_FRAME_HDRSIZE;
+       aFrame = (u8 *)sqlite3_malloc64(szFrame);
+       if( !aFrame ){
+         return SQLITE_NOMEM_BKPT;
+       }
+       aData = &aFrame[WAL_FRAME_HDRSIZE];
+   
+       /* Read all frames from the log file. */
+       iFrame = 0;
+       for(iOffset=WAL_HDRSIZE; (iOffset+szFrame)<=nSize; iOffset+=szFrame){
+         u32 pgno;                   /* Database page number for frame */
+         u32 nTruncate;              /* dbsize field from frame header */
+   
+         /* Read and decode the next log frame. */
+         iFrame++;
+         rc = sqlite3OsRead(pWalFd, aFrame, szFrame, iOffset);
+         if( rc!=SQLITE_OK ) break;
+         isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame);
+         if( !isValid ) break;
+         rc = walIndexAppend(pWal, iWal, iFrame, pgno);
+         if( rc!=SQLITE_OK ) break;
+   
+         /* If nTruncate is non-zero, this is a commit record. */
+         if( nTruncate ){
+           pWal->hdr.mxFrame = iFrame;
+           pWal->hdr.nPage = nTruncate;
+           pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16));
+           testcase( szPage<=32768 );
+           testcase( szPage>=65536 );
+           aFrameCksum[0] = pWal->hdr.aFrameCksum[0];
+           aFrameCksum[1] = pWal->hdr.aFrameCksum[1];
+         }
+       }
+   
+       sqlite3_free(aFrame);
+     }else if( pbZero ){
+       *pbZero = 1;
+     }
+   }
+ 
+   pWal->hdr.aFrameCksum[0] = aFrameCksum[0];
+   pWal->hdr.aFrameCksum[1] = aFrameCksum[1];
+ 
+   return rc;
+ }
+ 
+ static int walOpenWal2(Wal *pWal){
+   int rc = SQLITE_OK;
+   if( !isOpen(pWal->apWalFd[1]) ){
+     int f = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL);
+     rc = sqlite3OsOpen(pWal->pVfs, pWal->zWalName2, pWal->apWalFd[1], f, &f);
+   }
+   return rc;
+ }
+ 
+ static int walTruncateWal2(Wal *pWal){
+   int bIs;
+   int rc;
+   assert( !isOpen(pWal->apWalFd[1]) );
+   rc = sqlite3OsAccess(pWal->pVfs, pWal->zWalName2, SQLITE_ACCESS_EXISTS, &bIs);
+   if( rc==SQLITE_OK && bIs ){
+     rc = walOpenWal2(pWal);
+     if( rc==SQLITE_OK ){
+       rc = sqlite3OsTruncate(pWal->apWalFd[1], 0);
+       sqlite3OsClose(pWal->apWalFd[1]);
+     }
+   }
+   return rc;
+ }
  
  /*
  ** Recover the wal-index by reading the write-ahead log file. 
@@@ -1381,11 -1821,16 +1822,17 @@@ int sqlite3WalOpen
    pRet->syncHeader = 1;
    pRet->padToSectorBoundary = 1;
    pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE);
 +  sqlite3FastPrngInit(&pRet->sPrng);
+   pRet->bWal2 = bWal2;
  
-   /* Open file handle on the write-ahead log file. */
+   pRet->zWalName2 = (char*)pRet + sizeof(Wal) + 2*pVfs->szOsFile;
+   memcpy(pRet->zWalName2, zWalName, nWalName);
+   pRet->zWalName2[nWalName] = '2';
+   pRet->zWalName2[nWalName+1] = '\0';
+ 
+   /* Open a file handle on the first write-ahead log file. */
    flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL);
-   rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags);
+   rc = sqlite3OsOpen(pVfs, zWalName, pRet->apWalFd[0], flags, &flags);
    if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){
      pRet->readOnly = WAL_RDONLY;
    }

diff --cc src/wal.h
index 70247f1c2b43cf0a3c54cc2d57eafea811b16b51,7e700692a45cbd83159c7f3592d8c5ca954c6e51..c7f6c4b28c9789c219db45ccaba617eab1486f3c
--- 1/src/wal.h
--- 2/src/wal.h
+++ b/src/wal.h
@@@ -155,8 -156,11 +156,14 @@@ int sqlite3WalFramesize(Wal *pWal)
  /* Return the sqlite3_file object for the WAL file */
  sqlite3_file *sqlite3WalFile(Wal *pWal);
  
+ /* Return the journal mode (WAL or WAL2) used by this Wal object. */
+ int sqlite3WalJournalMode(Wal *pWal);
+ 
+ /* sqlite3_wal_info() data */
+ int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame);
+ 
 +/* sqlite3_wal_info() data */
 +int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame);
 +
  #endif /* ifndef SQLITE_OMIT_WAL */
  #endif /* SQLITE_WAL_H */

diff --cc test/permutations.test
Simple merge