First attempt at making features work together. Only the most minimal testing

so far.

FossilOrigin-Name: fd707001f0afb1cf32cfeeda3ec7b5622eb49ddedf8fec1a7aa4c8841c77c37a

diff --git a/manifest b/manifest
index dd9771b7e3f699cee8d8532279403f8ef67585c9..dc2ae521ba52baa347010f77cf139532ae097fce 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sthe\swal2\sand\sbegin-concurrent\scode.\sBoth\sfeatures\swork,\sbut\snot\sat\sthe\nsame\stime.
-D 2018-12-03T20:49:34.015
+C First\sattempt\sat\smaking\sfeatures\swork\stogether.\sOnly\sthe\smost\sminimal\stesting\nso\sfar.
+D 2018-12-04T19:41:07.389
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c
@@ -593,7 +593,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7
 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392
 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 88e424b4de9cb836166a967b5b8a6de7f90cdd493dd1d04a1b855fa58dbe7212
+F src/wal.c 4b7ad0e623e01ede1ecb2cae24548c29e51ce5a21b2fd91af26b4bb447b72cc4
 F src/wal.h b42fc8081cd1765d4d4dd99b33f2db2f71128f4e25ff8c08d1a346f5af62e27a
 F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66
 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e
@@ -1606,6 +1606,7 @@ F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bd
 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
 F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477
 F test/wal2.test a225bafac35a47765b890bacdeb57e5e81039f21cc18a1e8ce88eb76e56b843c
+F test/wal2concurrent.test d0af64c3b113993553acfeeb64d207fb6d924e498fd953dbf31040a566ec10df
 F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c
 F test/wal2simple.test 8c9dfb8f1bca01a0deb57f7074cdb83865c2292e89b13f7a51a1c160dca3f5f4
 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2
@@ -1795,10 +1796,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 480be916c840e9e28010c22cf29a8396502c9e6387f31f750260c6290f58e9a1
-R 41c3c1de1e62db08c33d5cf08c11006b
-T *branch * begin-concurrent-wal2
-T *sym-begin-concurrent-wal2 *
-T -sym-begin-concurrent *
+P b7281a1caa574870a071bea3e96b1d8210c28c17f9094449b3ce1a42b311e6a1
+R db024f31bd5bbdadf08ded59e2f54e6f
 U dan
-Z f5d7b930a621f73c3ef64e6a734bda83
+Z abc3921392e51c645845b41c65979599

diff --git a/manifest.uuid b/manifest.uuid
index a398ac8e878de68e5291c8eb773120287089646d..e66f70d089cb7295c5df6dc10e0afd17377b4bf1 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b7281a1caa574870a071bea3e96b1d8210c28c17f9094449b3ce1a42b311e6a1
\ No newline at end of file
+fd707001f0afb1cf32cfeeda3ec7b5622eb49ddedf8fec1a7aa4c8841c77c37a
\ No newline at end of file

diff --git a/src/wal.c b/src/wal.c
index 5b50c99dfae87c14d5903ab33e6b2b2b7c31da81..3c0e73967d969a7a2f8f074f09da0138da247c2e 100644 (file)
--- a/src/wal.c
+++ b/src/wal.c
@@ -2850,7 +2850,7 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
   ** even if some external agent does a "chmod" to make the shared-memory
   ** writable by us, until sqlite3OsShmUnmap() has been called.
   ** This is a requirement on the VFS implementation.
-   */
+  */
   rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy);
   assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */
   if( rc!=SQLITE_READONLY_CANTINIT ){
@@ -3531,23 +3531,32 @@ int sqlite3WalFindFrame(
   /* This routine is only be called from within a read transaction. */
   assert( pWal->readLock!=WAL_LOCK_NONE );
 
-  /* If this is a wal2 system, the client must have a partial-wal lock 
-  ** on wal file iApp. Or if it is a wal system, iApp==0 must be true.  */
-  assert( bWal2==0 || iApp==1
-       || pWal->readLock==WAL_LOCK_PART1 || pWal->readLock==WAL_LOCK_PART1_FULL2
-  );
-  assert( bWal2==0 || iApp==0
-       || pWal->readLock==WAL_LOCK_PART2 || pWal->readLock==WAL_LOCK_PART2_FULL1
-  );
-  assert( bWal2 || iApp==0 );
+  /* If this is a regular wal system, then iApp must be set to 0 (there is
+  ** only one wal file, after all). Or, if this is a wal2 system and the
+  ** write-lock is not held, the client must have a partial-wal lock on wal 
+  ** file iApp. This is not always true if the write-lock is held and this
+  ** function is being called after WalLockForCommit() as part of committing
+  ** a CONCURRENT transaction.  */
+#ifdef SQLITE_DEBUG
+  if( bWal2 ){
+    if( pWal->writeLock==0 ){
+      int l = pWal->readLock;
+      assert( iApp==1 || l==WAL_LOCK_PART1 || l==WAL_LOCK_PART1_FULL2 );
+      assert( iApp==0 || l==WAL_LOCK_PART2 || l==WAL_LOCK_PART2_FULL1 );
+    }
+  }else{
+    assert( iApp==0 );
+  }
+#endif
 
   /* Return early if read-lock 0 is held. */
   if( (pWal->readLock==0 && pWal->bShmUnreliable==0) ){
+    assert( !bWal2 );
     *piRead = 0;
     return SQLITE_OK;
   }
 
-  /* Search the wal file that the client holds a partial lock on first */
+  /* Search the wal file that the client holds a partial lock on first. */
   rc = walSearchWal(pWal, iApp, pgno, &iRead);
 
   /* If the requested page was not found, no error has occured, and 
@@ -3556,6 +3565,10 @@ int sqlite3WalFindFrame(
   if( rc==SQLITE_OK && bWal2 && iRead==0 && (
         pWal->readLock==WAL_LOCK_PART1_FULL2 
      || pWal->readLock==WAL_LOCK_PART2_FULL1
+#ifndef SQLITE_OMIT_CONCURRENT
+     || (pWal->readLock==WAL_LOCK_PART1 && iApp==1)
+     || (pWal->readLock==WAL_LOCK_PART2 && iApp==0)
+#endif
   )){
     rc = walSearchWal(pWal, !iApp, pgno, &iRead);
   }
@@ -3784,67 +3797,104 @@ int sqlite3WalLockForCommit(
       ** was opened.  */
       rc = SQLITE_BUSY_SNAPSHOT;
     }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){
+      int bWal2 = isWalMode2(pWal);
       int iHash;
       int iLastHash = walFramePage(head.mxFrame);
-      u32 iFirst = pWal->hdr.mxFrame+1;     /* First wal frame to check */
-      if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){
-        assert( pWal->readLock==0 );
-        iFirst = 1;
-      }
-      for(iHash=walFramePage(iFirst); iHash<=iLastHash; iHash++){
-        WalHashLoc sLoc;
+      int nLoop = 1+(bWal2 && walidxGetFile(&head)!=walidxGetFile(&pWal->hdr));
+      int iLoop;
+      
+
+      assert( nLoop==1 || nLoop==2 );
+      for(iLoop=0; iLoop<nLoop && rc==SQLITE_OK; iLoop++){
+        u32 iFirst;               /* First (external) wal frame to check */
+        u32 iLastHash;            /* Last hash to check this loop */
+        u32 mxFrame;              /* Last (external) wal frame to check */
+
+        if( bWal2==0 ){
+          assert( iLoop==0 );
+          /* Special case for wal mode. If this concurrent transaction was
+          ** opened after the entire wal file had been checkpointed, and
+          ** another connection has since wrapped the wal file, then we wish to
+          ** iterate through every frame in the new wal file - not just those
+          ** that follow the current value of pWal->hdr.mxFrame (which will be
+          ** set to the size of the old, now overwritten, wal file). This
+          ** doesn't come up in wal2 mode, as in wal2 mode the client always
+          ** has a PART lock on one of the wal files, preventing it from being
+          ** checkpointed or overwritten. */
+          iFirst = pWal->hdr.mxFrame+1;
+          if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){
+            assert( pWal->readLock==0 );
+            iFirst = 1;
+          }
+          mxFrame = head.mxFrame;
+        }else{
+          int iA = walidxGetFile(&pWal->hdr);
+          if( iLoop==0 ){
+            iFirst = walExternalEncode(iA, 1+walidxGetMxFrame(&pWal->hdr, iA));
+            mxFrame = walExternalEncode(iA, walidxGetMxFrame(&head, iA));
+          }else{
+            iFirst = walExternalEncode(!iA, 1);
+            mxFrame = walExternalEncode(!iA, walidxGetMxFrame(&head, !iA));
+          }
+        }
+        iLastHash = walFramePage(mxFrame);
 
-        rc = walHashGet(pWal, iHash, &sLoc);
-        if( rc==SQLITE_OK ){
-          u32 i, iMin, iMax;
-          assert( head.mxFrame>=sLoc.iZero );
-          iMin = (sLoc.iZero >= iFirst) ? 1 : (iFirst - sLoc.iZero);
-          iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE;
-          if( iMax>(head.mxFrame-sLoc.iZero) ) iMax = (head.mxFrame-sLoc.iZero);
-          for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){
-            PgHdr *pPg;
-            if( sLoc.aPgno[i]==1 ){
-              /* Check that the schema cookie has not been modified. If
-              ** it has not, the commit can proceed. */
-              u8 aNew[4];
-              u8 *aOld = &((u8*)pPage1->pData)[40];
-              int sz;
-              i64 iOffset;
-              sz = pWal->hdr.szPage;
-              sz = (sz&0xfe00) + ((sz&0x0001)<<16);
-              iOffset = walFrameOffset(i+sLoc.iZero, sz) + WAL_FRAME_HDRSIZE+40;
-              rc = sqlite3OsRead(pWal->apWalFd[0], aNew, sizeof(aNew), iOffset);
-              if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){
+        for(iHash=walFramePage(iFirst); iHash<=iLastHash; iHash += (1+bWal2)){
+          WalHashLoc sLoc;
+
+          rc = walHashGet(pWal, iHash, &sLoc);
+          if( rc==SQLITE_OK ){
+            u32 i, iMin, iMax;
+            assert( mxFrame>=sLoc.iZero );
+            iMin = (sLoc.iZero >= iFirst) ? 1 : (iFirst - sLoc.iZero);
+            iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE;
+            if( iMax>(mxFrame-sLoc.iZero) ) iMax = (mxFrame-sLoc.iZero);
+            for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){
+              PgHdr *pPg;
+              if( sLoc.aPgno[i]==1 ){
+                /* Check that the schema cookie has not been modified. If
+                ** it has not, the commit can proceed. */
+                u8 aNew[4];
+                u8 *aOld = &((u8*)pPage1->pData)[40];
+                int sz;
+                i64 iOffset;
+                sz = pWal->hdr.szPage;
+                sz = (sz&0xfe00) + ((sz&0x0001)<<16);
+                iOffset = walFrameOffset(i+sLoc.iZero, sz)+WAL_FRAME_HDRSIZE+40;
+                rc = sqlite3OsRead(pWal->apWalFd[0], aNew,sizeof(aNew),iOffset);
+                if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){
+                  rc = SQLITE_BUSY_SNAPSHOT;
+                }
+              }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i]) ){
+                *piConflict = sLoc.aPgno[i];
                 rc = SQLITE_BUSY_SNAPSHOT;
-              }
-            }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i]) ){
-              *piConflict = sLoc.aPgno[i];
-              rc = SQLITE_BUSY_SNAPSHOT;
-            }else if( (pPg = sqlite3PagerLookup(pPager, sLoc.aPgno[i])) ){
-              /* Page aPgno[i], which is present in the pager cache, has been
-              ** modified since the current CONCURRENT transaction was started.
-              ** However it was not read by the current transaction, so is not
-              ** a conflict. There are two possibilities: (a) the page was
-              ** allocated at the of the file by the current transaction or 
-              ** (b) was present in the cache at the start of the transaction.
-              **
-              ** For case (a), do nothing. This page will be moved within the
-              ** database file by the commit code to avoid the conflict. The
-              ** call to PagerUnref() is to release the reference grabbed by
-              ** the sqlite3PagerLookup() above.  
-              **
-              ** In case (b), drop the page from the cache - otherwise
-              ** following the snapshot upgrade the cache would be inconsistent
-              ** with the database as stored on disk. */
-              if( sqlite3PagerIswriteable(pPg) ){
-                sqlite3PagerUnref(pPg);
-              }else{
-                sqlite3PcacheDrop(pPg);
+              }else if( (pPg = sqlite3PagerLookup(pPager, sLoc.aPgno[i])) ){
+                /* Page aPgno[i], which is present in the pager cache, has been
+                ** modified since the current CONCURRENT transaction was
+                ** started.  However it was not read by the current
+                ** transaction, so is not a conflict. There are two
+                ** possibilities: (a) the page was allocated at the of the file
+                ** by the current transaction or (b) was present in the cache
+                ** at the start of the transaction.
+                **
+                ** For case (a), do nothing. This page will be moved within the
+                ** database file by the commit code to avoid the conflict. The
+                ** call to PagerUnref() is to release the reference grabbed by
+                ** the sqlite3PagerLookup() above.  
+                **
+                ** In case (b), drop the page from the cache - otherwise
+                ** following the snapshot upgrade the cache would be
+                ** inconsistent with the database as stored on disk. */
+                if( sqlite3PagerIswriteable(pPg) ){
+                  sqlite3PagerUnref(pPg);
+                }else{
+                  sqlite3PcacheDrop(pPg);
+                }
               }
             }
           }
+          if( rc!=SQLITE_OK ) break;
         }
-        if( rc!=SQLITE_OK ) break;
       }
     }
   }
@@ -3876,6 +3926,7 @@ int sqlite3WalUpgradeSnapshot(Wal *pWal){
   ** any reads performed between now and committing the transaction will
   ** read from the old snapshot - not the one just upgraded to.  */
   if( pWal->readLock==0 && pWal->hdr.mxFrame!=walCkptInfo(pWal)->nBackfill ){
+    assert( isWalMode2(pWal)==0 );
     rc = walUpgradeReadlock(pWal);
   }
   return rc;
@@ -3959,7 +4010,6 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){
 */
 void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){
   int iWal = walidxGetFile(&pWal->hdr);
-  assert( pWal->writeLock );
   assert( isWalMode2(pWal) || iWal==0 );
   aWalData[0] = walidxGetMxFrame(&pWal->hdr, iWal);
   aWalData[1] = pWal->hdr.aFrameCksum[0];

diff --git a/test/wal2concurrent.test b/test/wal2concurrent.test
new file mode 100644 (file)
index 0000000..db0c848
--- /dev/null
+++ b/test/wal2concurrent.test
@@ -0,0 +1,84 @@
+# 2015 July 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set ::testprefix wal2concurrent
+
+ifcapable !concurrent {
+  finish_test
+  return
+}
+
+
+#-------------------------------------------------------------------------
+# Warm-body test.
+#
+sqlite3 db2 test.db
+do_execsql_test 1.0 {
+  PRAGMA page_size = 1024;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(y);
+  PRAGMA journal_size_limit = 5000;
+  PRAGMA journal_mode = wal2;
+} {5000 wal2}
+
+do_execsql_test 1.1 {
+  INSERT INTO t1 VALUES(1);
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(2);
+} {}
+
+do_test 1.2 {
+  execsql { 
+    PRAGMA journal_size_limit = 5000;
+    INSERT INTO t1 VALUES(3) 
+  } db2
+  catchsql { COMMIT }
+} {1 {database is locked}}
+
+do_catchsql_test 1.3 { COMMIT   } {1 {database is locked}}
+do_catchsql_test 1.4 { ROLLBACK } {0 {}}
+
+do_test 1.5 {
+  list [file size test.db-wal] [file size test.db-wal2]
+} {2128 0}
+
+do_execsql_test 1.6 {
+  BEGIN CONCURRENT;
+    INSERT INTO t1 VALUES(2);
+} {}
+
+do_test 1.7 {
+  execsql { INSERT INTO t2 VALUES(randomblob(4000)) } db2
+  list [file size test.db-wal] [file size test.db-wal2]
+} {7368 0}
+
+do_test 1.8 {
+  execsql { 
+    INSERT INTO t2 VALUES(1);
+    INSERT INTO t1 VALUES(5);
+  } db2
+  list [file size test.db-wal] [file size test.db-wal2]
+} {7368 2128}
+
+do_catchsql_test 1.9  { COMMIT   } {1 {database is locked}}
+do_catchsql_test 1.10 { ROLLBACK } {0 {}}
+
+db close
+sqlite3 db test.db
+do_execsql_test 1.11 { SELECT * FROM t1 } {1 3 5}
+do_execsql_test 1.12 { SELECT count(*) FROM t2 } {2}
+
+finish_test
+