From 72be10451e85d8a6508294b6f1f1c6e05c532721 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 17 Nov 2021 14:21:38 +0000 Subject: [PATCH] Apply the begin-concurrent-report patch to the wal2 branch. FossilOrigin-Name: 8eef87821357d1656d8f8dc08fd12ef0373f31fa1c5c11be7f4a95b173992a06 --- manifest | 49 +++-- manifest.uuid | 2 +- src/backup.c | 4 +- src/bitvec.c | 76 +++++++ src/btree.c | 572 ++++++++++++++++++++++++++++++++++++++++++++++-- src/btree.h | 18 ++ src/btreeInt.h | 1 + src/insert.c | 15 +- src/main.c | 4 + src/pager.c | 8 + src/pager.h | 5 +- src/sqlite.h.in | 3 + src/sqliteInt.h | 37 ++++ src/test1.c | 49 +++++ src/test2.c | 2 +- src/vdbe.c | 53 +++-- src/vdbeInt.h | 1 + src/vdbeapi.c | 1 + src/wherecode.c | 1 + 19 files changed, 839 insertions(+), 62 deletions(-) diff --git a/manifest b/manifest index d69d7f08ae..ae4e566bd0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\strunk\senhancements\sinto\sthe\sbegin-concurrent-pnu-wal2\sbranch. -D 2021-11-09T15:18:21.391 +C Apply\sthe\sbegin-concurrent-report\spatch\sto\sthe\swal2\sbranch. +D 2021-11-17T14:21:38.023 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -495,12 +495,12 @@ F src/alter.c 23743384e59f9d36df870ce41adfdf7934fd0adb619d7fa6fd1aac77c28a7533 F src/analyze.c 7518b99e07c5494111fe3bd867f28f804b6c5c1ad0703ec3d116de9bab3fa516 F src/attach.c e3f9d9a2a4a844750f3f348f37afb244535f21382cbfcd840152cb21cb41cfaf F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf -F src/backup.c 3014889fa06e20e6adfa0d07b60097eec1f6e5b06671625f476a714d2356513d -F src/bitvec.c 3907fcbe8a0c8c2db58d97087d15cdabbf2842adb9125df9ab9ff87d3db16775 +F src/backup.c 0d3f5004d5c85f257add00aee97687d40790a471a5d56306043ed9a7b53d6994 +F src/bitvec.c e67e4ebed581b5f0aa9155abd88177ed8b63eb40e9266b1483ce7f6eb9257477 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 5fa29b79523fbea1dc6b2259be137192b3032e296f2374a63b8980354c15d3a4 -F src/btree.h 900067641b64d619e6e2a93bd115c952a52f41d3bee32e551e2a4ceee05fc431 -F src/btreeInt.h 3f19f0be5af0b68cff55e58df4b11e7a0692d9e8a820ceaeba4084659a86cf28 +F src/btree.c 24c4e0e948ed819bbf09d22f2045705e3f824e59e9302d6227e7a098fed722c5 +F src/btree.h 1c1508834794c0d15fe15d23792cd42deefd8140ad782899eab258d081a0adfc +F src/btreeInt.h 614e75ff0bfdd26f4270ba178cd5a41f904aee7be203612fbe7fc2215fc83d9c F src/build.c f0cd84c6186df95fb679471ff8a52ecf6dba4df340891755b2e28a89128a732f F src/callback.c 106b585da1edd57d75fa579d823a5218e0bf37f191dbf7417eeb4a8a9a267dbc F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e @@ -518,10 +518,10 @@ F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 7fcbbe9114ac402ea3c0c6a3810f13fc89cae8131ea1659ec472be7caac10192 +F src/insert.c 5a9373456f86b024588e36169cfd151a58a42b8a77e4701e8c40fff766dcce92 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c e1dcff1c916bf6834e150b492eddda5d9792453182d2ad64294d2266b6e93c4c -F src/main.c 8dd12c04918305f5fb27bacbaca65505824d4eaf5e74605d6809362f6d23b020 +F src/main.c bd7befda8fde5f4b1fda67e0ef6c1db4b2a311c195a7cc5ac39405163a950c67 F src/malloc.c ef796bcc0e81d845d59a469f1cf235056caf9024172fd524e32136e65593647b F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -544,8 +544,8 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 7cc2a76b867cc3a7e9de7923e90485ee566cc06a60b2047231791ad769939c31 F src/os_win.c 77d39873836f1831a9b0b91894fec45ab0e9ca8e067dc8c549e1d1eca1566fe9 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 17e8d78f77faa53bd91f7ee148315135c867636ac182b08eb29ce37d1df9b802 -F src/pager.h 034a193e53d6649bdb641aa996e38094dbb7cbe365d8c10eba871a38a0f5ebb3 +F src/pager.c 92357f9557226458b77376f93fe7c7b90823e7c883ce4a509db69086c2506cde +F src/pager.h 915608bab08d0f233a8c8095ab24a16f5c16bda40877c74268ccd8ed0eb2a71f F src/parse.y 59631359574901cc5cd4780939a6740f6bc597bd473334e744c1a1c32d9adef3 F src/pcache.c 084e638432c610f95aea72b8509f0845d2791293f39d1b82f0c0a7e089c3bb6b F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 @@ -559,16 +559,16 @@ F src/resolve.c 4a1db4aadd802683db40ca2dbbb268187bd195f10cbdb7206dbd8ac988795571 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 1f188fd3aa15d8cb5e5397b8ba91102b3b9b2803cc50e5df770929aa82a34a1c F src/shell.c.in f8854bcb0d14707d661732698d5210d7f01694000c46e8014b323ad18f575be6 -F src/sqlite.h.in 32f7d112cb85d3f2fd56500e11a6bd6d19a969306650200dcde97c2638cfecfc +F src/sqlite.h.in 5c7c193579e84dde8b0f2a9842170e7ecc74ecadfd26503a66873edacbb5114c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839 -F src/sqliteInt.h d5e9880fb0b512d2e0b73192d7fdb147288c5c7865db8bc4bbf2a1b890e8c425 +F src/sqliteInt.h 22b0a54cc5027252065936723ae06e3790fb34e3a681438ce24ba3564dafbad9 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c 428e813dabf82804bc13196af35a0c3c6ef4347fe557fa6717c5c66bba6e8520 -F src/test1.c 2785b6fd05e4810f7215bf8de01c15670f39a19e49e3d927045f5e72ab08b4cd -F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 +F src/test1.c 1ab25d4bd0133cd4d6a182118dad8bb3fa5724c23596216febb370c164e8ec23 +F src/test2.c cb988be1ee1b972dc471e6b076087cfa9cb9ce5c2aa31e98ecf41c3256a72a53 F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644 F src/test4.c 7c4420e01c577b5c4add2cb03119743b1a357543d347773b9e717195ea967159 F src/test5.c 328aae2c010c57a9829d255dc099d6899311672d @@ -629,10 +629,10 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 30df8356e231dad33be10bb27897655002668343280004ba28c734489414a167 F src/vacuum.c 72867c740476d13f1c397015e4d3168b4e96a237a80b9afa67e1bb8500bfeeab -F src/vdbe.c c403995aed4a9293f10e53b111aebcfdb9c3349e442c7df337cec083f5b7cdae +F src/vdbe.c 5c59b21df2dec8fbdef6a97e9448bf812b90e79f07e6548f079c48f6ba32b707 F src/vdbe.h 25dabb25c7e157b84e59260cfb5b466c3ac103ede9f36f4db371332c47601abe -F src/vdbeInt.h 31fbabdc1ed61d9695337dfe5269ea94e1cf615c17f5cafeaa1bb01066820bab -F src/vdbeapi.c f4bd14b42d9717a35baac5f20c0c4bfccbf3691d64dc6ec02fc0ef19374d3c92 +F src/vdbeInt.h 2f0878748f6cea86736ea3e035266afd54439303645fe62a7c21f40fee95e424 +F src/vdbeapi.c b302a21521c33e10efee0fffd6e1381145f1c7e1af17a478d9df58600295869e F src/vdbeaux.c ec21bd1475c69543d961b241b51ad4321342ae1ec1d64969063efbb3db7d23a0 F src/vdbeblob.c 292e96c01c4219fca71d74e1002906d43eb232af4bd83f7552a3faec741f3eb8 F src/vdbemem.c a3d91dc9bb9ef725db77e4e9de7e1acef43192c9f8406c307665d503e3c2837c @@ -646,7 +646,7 @@ F src/wal.h 7a733af13b966ecb81872ce397e862116b3575ea53245b90b139a2873ee87825 F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c de0d4ff409c7b62a8803f9f267cc2c7fedddbc00de9ab7b5382c507383c18665 F src/whereInt.h 83877a75a1bce056ea44aff02f1dfa958ad1d6038c213ddadb8652003b45151d -F src/wherecode.c 1f5b62f46d284c8886945eb7438415bc27e23e87bb60b9ee468fa6bd31268f33 +F src/wherecode.c 5c85a71ff79257c36a253fdf3b754ab080ffad8b29db3202e3b307d9a9550181 F src/whereexpr.c 17bdbf4f5b490e70a18635498f0b910a558f953a9bf80af7f19cbde6e60e6825 F src/window.c 5d3b397b0c026d0ff5890244ac41359e524c01ae31e78782e1ff418c3e271a9e F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1963,7 +1963,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1958e6facaaca8e695ae3d7e79acdba0025d6221653397ea45f9b8daa56c8d9b 0894f59569378c41207b6cef43dbff9a4873582709d954c809c18f42d918aa03 -R d51c1503abf10965a7f6b01b91b3b4a0 -U drh -Z 909ec6a0986c60628db195c4c35452d9 +P 07bc13395de3e45fec2cd571f1c9d0776e198768a4a0dd00479347ca8058a003 +R 4c825608d3187912cbc1c1e3ae86f450 +T *branch * begin-concurrent-report-wal2 +T *sym-begin-concurrent-report-wal2 * +T -sym-begin-concurrent-pnu-wal2 * +U dan +Z 222663a8be03443702b34e9ef2c4cbea diff --git a/manifest.uuid b/manifest.uuid index bb085dc380..b4436e7e28 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07bc13395de3e45fec2cd571f1c9d0776e198768a4a0dd00479347ca8058a003 \ No newline at end of file +8eef87821357d1656d8f8dc08fd12ef0373f31fa1c5c11be7f4a95b173992a06 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 2b286de6bd..62ff3b91d9 100644 --- a/src/backup.c +++ b/src/backup.c @@ -503,7 +503,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ } } if( rc==SQLITE_OK ){ - rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + rc = sqlite3PagerCommitPhaseOne(pDestPager, p->pDest, 0, 1); } /* Write the extra pages and truncate the database file as required */ @@ -532,7 +532,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ } }else{ sqlite3PagerTruncateImage(pDestPager, nDestTruncate); - rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0); + rc = sqlite3PagerCommitPhaseOne(pDestPager, p->pDest, 0, 0); } /* Finish committing the transaction to the destination database. */ diff --git a/src/bitvec.c b/src/bitvec.c index 9dcf8340d8..526d4f4a3f 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -299,6 +299,82 @@ u32 sqlite3BitvecSize(Bitvec *p){ return p->iSize; } +static int bitvecCount(Bitvec *p){ + int nRet = 0; + if( p ){ + if( p->iDivisor ){ + int i; + for(i=0; iu.apSub[i]); + } + }else if( p->iSize<=BITVEC_NBIT ){ + int i; + for(i=0; iu.aBitmap[i]; + assert( sizeof(x)==1 ); + if( x & 0x80 ) nRet++; + if( x & 0x40 ) nRet++; + if( x & 0x20 ) nRet++; + if( x & 0x10 ) nRet++; + if( x & 0x08 ) nRet++; + if( x & 0x04 ) nRet++; + if( x & 0x02 ) nRet++; + if( x & 0x01 ) nRet++; + } + }else{ + nRet += p->nSet; + } + } + return nRet; +} + +static int bitvecArray(Bitvec *p, u32 *aElem, u32 iOff){ + int nRet = 0; + if( p ){ + if( p->iDivisor ){ + int i; + for(i=0; iu.apSub[i], &aElem[nRet], i*p->iDivisor + iOff); + } + }else if( p->iSize<=BITVEC_NBIT ){ + int i; + for(i=0; iu.aBitmap[i]; + assert( sizeof(x)==1 ); + if( x & 0x01 ){ aElem[nRet++] = iOff + i*8 + 0; } + if( x & 0x02 ){ aElem[nRet++] = iOff + i*8 + 1; } + if( x & 0x04 ){ aElem[nRet++] = iOff + i*8 + 2; } + if( x & 0x08 ){ aElem[nRet++] = iOff + i*8 + 3; } + if( x & 0x10 ){ aElem[nRet++] = iOff + i*8 + 4; } + if( x & 0x20 ){ aElem[nRet++] = iOff + i*8 + 5; } + if( x & 0x40 ){ aElem[nRet++] = iOff + i*8 + 6; } + if( x & 0x80 ){ aElem[nRet++] = iOff + i*8 + 7; } + } + }else{ + int i; + for(i=0; iu.aHash[i] ){ + aElem[nRet++] = p->u.aHash[i]+iOff-1; + } + } + assert( nRet==p->nSet ); + } + } + return nRet; +} + +int sqlite3BitvecArray(Bitvec *p, u32 **pa, int *pn){ + int nElem; + u32 *aElem; + nElem = bitvecCount(p); + aElem = sqlite3MallocZero(nElem*sizeof(u32)); + if( aElem==0 ) return SQLITE_NOMEM; + bitvecArray(p, aElem, 1); + *pa = aElem; + *pn = nElem; + return SQLITE_OK; +} + #ifndef SQLITE_UNTESTABLE /* ** Let V[] be an array of unsigned characters sufficient to hold diff --git a/src/btree.c b/src/btree.c index 0d8c983518..3dc9a4550a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4509,7 +4509,7 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zSuperJrnl){ rc = btreeFixUnlocked(p); } if( rc==SQLITE_OK ){ - rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zSuperJrnl, 0); + rc = sqlite3PagerCommitPhaseOne(pBt->pPager, p, zSuperJrnl, 0); } sqlite3BtreeLeave(p); } @@ -4994,6 +4994,20 @@ void sqlite3BtreeCursorZero(BtCursor *p){ memset(p, 0, offsetof(BtCursor, BTCURSOR_FIRST_UNINIT)); } +void sqlite3BtreeScanDeref(CursorScan *pScan){ + if( pScan ){ + pScan->nRef--; + assert( pScan->nRef>=0 ); + if( pScan->nRef==0 ){ + sqlite3KeyInfoUnref(pScan->pKeyInfo); + sqlite3_free(pScan->aMax); + sqlite3_free(pScan->aMin); + sqlite3_free(pScan->aLimit); + sqlite3_free(pScan); + } + } +} + /* ** Close a cursor. The read lock on the database file is released ** when the last cursor is closed. @@ -5020,6 +5034,8 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){ unlockBtreeIfUnused(pBt); sqlite3_free(pCur->aOverflow); sqlite3_free(pCur->pKey); + sqlite3BtreeScanDeref(pCur->pCScan); + pCur->pCScan = 0; if( (pBt->openFlags & BTREE_SINGLE) && pBt->pCursor==0 ){ /* Since the BtShared is not sharable, there is no need to ** worry about the missing sqlite3BtreeLeave() call here. */ @@ -5904,6 +5920,523 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ return rc; } +#include "vdbeInt.h" + +static void btreeScanFormatKey( + sqlite3 *db, + StrAccum *pAcc, + u8 *aKey, + i64 nKey +){ + if( aKey==0 ){ + sqlite3_str_appendf(pAcc, "%lld", nKey); + }else{ + int ii; + KeyInfo ki; + UnpackedRecord rec; + Mem aMem[6]; + memset(&ki, 0, sizeof(ki)); + memset(&rec, 0, sizeof(rec)); + memset(aMem, 0, sizeof(aMem)); + + ki.db = db; + ki.enc = ENC(db); + ki.nKeyField = ArraySize(aMem); + rec.aMem = aMem; + rec.nField = ArraySize(aMem); + + sqlite3VdbeRecordUnpack(&ki, nKey, aKey, &rec); + sqlite3_str_appendf(pAcc, "{"); + for(ii=0; iiflags & MEM_Null ){ + sqlite3_str_appendf(pAcc, "NULL"); + }else + if( pMem->flags & MEM_Int ){ + sqlite3_str_appendf(pAcc, "%lld", pMem->u.i); + }else + if( pMem->flags & MEM_Real ){ + sqlite3_str_appendf(pAcc, "%f", pMem->u.r); + }else + if( pMem->flags & MEM_Str ){ + sqlite3_str_appendf(pAcc, "%.*Q", pMem->n, pMem->z); + }else{ + int jj; + sqlite3_str_appendf(pAcc, "X'"); + for(jj=0; jjn; jj++){ + sqlite3_str_appendf(pAcc, "%.2X", (u8)pMem->z[jj]); + } + sqlite3_str_appendf(pAcc, "'"); + } + if( ii!=rec.nField-1 ){ + sqlite3_str_appendf(pAcc, ","); + } + } + sqlite3_str_appendf(pAcc, "}"); + } +} + +/* +** Compare scan keys (aKey1/nKey1) and (aKey2/nKey2). Return an integer +** less than 0, equal to 0, or greater than 0 if key1 is less than, equal +** to or greater than key2. i.e. a value with the same sign and zeroness +** as: +** +** (aKey1/nKey1) - (aKey2/nKey2) +*/ +static int btreeScanCompare( + sqlite3 *db, + CursorScan *pScan, + const u8 *aKey1, i64 nKey1, + const u8 *aKey2, i64 nKey2 +){ + int res = 0; + assert( (aKey1==0)==(aKey2==0) ); + if( aKey1 ){ + UnpackedRecord rec; + Mem aMem[32]; + memset(&rec, 0, sizeof(rec)); + memset(aMem, 0, sizeof(aMem)); + + rec.aMem = aMem; + rec.nField = pScan->pKeyInfo->nAllField; + rec.pKeyInfo = pScan->pKeyInfo; + sqlite3VdbeRecordUnpack(pScan->pKeyInfo, nKey2, aKey2, &rec); + res = sqlite3VdbeRecordCompare(nKey1, aKey1, &rec); + }else{ + if( nKey1nKey2 ) res = +1; + } + return res; +} + +void sqlite3_begin_concurrent_report_enable(sqlite3 *db, int bEnable){ + db->bConcurrentReport = bEnable; +} + +const char *sqlite3_begin_concurrent_report(sqlite3 *db){ + sqlite3DbFree(db, db->zBCReport); + db->zBCReport = 0; + if( db->pCScanList ){ + CursorScan *p; + StrAccum accum; + sqlite3StrAccumInit(&accum, db, 0, 0, 64*1024*1024); + for(p=db->pCScanList; p; p=p->pNext){ + + if( p->flags & (CURSORSCAN_PGWRITE|CURSORSCAN_PGFAIL) ){ + u32 *aPg = (u32*)p->aMax; + int nPg = p->iMax; + int i; + if( aPg ){ + sqlite3_str_appendf(&accum, "W:{%d", (int)aPg[0]); + for(i=1; iaMin; + nPg = p->iMin; + sqlite3_str_appendf(&accum, "%s:{%d", (p->aMax?"R":"F"), (int)aPg[0]); + for(i=1; iflags & CURSORSCAN_WRITE ){ + sqlite3_str_appendf(&accum, "%d<-(%lld)", p->tnum, p->iMin); + btreeScanFormatKey(db, &accum, p->aLimit, p->iLimit); + sqlite3_str_appendf(&accum, "\n"); + }else{ + if( p->flags & CURSORSCAN_LIMITVALID ){ + if( p->flags & CURSORSCAN_LIMITMAX ){ + if( (p->flags & CURSORSCAN_MAXVALID)==0 + || btreeScanCompare(db,p,p->aLimit,p->iLimit,p->aMax,p->iMax)<=0 + ){ + sqlite3_free(p->aMax); + p->iMax = p->iLimit; + p->aMax = p->aLimit; + p->aLimit = 0; + p->flags &= ~CURSORSCAN_MAXINCL; + p->flags |= (p->flags&CURSORSCAN_LIMITINCL)?CURSORSCAN_MAXINCL:0; + p->flags |= CURSORSCAN_MAXVALID; + } + }else{ + if( (p->flags & CURSORSCAN_MINVALID)==0 + || btreeScanCompare(db,p,p->aLimit,p->iLimit,p->aMin,p->iMin)>=0 + ){ + sqlite3_free(p->aMin); + p->iMin = p->iLimit; + p->aMin = p->aLimit; + p->aLimit = 0; + p->flags &= ~CURSORSCAN_MININCL; + p->flags |= (p->flags&CURSORSCAN_LIMITINCL)?CURSORSCAN_MININCL:0; + p->flags |= CURSORSCAN_MINVALID; + } + } + p->flags &= ~CURSORSCAN_LIMITVALID; + } + + sqlite3_str_appendf(&accum, "%d:%s", p->tnum, + (p->flags & CURSORSCAN_MININCL) ? "[" : "(" + ); + if( p->flags & CURSORSCAN_MINVALID ){ + btreeScanFormatKey(db, &accum, p->aMin, p->iMin); + }else{ + sqlite3_str_appendf(&accum, "EOF"); + } + sqlite3_str_appendf(&accum, ".."); + if( p->flags & CURSORSCAN_MAXVALID ){ + btreeScanFormatKey(db, &accum, p->aMax, p->iMax); + }else{ + sqlite3_str_appendf(&accum, "EOF"); + } + sqlite3_str_appendf(&accum, "%s\n", + (p->flags & CURSORSCAN_MAXINCL) ? "]" : ")" + ); + } + } + db->zBCReport = sqlite3StrAccumFinish(&accum); + } + return db->zBCReport; +} + +static u32 btreeScanSerialType(Mem *pMem){ + if( pMem->flags & MEM_Int ) return 6; + if( pMem->flags & MEM_Real ) return 7; + if( pMem->flags & MEM_Str ) return (pMem->n * 2)+13; + if( pMem->flags & MEM_Blob ) return (pMem->n * 2)+12; + return 0; +} + +static void btreeScanSet( + CursorScan *pNew, + UnpackedRecord *pKey, + i64 iKey, + i64 *piKey, + u8 **paKey +){ + if( pKey==0 ){ + assert( *paKey==0 ); + *piKey = iKey; + }else{ + int ii; + u8 *aRec = 0; + int nByte = 0; + int nHdr = 0; + int nSize = 0; + + sqlite3_free(*paKey); + *paKey = 0; + + for(ii=0; iinField; ii++){ + Mem *pMem = &pKey->aMem[ii]; + u32 serial_type = btreeScanSerialType(pMem); + nByte += sqlite3VdbeSerialTypeLen(serial_type); + nHdr += sqlite3VarintLen(serial_type); + } + + nSize = sqlite3VarintLen(nHdr); + if( sqlite3VarintLen(nSize+nHdr)>nSize ) nSize++; + nHdr += nSize; + + aRec = (u8*)sqlite3_malloc(nHdr+nByte); + if( aRec==0 ){ + pNew->flags |= CURSORSCAN_OOM; + }else{ + int iOff = 0; + iOff += sqlite3PutVarint(&aRec[iOff], nHdr); + for(ii=0; iinField; ii++){ + u32 serial_type = btreeScanSerialType(&pKey->aMem[ii]); + iOff += sqlite3PutVarint(&aRec[iOff], serial_type); + } + for(ii=0; iinField; ii++){ + Mem *pMem = &pKey->aMem[ii]; + u32 serial_type = btreeScanSerialType(pMem); + iOff += sqlite3VdbeSerialPut(&aRec[iOff], pMem, serial_type); + } + assert( iOff==(nHdr+nByte) ); + *paKey = aRec; + *piKey = iOff; + } + } +} + +static void btreeScanSetKey(BtCursor *pCsr, i64 *piKey, u8 **paKey){ + if( pCsr->curIntKey ){ + *piKey = sqlite3BtreeIntegerKey(pCsr); + }else{ + int rc; + u32 nKey = sqlite3BtreePayloadSize(pCsr); + u8 *aKey = sqlite3_malloc(nKey); + if( aKey==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3BtreePayload(pCsr, 0, nKey, aKey); + } + if( rc ){ + sqlite3_free(aKey); + pCsr->pCScan->flags |= CURSORSCAN_OOM; + }else{ + sqlite3_free(*paKey); + *piKey = nKey; + *paKey = aKey; + } + } +} + +static void btreeScanNext(BtCursor *pCsr, int bPrev){ + CursorScan *pCScan = pCsr->pCScan; + if( pCScan ){ + if( bPrev ){ + if( sqlite3BtreeEof(pCsr) ){ + pCScan->flags &= ~(CURSORSCAN_MINVALID|CURSORSCAN_MININCL); + }else{ + btreeScanSetKey(pCsr, &pCScan->iMin, &pCScan->aMin); + pCScan->flags |= CURSORSCAN_MINVALID|CURSORSCAN_MININCL; + } + }else{ + if( sqlite3BtreeEof(pCsr) ){ + pCScan->flags &= ~(CURSORSCAN_MAXVALID|CURSORSCAN_MAXINCL); + }else{ + btreeScanSetKey(pCsr, &pCScan->iMax, &pCScan->aMax); + pCScan->flags |= CURSORSCAN_MAXVALID|CURSORSCAN_MAXINCL; + } + } + } +} + +int sqlite3BtreeScanLimit( + BtCursor *pCsr, + UnpackedRecord *pKey, + i64 iKey, + int opcode +){ + CursorScan *pScan = pCsr->pCScan; + if( pScan && (pScan->flags & CURSORSCAN_LIMITVALID)==0 ){ + btreeScanSet(pScan, pKey, iKey, &pScan->iLimit, &pScan->aLimit); + pScan->flags |= CURSORSCAN_LIMITVALID; + switch( opcode ){ + case OP_IdxLT: + case OP_Lt: + pScan->flags |= CURSORSCAN_LIMITINCL; + case OP_IdxLE: + case OP_Le: + break; + + case OP_IdxGT: + case OP_Gt: + pScan->flags |= CURSORSCAN_LIMITINCL; + case OP_IdxGE: + case OP_Ge: + pScan->flags |= CURSORSCAN_LIMITMAX; + break; + } + } + return SQLITE_OK; +} + +static void btreeScanCopy( + CursorScan *p, + i64 *piOut, u8 **apOut, + i64 iIn, u8 *aIn +){ + *piOut = iIn; + if( aIn ){ + (*apOut) = (u8*)sqlite3_malloc(iIn); + if( 0==(*apOut) ){ + p->flags |= CURSORSCAN_OOM; + }else{ + memcpy(*apOut, aIn, iIn); + } + } +} + +int sqlite3BtreeScanStart( + BtCursor *pCsr, + UnpackedRecord *pKey, + i64 iKey, + int opcode, + int eqOnly +){ + Btree *pBtree = pCsr->pBtree; + sqlite3 *db = pBtree->db; + if( db->bConcurrentReport + && db->eConcurrent + && db->aDb[0].pBt==pBtree + && sqlite3PagerIsWal(pBtree->pBt->pPager) + ){ + CursorScan *pNew; + pNew = (CursorScan*)sqlite3MallocZero(sizeof(CursorScan)); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->pKeyInfo = sqlite3KeyInfoRef(pCsr->pKeyInfo); + pNew->tnum = (int)pCsr->pgnoRoot; + + if( pCsr->pCScan ){ + sqlite3BtreeScanDeref(pCsr->pCScan); + } + pCsr->pCScan = pNew; + pNew->pNext = db->pCScanList; + db->pCScanList = pNew; + pNew->nRef = 2; + switch( opcode ){ + case OP_Rewind: + btreeScanNext(pCsr, 0); + break; + + case OP_Last: + btreeScanNext(pCsr, 1); + break; + + case OP_SeekLE: + pNew->flags |= CURSORSCAN_MAXINCL; + case OP_SeekLT: + pNew->flags |= CURSORSCAN_MAXVALID; + btreeScanSet(pNew, pKey, iKey, &pNew->iMax, &pNew->aMax); + if( eqOnly ){ + btreeScanCopy( + pNew, &pNew->iLimit, &pNew->aLimit, pNew->iMax, pNew->aMax + ); + pNew->flags |= CURSORSCAN_LIMITVALID|CURSORSCAN_LIMITINCL; + }else{ + btreeScanNext(pCsr, 1); + } + break; + + case OP_SeekGE: + pNew->flags |= CURSORSCAN_MININCL; + case OP_SeekGT: + pNew->flags |= CURSORSCAN_MINVALID; + btreeScanSet(pNew, pKey, iKey, &pNew->iMin, &pNew->aMin); + if( eqOnly ){ + btreeScanCopy( + pNew, &pNew->iLimit, &pNew->aLimit, pNew->iMin, pNew->aMin + ); + pNew->flags |= CURSORSCAN_LIMITVALID|CURSORSCAN_LIMITINCL; + pNew->flags |= CURSORSCAN_LIMITMAX; + }else{ + btreeScanNext(pCsr, 0); + } + break; + + case OP_SeekRowid: + case OP_NotExists: + case OP_DeferredSeek: + assert( pKey==0 ); + pNew->iMin = pNew->iMax = iKey; + pNew->flags |= (CURSORSCAN_MININCL|CURSORSCAN_MAXINCL); + pNew->flags |= CURSORSCAN_MINVALID|CURSORSCAN_MAXVALID; + break; + } + } + return SQLITE_OK; +} + +void sqlite3BtreeScanDerefList(CursorScan *pList){ + CursorScan *p; + CursorScan *pNext; + for(p=pList; p; p=pNext){ + pNext = p->pNext; + sqlite3BtreeScanDeref(p); + } +} + +int sqlite3BtreeScanDirty(Btree *pBtree, Bitvec *pRead, PgHdr *pList){ + int rc = SQLITE_OK; + sqlite3 *db = pBtree->db; + if( db->bConcurrentReport + && db->eConcurrent + && db->aDb[0].pBt==pBtree + ){ + CursorScan *pNew; + u32 *aPg = 0; + PgHdr *p; + int nPg = 0; + + assert( sqlite3PagerIsWal(pBtree->pBt->pPager) ); + + pNew = (CursorScan*)sqlite3MallocZero(sizeof(CursorScan)); + if( pNew==0 ) return SQLITE_NOMEM; + for(p=pList; p; p=p->pDirty) nPg++; + if( pList ){ + aPg = sqlite3MallocZero(sizeof(u32*)*nPg); + } + + if( pList && aPg==0 ){ + rc = SQLITE_NOMEM; + }else{ + int nMin; + nPg = 0; + for(p=pList; p; p=p->pDirty){ + aPg[nPg++] = p->pgno; + } + pNew->iMax = nPg; + pNew->aMax = (u8*)aPg; + + rc = sqlite3BitvecArray(pRead, (u32**)&pNew->aMin, &nMin); + pNew->iMin = (i64)nMin; + } + + pNew->nRef = 1; + pNew->flags |= pList ? CURSORSCAN_PGWRITE : CURSORSCAN_PGFAIL; + pNew->pNext = pBtree->db->pCScanList; + pBtree->db->pCScanList = pNew; + } + return rc; +} + +int sqlite3BtreeScanWrite( + BtCursor *pCsr, + int op, + i64 rowid, + const u8 *a, + int n +){ + int rc = SQLITE_OK; + Btree *pBtree = pCsr->pBtree; + sqlite3 *db = pBtree->db; + if( db->bConcurrentReport + && db->eConcurrent + && db->aDb[0].pBt==pBtree + && sqlite3PagerIsWal(pBtree->pBt->pPager) + ){ + sqlite3 *db = pCsr->pBtree->db; + CursorScan *pNew; + pNew = (CursorScan*)sqlite3MallocZero(sizeof(CursorScan)); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->tnum = (int)pCsr->pgnoRoot; + pNew->pNext = db->pCScanList; + db->pCScanList = pNew; + pNew->nRef = 1; + pNew->flags = CURSORSCAN_WRITE; + pNew->iMin = (pCsr->curIntKey ? rowid : 0); + + if( a ){ + pNew->aLimit = sqlite3_malloc(n); + if( pNew->aLimit==0 ){ + pNew->flags |= CURSORSCAN_OOM; + rc = SQLITE_NOMEM; + }else{ + memcpy(pNew->aLimit, a, n); + pNew->iLimit = n; + } + }else{ + pNew->iLimit = sqlite3BtreePayloadSize(pCsr); + pNew->aLimit = sqlite3_malloc(pNew->iLimit); + if( pNew->aLimit==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3BtreePayload(pCsr, 0, pNew->iLimit, pNew->aLimit); + } + if( rc ){ + pNew->flags |= CURSORSCAN_OOM; + } + } + } + return rc; +} + + /* Move the cursor so that it points to an entry in a table (a.k.a INTKEY) ** table near the key intKey. Return a success code. ** @@ -6384,23 +6917,29 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ } } int sqlite3BtreeNext(BtCursor *pCur, int flags){ + int rc; MemPage *pPage; UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ assert( cursorOwnsBtShared(pCur) ); assert( flags==0 || flags==1 ); pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); - if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur); - pPage = pCur->pPage; - if( (++pCur->ix)>=pPage->nCell ){ - pCur->ix--; - return btreeNext(pCur); - } - if( pPage->leaf ){ - return SQLITE_OK; + if( pCur->eState!=CURSOR_VALID ){ + rc = btreeNext(pCur); }else{ - return moveToLeftmost(pCur); + pPage = pCur->pPage; + if( (++pCur->ix)>=pPage->nCell ){ + pCur->ix--; + rc = btreeNext(pCur); + }else if( pPage->leaf ){ + rc = SQLITE_OK; + }else{ + rc = moveToLeftmost(pCur); + } } + + btreeScanNext(pCur, 0); + return rc; } /* @@ -6473,6 +7012,7 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){ return rc; } int sqlite3BtreePrevious(BtCursor *pCur, int flags){ + int rc = SQLITE_OK; assert( cursorOwnsBtShared(pCur) ); assert( flags==0 || flags==1 ); UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ @@ -6482,10 +7022,13 @@ int sqlite3BtreePrevious(BtCursor *pCur, int flags){ || pCur->ix==0 || pCur->pPage->leaf==0 ){ - return btreePrevious(pCur); + rc = btreePrevious(pCur); + }else{ + pCur->ix--; } - pCur->ix--; - return SQLITE_OK; + + btreeScanNext(pCur, 1); + return rc; } /* @@ -11353,6 +11896,9 @@ int sqlite3BtreeExclusiveLock(Btree *p){ #ifdef SQLITE_OMIT_CONCURRENT assert( pgno==0 ); #else + if( rc==SQLITE_BUSY_SNAPSHOT ){ + sqlite3PagerScanFailure(p, pBt->pPager); + } if( rc==SQLITE_BUSY_SNAPSHOT && pgno ){ PgHdr *pPg = 0; int rc2 = sqlite3PagerGet(pBt->pPager, pgno, &pPg, 0); diff --git a/src/btree.h b/src/btree.h index f736d523e7..c14d23b4e5 100644 --- a/src/btree.h +++ b/src/btree.h @@ -410,4 +410,22 @@ int sqlite3BtreeTransferRow(BtCursor*, BtCursor*, i64); # define sqlite3SchemaMutexHeld(X,Y,Z) 1 #endif +int sqlite3BtreeScanStart( + BtCursor *pCsr, + UnpackedRecord *pKey, + i64 iKey, + int opcode, + int eqOnly +); +int sqlite3BtreeScanLimit( + BtCursor *pCsr, + UnpackedRecord *pKey, + i64 iKey, + int opcode +); +int sqlite3BtreeScanWrite(BtCursor *pCsr, int op, i64 id, const u8 *a, int n); +typedef struct PgHdr PgHdr; +int sqlite3BtreeScanDirty(Btree *pBtree, Bitvec *pRead, PgHdr *pList); + + #endif /* SQLITE_BTREE_H */ diff --git a/src/btreeInt.h b/src/btreeInt.h index 2551baa1be..d5b4650d28 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -537,6 +537,7 @@ struct BtCursor { Btree *pBtree; /* The Btree to which this cursor belongs */ Pgno *aOverflow; /* Cache of overflow page locations */ void *pKey; /* Saved key that was cursor last known position */ + CursorScan *pCScan; /* All fields above are zeroed when the cursor is allocated. See ** sqlite3BtreeCursorZero(). Fields that follow must be manually ** initialized. */ diff --git a/src/insert.c b/src/insert.c index 97205cb2ae..578115bf55 100644 --- a/src/insert.c +++ b/src/insert.c @@ -2487,7 +2487,8 @@ static void codeWithoutRowidPreupdate( Parse *pParse, /* Parse context */ Table *pTab, /* Table being updated */ int iCur, /* Cursor number for table */ - int regData /* Data containing new record */ + int regData, /* Data containing new record */ + u16 flags ){ Vdbe *v = pParse->pVdbe; int r = sqlite3GetTempReg(pParse); @@ -2495,11 +2496,11 @@ static void codeWithoutRowidPreupdate( assert( 0==(pParse->db->mDbFlags & DBFLAG_Vacuum) || CORRUPT_DB ); sqlite3VdbeAddOp2(v, OP_Integer, 0, r); sqlite3VdbeAddOp4(v, OP_Insert, iCur, regData, r, (char*)pTab, P4_TABLE); - sqlite3VdbeChangeP5(v, OPFLAG_ISNOOP); + sqlite3VdbeChangeP5(v, flags); sqlite3ReleaseTempReg(pParse, r); } #else -# define codeWithoutRowidPreupdate(a,b,c,d) +# define codeWithoutRowidPreupdate(a,b,c,d,e) #endif /* @@ -2550,8 +2551,10 @@ void sqlite3CompleteInsertion( assert( pParse->nested==0 ); pik_flags |= OPFLAG_NCHANGE; pik_flags |= (update_flags & OPFLAG_SAVEPOSITION); - if( update_flags==0 ){ - codeWithoutRowidPreupdate(pParse, pTab, iIdxCur+i, aRegIdx[i]); + if( 1 || update_flags==0 ){ + codeWithoutRowidPreupdate(pParse, pTab, iIdxCur+i, aRegIdx[i], + OPFLAG_ISNOOP|(update_flags & OPFLAG_ISUPDATE) + ); } } sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i], @@ -3100,7 +3103,7 @@ static int xferOptimization( && !HasRowid(pDest) && IsPrimaryKeyIndex(pDestIdx) ){ - codeWithoutRowidPreupdate(pParse, pDest, iDest, regData); + codeWithoutRowidPreupdate(pParse, pDest, iDest, regData, OPFLAG_ISNOOP); } } sqlite3VdbeAddOp2(v, OP_IdxInsert, iDest, regData); diff --git a/src/main.c b/src/main.c index 3692ab73a2..d6173ee88d 100644 --- a/src/main.c +++ b/src/main.c @@ -1318,6 +1318,10 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ ** passed to sqlite3_close (meaning that it is a zombie). Therefore, ** go ahead and free all resources. */ + sqlite3BtreeScanDerefList(db->pCScanList); + db->pCScanList = 0; + sqlite3DbFree(db, db->zBCReport); + db->zBCReport = 0; /* If a transaction is open, roll it back. This also ensures that if ** any database schemas have been modified by an uncommitted transaction diff --git a/src/pager.c b/src/pager.c index a28e5121a0..15f13d9c54 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6463,6 +6463,10 @@ int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1, Pgno *piConflict){ } #ifndef SQLITE_OMIT_CONCURRENT +void sqlite3PagerScanFailure(Btree *pBt, Pager *pPager){ + sqlite3BtreeScanDirty(pBt, pPager->pAllRead, 0); +} + /* ** This function is called as part of committing an CONCURRENT transaction. ** At this point the wal WRITER lock is held, and all pages in the cache @@ -6536,6 +6540,7 @@ void sqlite3PagerDropExclusiveLock(Pager *pPager){ */ int sqlite3PagerCommitPhaseOne( Pager *pPager, /* Pager object */ + Btree *pBtree, const char *zSuper, /* If not NULL, the super-journal name */ int noSync /* True to omit the xSync on the db file */ ){ @@ -6583,6 +6588,9 @@ int sqlite3PagerCommitPhaseOne( if( ALWAYS(pList) ){ rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1); } + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeScanDirty(pBtree, pPager->pAllRead, pList); + } sqlite3PagerUnref(pPageOne); if( rc==SQLITE_OK ){ sqlite3PcacheCleanAll(pPager->pPCache); diff --git a/src/pager.h b/src/pager.h index 97659d4377..49d516ea88 100644 --- a/src/pager.h +++ b/src/pager.h @@ -42,6 +42,8 @@ typedef struct Pager Pager; */ typedef struct PgHdr DbPage; +typedef struct Btree Btree; + /* ** Page number PAGER_MJ_PGNO is never used in an SQLite database (it is ** reserved for working around a windows/posix incompatibility). It is @@ -178,7 +180,7 @@ void *sqlite3PagerGetExtra(DbPage *); /* Functions used to manage pager transactions and savepoints. */ void sqlite3PagerPagecount(Pager*, int*); int sqlite3PagerBegin(Pager*, int exFlag, int); -int sqlite3PagerCommitPhaseOne(Pager*,const char *zSuper, int); +int sqlite3PagerCommitPhaseOne(Pager*,Btree*,const char *zSuper, int); int sqlite3PagerExclusiveLock(Pager*, DbPage *pPage1, Pgno*); int sqlite3PagerSync(Pager *pPager, const char *zSuper); int sqlite3PagerCommitPhaseTwo(Pager*); @@ -249,6 +251,7 @@ void sqlite3PagerDropExclusiveLock(Pager*); int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage*); void sqlite3PagerSetDbsize(Pager *pPager, Pgno); int sqlite3PagerIsWal(Pager*); +void sqlite3PagerScanFailure(Btree *pBt, Pager *pPager); #else # define sqlite3PagerEndConcurrent(x) #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 7574c99ac9..a046e582e3 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -10083,6 +10083,9 @@ int sqlite3_deserialize( #define SQLITE_DESERIALIZE_RESIZEABLE 2 /* Resize using sqlite3_realloc64() */ #define SQLITE_DESERIALIZE_READONLY 4 /* Database is read-only */ +const char *sqlite3_begin_concurrent_report(sqlite3*); +void sqlite3_begin_concurrent_report_enable(sqlite3 *db, int bEnable); + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0c13dafea8..a09d5c19a6 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1513,6 +1513,37 @@ void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); #endif /* SQLITE_OMIT_DEPRECATED */ #define SQLITE_TRACE_NONLEGACY_MASK 0x0f /* Normal flags */ +#define CURSORSCAN_WRITE 0x0001 +#define CURSORSCAN_MINVALID 0x0002 +#define CURSORSCAN_MAXVALID 0x0004 +#define CURSORSCAN_REVERSE 0x0008 +#define CURSORSCAN_MININCL 0x0010 +#define CURSORSCAN_MAXINCL 0x0020 + +#define CURSORSCAN_LIMITVALID 0x0040 +#define CURSORSCAN_LIMITINCL 0x0080 +#define CURSORSCAN_LIMITMAX 0x0100 + +#define CURSORSCAN_OOM 0x0200 +#define CURSORSCAN_PGWRITE 0x0400 +#define CURSORSCAN_PGFAIL 0x0800 + +typedef struct CursorScan CursorScan; +struct CursorScan { + int tnum; /* Root page of scanned b-tree */ + int flags; /* Mask of CURSORSCAN_* flags */ + i64 iMin; + i64 iMax; + i64 iLimit; + u8 *aMin; + u8 *aMax; + u8 *aLimit; + int nRef; /* Number of pointers to this structure */ + KeyInfo *pKeyInfo; /* KeyInfo structure for indexes */ + CursorScan *pNext; /* Next CursorScan object in list */ +}; + + /* ** Maximum number of sqlite3.aDb[] entries. This is the number of attached ** databases plus 2 for "main" and "temp". @@ -1542,6 +1573,9 @@ struct sqlite3 { u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ u8 eConcurrent; /* CONCURRENT_* value */ + u8 bConcurrentReport; /* Concurrent transaction reports enabled */ + CursorScan *pCScanList; + char *zBCReport; u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ u8 bBenignMalloc; /* Do not require OOMs if true */ @@ -4516,6 +4550,7 @@ int sqlite3BitvecSet(Bitvec*, u32); void sqlite3BitvecClear(Bitvec*, u32, void*); void sqlite3BitvecDestroy(Bitvec*); u32 sqlite3BitvecSize(Bitvec*); +int sqlite3BitvecArray(Bitvec*, u32 **pa, int *pn); #ifndef SQLITE_UNTESTABLE int sqlite3BitvecBuiltinTest(int,int*); #endif @@ -5337,4 +5372,6 @@ void sqlite3VectorErrorMsg(Parse*, Expr*); const char **sqlite3CompileOptions(int *pnOpt); #endif +void sqlite3BtreeScanDerefList(CursorScan*); + #endif /* SQLITEINT_H */ diff --git a/src/test1.c b/src/test1.c index 55b78a3008..39ab8daf07 100644 --- a/src/test1.c +++ b/src/test1.c @@ -8115,6 +8115,53 @@ static int SQLITE_TCLAPI test_mmap_warm( } } +/* +** Usage: sqlite3_begin_concurrent_report DB +*/ +static int SQLITE_TCLAPI test_begin_concurrent_report( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zReport = 0; + sqlite3 *db; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zReport = sqlite3_begin_concurrent_report(db); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport?zReport:"", -1)); + return TCL_OK; +} + +/* +** Usage: sqlite3_begin_concurrent_report_enable DB ENABLE +*/ +static int SQLITE_TCLAPI test_begin_concurrent_report_enable( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + int iVal; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB ENABLE"); + return TCL_ERROR; + } + + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[2], &iVal) ) return TCL_ERROR; + sqlite3_begin_concurrent_report_enable(db, iVal); + Tcl_ResetResult(interp); + return TCL_OK; +} + /* ** Usage: test_write_db DB OFFSET DATA ** @@ -8653,6 +8700,8 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_config_sorterref", test_config_sorterref, 0 }, { "sqlite3_autovacuum_pages", test_autovacuum_pages, 0 }, { "decode_hexdb", test_decode_hexdb, 0 }, + { "sqlite3_begin_concurrent_report", test_begin_concurrent_report, 0 }, + { "sqlite3_begin_concurrent_report_enable", test_begin_concurrent_report_enable, 0 }, { "test_write_db", test_write_db, 0 }, { "sqlite3_register_cksumvfs", test_register_cksumvfs, 0 }, { "sqlite3_unregister_cksumvfs", test_unregister_cksumvfs, 0 }, diff --git a/src/test2.c b/src/test2.c index 850e1e1a04..86ac938faf 100644 --- a/src/test2.c +++ b/src/test2.c @@ -147,7 +147,7 @@ static int SQLITE_TCLAPI pager_commit( return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); - rc = sqlite3PagerCommitPhaseOne(pPager, 0, 0); + rc = sqlite3PagerCommitPhaseOne(pPager, 0, 0, 0); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; diff --git a/src/vdbe.c b/src/vdbe.c index b8152fbfd0..e59031c0cc 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -91,7 +91,7 @@ static void updateMaxBlobsize(Mem *p){ ** hook are enabled for database connect DB. */ #ifdef SQLITE_ENABLE_PREUPDATE_HOOK -# define HAS_UPDATE_HOOK(DB) ((DB)->xPreUpdateCallback||(DB)->xUpdateCallback) +# define HAS_UPDATE_HOOK(DB) ((DB)->xPreUpdateCallback||(DB)->xUpdateCallback||(DB)->eConcurrent) #else # define HAS_UPDATE_HOOK(DB) ((DB)->xUpdateCallback) #endif @@ -2060,6 +2060,11 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ if( (flags1 & flags3 & MEM_Int)!=0 ){ assert( (pOp->p5 & SQLITE_AFF_MASK)!=SQLITE_AFF_TEXT || CORRUPT_DB ); /* Common case of comparison of two integers */ + if( pOp->p4type==P4_INT32 ){ + sqlite3BtreeScanLimit( + p->apCsr[pOp->p4.i]->uc.pCursor, 0, pIn1->u.i, pOp->opcode + ); + } if( pIn3->u.i > pIn1->u.i ){ iCompare = +1; if( sqlite3aGTb[pOp->opcode] ){ @@ -2145,8 +2150,8 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ flags3 = (pIn3->flags & ~MEM_TypeMask) | (flags3 & MEM_TypeMask); } } - assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); - res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); + /* assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); */ + res = sqlite3MemCompare(pIn3,pIn1,pOp->p4type==P4_COLLSEQ?pOp->p4.pColl:0); } /* At this point, res is negative, zero, or positive if reg[P1] is @@ -3609,6 +3614,10 @@ case OP_AutoCommit: { }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; }else{ + if( desiredAutoCommit==0 ){ + sqlite3BtreeScanDerefList(db->pCScanList); + db->pCScanList = 0; + } db->autoCommit = (u8)desiredAutoCommit; } hrc = sqlite3VdbeHalt(p); @@ -4481,6 +4490,7 @@ case OP_SeekGT: { /* jump, in3, group */ } } rc = sqlite3BtreeTableMoveto(pC->uc.pCursor, (u64)iKey, 0, &res); + sqlite3BtreeScanStart(pC->uc.pCursor, 0, iKey, pOp->opcode, 0); pC->movetoTarget = iKey; /* Used by OP_Delete */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; @@ -4528,6 +4538,7 @@ case OP_SeekGT: { /* jump, in3, group */ #endif r.eqSeen = 0; rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, &r, &res); + sqlite3BtreeScanStart(pC->uc.pCursor, &r, 0, pOp->opcode, eqOnly); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } @@ -5056,6 +5067,7 @@ notExistsWithKey: assert( pCrsr!=0 ); res = 0; rc = sqlite3BtreeTableMoveto(pCrsr, iKey, 0, &res); + sqlite3BtreeScanStart(pCrsr, 0, iKey, pOp->opcode, 0); assert( rc==SQLITE_OK || res==0 ); pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; @@ -5308,6 +5320,9 @@ case OP_Insert: { #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /* Invoke the pre-update hook, if any */ if( pTab ){ + sqlite3BtreeScanWrite( + pC->uc.pCursor, SQLITE_INSERT, pC->movetoTarget, (u8*)pData->z, pData->n + ); if( db->xPreUpdateCallback && !(pOp->p5 & OPFLAG_ISUPDATE) ){ sqlite3VdbePreUpdateHook(p,pC,SQLITE_INSERT,zDb,pTab,x.nKey,pOp->p2,-1); } @@ -5461,17 +5476,20 @@ case OP_Delete: { #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /* Invoke the pre-update-hook if required. */ - assert( db->xPreUpdateCallback==0 || pTab==pOp->p4.pTab ); - if( db->xPreUpdateCallback && pTab ){ - assert( !(opflags & OPFLAG_ISUPDATE) - || HasRowid(pTab)==0 - || (aMem[pOp->p3].flags & MEM_Int) - ); - sqlite3VdbePreUpdateHook(p, pC, - (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE, - zDb, pTab, pC->movetoTarget, - pOp->p3, -1 - ); + if( pOp->p4.pTab ){ + rc = sqlite3BtreeScanWrite(pC->uc.pCursor, 0, pC->movetoTarget, 0, 0); + if( rc ) goto abort_due_to_error; + if( db->xPreUpdateCallback ){ + assert( !(opflags & OPFLAG_ISUPDATE) + || HasRowid(pTab)==0 + || (aMem[pOp->p3].flags & MEM_Int) + ); + sqlite3VdbePreUpdateHook(p, pC, + (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE, + zDb, pTab, pC->movetoTarget, + pOp->p3, -1 + ); + } } if( opflags & OPFLAG_ISNOOP ) break; #endif @@ -5773,6 +5791,7 @@ case OP_Last: { /* jump */ } } rc = sqlite3BtreeLast(pCrsr, &res); + sqlite3BtreeScanStart(pCrsr, 0, 0, pOp->opcode, 0); pC->nullRow = (u8)res; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -5876,6 +5895,7 @@ case OP_Rewind: { /* jump */ pCrsr = pC->uc.pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); + sqlite3BtreeScanStart(pCrsr, 0, 0, OP_Rewind, 0); pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } @@ -6333,6 +6353,7 @@ case OP_IdxGE: { /* jump */ } /* End of inlined sqlite3VdbeIdxKeyCompare() */ + sqlite3BtreeScanLimit(pC->uc.pCursor, &r, 0, pOp->opcode); assert( (OP_IdxLE&1)==(OP_IdxLT&1) && (OP_IdxGE&1)==(OP_IdxGT&1) ); if( (pOp->opcode&1)==(OP_IdxLT&1) ){ assert( pOp->opcode==OP_IdxLE || pOp->opcode==OP_IdxLT ); @@ -6343,7 +6364,9 @@ case OP_IdxGE: { /* jump */ } VdbeBranchTaken(res>0,2); assert( rc==SQLITE_OK ); - if( res>0 ) goto jump_to_p2; + if( res>0 ){ + goto jump_to_p2; + } break; } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 599d064165..0e33172c99 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -468,6 +468,7 @@ struct PreUpdate { VdbeCursor *pCsr; /* Cursor to read old values from */ int op; /* One of SQLITE_INSERT, UPDATE, DELETE */ u8 *aRecord; /* old.* database record */ + int nRecord; /* Size of aRecord in bytes */ KeyInfo keyinfo; UnpackedRecord *pUnpacked; /* Unpacked version of aRecord[] */ UnpackedRecord *pNewUnpacked; /* Unpacked version of new.* record */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 5eeb5d1c0e..b3abe88885 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1863,6 +1863,7 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ goto preupdate_old_out; } p->aRecord = aRec; + p->nRecord = nRec; } pMem = *ppValue = &p->pUnpacked->aMem[iIdx]; diff --git a/src/wherecode.c b/src/wherecode.c index 460ac4fe30..6f10b89f0a 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1616,6 +1616,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); + sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(iCur), P4_INT32); VdbeCoverageIf(v, testOp==OP_Le); VdbeCoverageIf(v, testOp==OP_Lt); VdbeCoverageIf(v, testOp==OP_Ge); -- 2.39.5