From: drh <> Date: Mon, 24 Apr 2023 21:08:11 +0000 (+0000) Subject: Hold the RTREE node cache open and defer writes until the very end of the X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=dd8bb7d912bf2a7e0beb16a3a916b33af2c8dfb3;p=thirdparty%2Fsqlite.git Hold the RTREE node cache open and defer writes until the very end of the INSERT statement, for a 20x performance improvement on the test script identified at [forum:/forumpost/6b8bca17bb884ef3|forum post 6b8bca17bb884ef3]. However, there are still bugs and tests will crash. FossilOrigin-Name: b35cb7458fe16390f3b1f0b2acfd3d51fabd6bf9c5f176a44d8b3cc932f74149 --- diff --git a/ext/rtree/geopoly.c b/ext/rtree/geopoly.c index 640cb86b20..b82b9f8f17 100644 --- a/ext/rtree/geopoly.c +++ b/ext/rtree/geopoly.c @@ -1772,14 +1772,14 @@ static sqlite3_module geopolyModule = { rtreeRowid, /* xRowid - read data */ geopolyUpdate, /* xUpdate - write data */ rtreeBeginTransaction, /* xBegin - begin transaction */ - rtreeEndTransaction, /* xSync - sync transaction */ - rtreeEndTransaction, /* xCommit - commit transaction */ - rtreeEndTransaction, /* xRollback - rollback transaction */ + rtreeCommit, /* xSync - sync transaction */ + rtreeCommit, /* xCommit - commit transaction */ + rtreeRollback, /* xRollback - rollback transaction */ geopolyFindFunction, /* xFindFunction - function overloading */ rtreeRename, /* xRename - rename the table */ rtreeSavepoint, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ + rtreeCacheWrite, /* xRelease */ + rtreeCacheAbandon, /* xRollbackTo */ rtreeShadowName /* xShadowName */ }; diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index da5e2a97a8..684c444c53 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -711,6 +711,7 @@ static int nodeAcquire( RTREE_IS_CORRUPT(pRtree); return SQLITE_CORRUPT_VTAB; } + if( pNode->nRef==0 ) pRtree->nNodeRef++; pNode->nRef++; *ppNode = pNode; return SQLITE_OK; @@ -886,6 +887,32 @@ static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ return rc; } +/* +** Scan the hash table. Verify that every hash table entry has nRef==0 +** and remove them. Any dirty pages are written if writeDirty is true. +*/ +static int rtreeResetHashTable(Rtree *pRtree, int writeDirty){ + int i; + int rc = SQLITE_OK; + for(i=0; iaHash[i]; + RtreeNode *pNext; + if( pNode==0 ) continue; + while( 1 /*exit-by-break*/ ){ + pNext = pNode->pNext; + if( pNode->isDirty && writeDirty ){ + rc = nodeWrite(pRtree, pNode); + if( rc ) writeDirty = 0; + } + sqlite3_free(pNode); + if( pNext==0 ) break; + pNode = pNext; + } + pRtree->aHash[i] = 0; + } + return rc; +} + /* ** Release a reference to a node. If the node is dirty and the reference ** count drops to zero, the node data is written to the database. @@ -904,11 +931,13 @@ static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ if( pNode->pParent ){ rc = nodeRelease(pRtree, pNode->pParent); } +#if 0 if( rc==SQLITE_OK ){ rc = nodeWrite(pRtree, pNode); } nodeHashDelete(pRtree, pNode); sqlite3_free(pNode); +#endif } } return rc; @@ -1016,6 +1045,7 @@ static void rtreeRelease(Rtree *pRtree){ pRtree->inWrTrans = 0; assert( pRtree->nCursor==0 ); nodeBlobReset(pRtree); + rtreeResetHashTable(pRtree, 0); assert( pRtree->nNodeRef==0 || pRtree->bCorrupt ); sqlite3_finalize(pRtree->pWriteNode); sqlite3_finalize(pRtree->pDeleteNode); @@ -3375,10 +3405,33 @@ static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ ** Called when a transaction completes (either by COMMIT or ROLLBACK). ** The sqlite3_blob object should be released at this point. */ -static int rtreeEndTransaction(sqlite3_vtab *pVtab){ +static int rtreeCommit(sqlite3_vtab *pVtab){ Rtree *pRtree = (Rtree *)pVtab; pRtree->inWrTrans = 0; nodeBlobReset(pRtree); + rtreeResetHashTable(pRtree, 1); + return SQLITE_OK; +} +static int rtreeRollback(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + pRtree->inWrTrans = 0; + nodeBlobReset(pRtree); + rtreeResetHashTable(pRtree, 0); + return SQLITE_OK; +} + +/* +** Called at the end of a statement to ensure that the cache +** has been cleared and that all changes have been written. +*/ +static int rtreeCacheWrite(sqlite3_vtab *pVtab, int n){ + UNUSED_PARAMETER(n); + rtreeResetHashTable((Rtree*)pVtab, 1); + return SQLITE_OK; +} +static int rtreeCacheAbandon(sqlite3_vtab *pVtab, int n){ + UNUSED_PARAMETER(n); + rtreeResetHashTable((Rtree*)pVtab, 0); return SQLITE_OK; } @@ -3494,14 +3547,14 @@ static sqlite3_module rtreeModule = { rtreeRowid, /* xRowid - read data */ rtreeUpdate, /* xUpdate - write data */ rtreeBeginTransaction, /* xBegin - begin transaction */ - rtreeEndTransaction, /* xSync - sync transaction */ - rtreeEndTransaction, /* xCommit - commit transaction */ - rtreeEndTransaction, /* xRollback - rollback transaction */ + rtreeCommit, /* xSync - sync transaction */ + rtreeCommit, /* xCommit - commit transaction */ + rtreeRollback, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ rtreeRename, /* xRename - rename the table */ rtreeSavepoint, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ + rtreeCacheWrite, /* xRelease */ + rtreeCacheAbandon, /* xRollbackTo */ rtreeShadowName /* xShadowName */ }; diff --git a/manifest b/manifest index 0509728ae1..91afee5ee2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\scompile-time\sdetection\sof\sarchitecture\sbyte-order\sin\sthe\sRTREE\nextension\sso\sthat\sit\sis\saligned\swith\sthe\slatest\senhancements\sin\sthe\score. -D 2023-04-24T19:23:42.196 +C Hold\sthe\sRTREE\snode\scache\sopen\sand\sdefer\swrites\suntil\sthe\svery\send\sof\sthe\nINSERT\sstatement,\sfor\sa\s20x\sperformance\simprovement\son\sthe\stest\sscript\nidentified\sat\s[forum:/forumpost/6b8bca17bb884ef3|forum\spost\s6b8bca17bb884ef3].\s\sHowever,\sthere\sare\sstill\sbugs\sand\stests\swill\scrash. +D 2023-04-24T21:08:11.797 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -399,8 +399,8 @@ F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc782 F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c335096108c12c01bddbadcec F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/geopoly.c 971e0b5bd9adaf0811feb8c0842a310811159da10319eb0e74fdb42bf26b99ca -F ext/rtree/rtree.c 925888f7672b326fc2a29b3a212ec3ae4aa2331507ecccfaf7f0ac0335020330 +F ext/rtree/geopoly.c 398b966772d81b9e6cd05caaf94f5d9d38b26f4245d4f0908b41c60024298412 +F ext/rtree/rtree.c 16bd8ba24aeaba2240b32ecdd1f4cb0ee3d4fe24c040d32622efd2f50dfca20b F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 F ext/rtree/rtree1.test d47f58832145fcfed9067bc457ca8664962196c4566c17a1ebd679367db55d11 F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d @@ -2059,9 +2059,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e79c95fc130fc302719690eb6391d96070aff825b2b51ef6c4ad459d9a8918d7 -Q +491bd51da5e2069078d7295396d80d2ccdc3a5871714fef948076939174e6acd -R 0ddcb2c385e38b7edc9b293e66b74615 +P 122431d3a7267ec83768316ab146c0557fb6c0577a4a47ac6ed3d7aa6811ca9a +R c1c53d72116931163ed425d9c8ab1799 +T *branch * rtree-batch-insert +T *sym-rtree-batch-insert * +T -sym-trunk * U drh -Z 660ef923c98b6f83ec3ee30adf820b34 +Z 089582dbdb98caa4b46ed3a4bef41b4b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 07be9e66cc..6c507a2652 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -122431d3a7267ec83768316ab146c0557fb6c0577a4a47ac6ed3d7aa6811ca9a \ No newline at end of file +b35cb7458fe16390f3b1f0b2acfd3d51fabd6bf9c5f176a44d8b3cc932f74149 \ No newline at end of file