Hold the RTREE node cache open and defer writes until the very end of the

INSERT statement, for a 20x performance improvement on the test script
identified at [forum:/forumpost/6b8bca17bb884ef3|forum post 6b8bca17bb884ef3].  However, there are still bugs and tests will crash.

FossilOrigin-Name: b35cb7458fe16390f3b1f0b2acfd3d51fabd6bf9c5f176a44d8b3cc932f74149

diff --git a/ext/rtree/geopoly.c b/ext/rtree/geopoly.c
index 640cb86b20a325018391706f7d666c7f3a982b65..b82b9f8f17c9f3077a5896110c3ad7684170e27e 100644 (file)
--- a/ext/rtree/geopoly.c
+++ b/ext/rtree/geopoly.c
@@ -1772,14 +1772,14 @@ static sqlite3_module geopolyModule = {
   rtreeRowid,                 /* xRowid - read data */
   geopolyUpdate,              /* xUpdate - write data */
   rtreeBeginTransaction,      /* xBegin - begin transaction */
-  rtreeEndTransaction,        /* xSync - sync transaction */
-  rtreeEndTransaction,        /* xCommit - commit transaction */
-  rtreeEndTransaction,        /* xRollback - rollback transaction */
+  rtreeCommit,                /* xSync - sync transaction */
+  rtreeCommit,                /* xCommit - commit transaction */
+  rtreeRollback,              /* xRollback - rollback transaction */
   geopolyFindFunction,        /* xFindFunction - function overloading */
   rtreeRename,                /* xRename - rename the table */
   rtreeSavepoint,             /* xSavepoint */
-  0,                          /* xRelease */
-  0,                          /* xRollbackTo */
+  rtreeCacheWrite,            /* xRelease */
+  rtreeCacheAbandon,          /* xRollbackTo */
   rtreeShadowName             /* xShadowName */
 };
 

diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index da5e2a97a8c3bc48bdd6ce00471d7cb74643822a..684c444c536d1a641014bb748106afcb8b555008 100644 (file)
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -711,6 +711,7 @@ static int nodeAcquire(
       RTREE_IS_CORRUPT(pRtree);
       return SQLITE_CORRUPT_VTAB;
     }
+    if( pNode->nRef==0 ) pRtree->nNodeRef++;
     pNode->nRef++;
     *ppNode = pNode;
     return SQLITE_OK;
@@ -886,6 +887,32 @@ static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){
   return rc;
 }
 
+/*
+** Scan the hash table.  Verify that every hash table entry has nRef==0
+** and remove them.  Any dirty pages are written if writeDirty is true.
+*/
+static int rtreeResetHashTable(Rtree *pRtree, int writeDirty){
+  int i;
+  int rc = SQLITE_OK;
+  for(i=0; i<HASHSIZE; i++){
+    RtreeNode *pNode = pRtree->aHash[i];
+    RtreeNode *pNext;
+    if( pNode==0 ) continue;
+    while( 1 /*exit-by-break*/ ){
+      pNext = pNode->pNext;
+      if( pNode->isDirty && writeDirty ){
+         rc = nodeWrite(pRtree, pNode);
+         if( rc ) writeDirty = 0;
+      }
+      sqlite3_free(pNode);
+      if( pNext==0 ) break;
+      pNode = pNext;
+    }
+    pRtree->aHash[i] = 0;
+  }
+  return rc;
+}
+
 /*
 ** Release a reference to a node. If the node is dirty and the reference
 ** count drops to zero, the node data is written to the database.
@@ -904,11 +931,13 @@ static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){
       if( pNode->pParent ){
         rc = nodeRelease(pRtree, pNode->pParent);
       }
+#if 0
       if( rc==SQLITE_OK ){
         rc = nodeWrite(pRtree, pNode);
       }
       nodeHashDelete(pRtree, pNode);
       sqlite3_free(pNode);
+#endif
     }
   }
   return rc;
@@ -1016,6 +1045,7 @@ static void rtreeRelease(Rtree *pRtree){
     pRtree->inWrTrans = 0;
     assert( pRtree->nCursor==0 );
     nodeBlobReset(pRtree);
+    rtreeResetHashTable(pRtree, 0);
     assert( pRtree->nNodeRef==0 || pRtree->bCorrupt );
     sqlite3_finalize(pRtree->pWriteNode);
     sqlite3_finalize(pRtree->pDeleteNode);
@@ -3375,10 +3405,33 @@ static int rtreeBeginTransaction(sqlite3_vtab *pVtab){
 ** Called when a transaction completes (either by COMMIT or ROLLBACK).
 ** The sqlite3_blob object should be released at this point.
 */
-static int rtreeEndTransaction(sqlite3_vtab *pVtab){
+static int rtreeCommit(sqlite3_vtab *pVtab){
   Rtree *pRtree = (Rtree *)pVtab;
   pRtree->inWrTrans = 0;
   nodeBlobReset(pRtree);
+  rtreeResetHashTable(pRtree, 1);
+  return SQLITE_OK;
+}
+static int rtreeRollback(sqlite3_vtab *pVtab){
+  Rtree *pRtree = (Rtree *)pVtab;
+  pRtree->inWrTrans = 0;
+  nodeBlobReset(pRtree);
+  rtreeResetHashTable(pRtree, 0);
+  return SQLITE_OK;
+}
+
+/*
+** Called at the end of a statement to ensure that the cache
+** has been cleared and that all changes have been written.
+*/
+static int rtreeCacheWrite(sqlite3_vtab *pVtab, int n){
+  UNUSED_PARAMETER(n);
+  rtreeResetHashTable((Rtree*)pVtab, 1);
+  return SQLITE_OK;
+}
+static int rtreeCacheAbandon(sqlite3_vtab *pVtab, int n){
+  UNUSED_PARAMETER(n);
+  rtreeResetHashTable((Rtree*)pVtab, 0);
   return SQLITE_OK;
 }
 
@@ -3494,14 +3547,14 @@ static sqlite3_module rtreeModule = {
   rtreeRowid,                 /* xRowid - read data */
   rtreeUpdate,                /* xUpdate - write data */
   rtreeBeginTransaction,      /* xBegin - begin transaction */
-  rtreeEndTransaction,        /* xSync - sync transaction */
-  rtreeEndTransaction,        /* xCommit - commit transaction */
-  rtreeEndTransaction,        /* xRollback - rollback transaction */
+  rtreeCommit,                /* xSync - sync transaction */
+  rtreeCommit,                /* xCommit - commit transaction */
+  rtreeRollback,              /* xRollback - rollback transaction */
   0,                          /* xFindFunction - function overloading */
   rtreeRename,                /* xRename - rename the table */
   rtreeSavepoint,             /* xSavepoint */
-  0,                          /* xRelease */
-  0,                          /* xRollbackTo */
+  rtreeCacheWrite,            /* xRelease */
+  rtreeCacheAbandon,          /* xRollbackTo */
   rtreeShadowName             /* xShadowName */
 };
 

diff --git a/manifest b/manifest
index 0509728ae13aa6c595af70e8ac59d97c608ac216..91afee5ee27c9fac9b36859d20968df1235d18cb 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthe\scompile-time\sdetection\sof\sarchitecture\sbyte-order\sin\sthe\sRTREE\nextension\sso\sthat\sit\sis\saligned\swith\sthe\slatest\senhancements\sin\sthe\score.
-D 2023-04-24T19:23:42.196
+C Hold\sthe\sRTREE\snode\scache\sopen\sand\sdefer\swrites\suntil\sthe\svery\send\sof\sthe\nINSERT\sstatement,\sfor\sa\s20x\sperformance\simprovement\son\sthe\stest\sscript\nidentified\sat\s[forum:/forumpost/6b8bca17bb884ef3|forum\spost\s6b8bca17bb884ef3].\s\sHowever,\sthere\sare\sstill\sbugs\sand\stests\swill\scrash.
+D 2023-04-24T21:08:11.797
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -399,8 +399,8 @@ F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc782
 F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c335096108c12c01bddbadcec
 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/geopoly.c 971e0b5bd9adaf0811feb8c0842a310811159da10319eb0e74fdb42bf26b99ca
-F ext/rtree/rtree.c 925888f7672b326fc2a29b3a212ec3ae4aa2331507ecccfaf7f0ac0335020330
+F ext/rtree/geopoly.c 398b966772d81b9e6cd05caaf94f5d9d38b26f4245d4f0908b41c60024298412
+F ext/rtree/rtree.c 16bd8ba24aeaba2240b32ecdd1f4cb0ee3d4fe24c040d32622efd2f50dfca20b
 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412
 F ext/rtree/rtree1.test d47f58832145fcfed9067bc457ca8664962196c4566c17a1ebd679367db55d11
 F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d
@@ -2059,9 +2059,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e79c95fc130fc302719690eb6391d96070aff825b2b51ef6c4ad459d9a8918d7
-Q +491bd51da5e2069078d7295396d80d2ccdc3a5871714fef948076939174e6acd
-R 0ddcb2c385e38b7edc9b293e66b74615
+P 122431d3a7267ec83768316ab146c0557fb6c0577a4a47ac6ed3d7aa6811ca9a
+R c1c53d72116931163ed425d9c8ab1799
+T *branch * rtree-batch-insert
+T *sym-rtree-batch-insert *
+T -sym-trunk *
 U drh
-Z 660ef923c98b6f83ec3ee30adf820b34
+Z 089582dbdb98caa4b46ed3a4bef41b4b
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index 07be9e66cc733d2c05e25e59771519f5f0e63fd6..6c507a26524ad637f13d03d28785164f8ad97ef6 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-122431d3a7267ec83768316ab146c0557fb6c0577a4a47ac6ed3d7aa6811ca9a
\ No newline at end of file
+b35cb7458fe16390f3b1f0b2acfd3d51fabd6bf9c5f176a44d8b3cc932f74149
\ No newline at end of file