Fix a problem with an fts5 secure-delete on a rowid/term pair that follows a legacy...

FossilOrigin-Name: 579aea0c28e01a79620ac758edc02db3a01baaa073e7773b8f0b6f610479520b

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 2b329656383974b4a10be2ef6acbf103f7d21aa9..0500d54a9e5c15b156af8633e7b565e53ade8570 100644 (file)
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -2904,7 +2904,6 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){
       assert_nc( i2!=0 );
       pRes->bTermEq = 1;
       if( p1->iRowid==p2->iRowid ){
-        p1->bDel = p2->bDel;
         return i2;
       }
       res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1;
@@ -5132,34 +5131,39 @@ static void fts5DoSecureDelete(
 
   /* Set variable bLastInDoclist to true if this entry happens to be
   ** the last rowid in the doclist for its term.  */
-  if( iNextOff>=iPgIdx ){
-    int pgno = pSeg->iLeafPgno+1;
-    fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
-    iNextOff = iPgIdx;
-  }else{
-    /* Loop through the page-footer. If iNextOff (offset of the
-    ** entry following the one we are removing) is equal to the 
-    ** offset of a key on this page, then the entry is the last 
-    ** in its doclist.  */
-    int iKeyOff = 0;
-    for(iIdx=0; iIdx<nIdx; /* no-op */){
-      u32 iVal = 0;
-      iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
-      iKeyOff += iVal;
-      if( iKeyOff==iNextOff ){
-        bLastInDoclist = 1;
+  if( pSeg->bDel==0 ){
+    if( iNextOff>=iPgIdx ){
+      int pgno = pSeg->iLeafPgno+1;
+      fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
+      iNextOff = iPgIdx;
+    }else{
+      /* Loop through the page-footer. If iNextOff (offset of the
+      ** entry following the one we are removing) is equal to the 
+      ** offset of a key on this page, then the entry is the last 
+      ** in its doclist.  */
+      int iKeyOff = 0;
+      for(iIdx=0; iIdx<nIdx; /* no-op */){
+        u32 iVal = 0;
+        iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+        iKeyOff += iVal;
+        if( iKeyOff==iNextOff ){
+          bLastInDoclist = 1;
+        }
       }
     }
-  }
 
-  /* If this is (a) the first rowid on a page and (b) is not followed by
-  ** another position list on the same page, set the "first-rowid" field
-  ** of the header to 0.  */
-  if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist || iNextOff==iPgIdx) ){
-    fts5PutU16(&aPg[0], 0);
+    /* If this is (a) the first rowid on a page and (b) is not followed by
+    ** another position list on the same page, set the "first-rowid" field
+    ** of the header to 0.  */
+    if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist || iNextOff==iPgIdx) ){
+      fts5PutU16(&aPg[0], 0);
+    }
   }
 
-  if( bLastInDoclist==0 ){
+  if( pSeg->bDel ){
+    iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta);
+    aPg[iOff++] = 0x01;
+  }else if( bLastInDoclist==0 ){
     if( iNextOff!=iPgIdx ){
       u64 iNextDelta = 0;
       iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta);
@@ -5271,6 +5275,15 @@ static void fts5DoSecureDelete(
     }
   }
 
+  /* Assuming no error has occurred, this block does final edits to the
+  ** leaf page before writing it back to disk. Input variables are:
+  **
+  **   nPg: Total initial size of leaf page.
+  **   iPgIdx: Initial offset of page footer.
+  **
+  **   iOff: Offset to move data to
+  **   iNextOff: Offset to move data from
+  */
   if( p->rc==SQLITE_OK ){
     const int nMove = nPg - iNextOff;     /* Number of bytes to move */
     int nShift = iNextOff - iOff;         /* Distance to move them */
@@ -5471,10 +5484,16 @@ static void fts5FlushOneHash(Fts5Index *p){
                 fts5WriteFlushLeaf(p, &writer);
               }
             }else{
-              int bDummy;
-              int nPos;
-              int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
-              nCopy += nPos;
+              int bDel = 0;
+              int nPos = 0;
+              int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDel);
+              if( bDel && bSecureDelete ){
+                fts5BufferAppendVarint(&p->rc, pBuf, nPos*2);
+                iOff += nCopy;
+                nCopy = nPos;
+              }else{
+                nCopy += nPos;
+              }
               if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
                 /* The entire poslist will fit on the current leaf. So copy
                 ** it in one go. */

diff --git a/ext/fts5/test/fts5secure6.test b/ext/fts5/test/fts5secure6.test
index ffb10cb24a2a9e488e7987cf4657568cbe706474..43d4f075e87d347d6ced6b5fba6055d796335ad5 100644 (file)
--- a/ext/fts5/test/fts5secure6.test
+++ b/ext/fts5/test/fts5secure6.test
@@ -18,7 +18,7 @@ db progress 1 progress_handler
 set ::PHC 0
 proc progress_handler {args} {
   incr ::PHC
-  if {($::PHC % 100000)==0}  breakpoint
+  # if {($::PHC % 100000)==0}  breakpoint
   return 0
 }
 
@@ -73,12 +73,12 @@ do_execsql_test 2.2 {
 #-------------------------------------------------------------------------
 reset_db
 
-do_execsql_test 2.0 {
+do_execsql_test 3.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x);
   INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) 
 }
 
-do_execsql_test 2.1 {
+do_execsql_test 3.1 {
   BEGIN;
     INSERT INTO t1(rowid, x) 
       VALUES(51869, 'when whenever where weress what turn'), 
@@ -91,5 +91,23 @@ do_execsql_test 3.2 {
   INSERT INTO t1(t1) VALUES('integrity-check');
 }
 
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(rowid, x) VALUES(10, 'one two');
+}
+do_execsql_test 4.1 {
+  UPDATE t1 SET x = 'one three' WHERE rowid=10;
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+do_execsql_test 4.2 {
+  DELETE FROM t1 WHERE rowid=10;
+}
+do_execsql_test 4.3 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
 finish_test
 

diff --git a/manifest b/manifest
index d9b10b76b565eccdfc527d7e45b6c22dbac61b7d..1976ce0af12d70d663024658ce8f7e86931dd7bf 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enable\sSQLITE_VTAB_INNOCUOUS\sfor\sFTS3,\sFTS5,\sand\sRTREE.
-D 2023-10-12T18:46:27.294
+C Fix\sa\sproblem\swith\san\sfts5\ssecure-delete\son\sa\srowid/term\spair\sthat\sfollows\sa\slegacy\sdelete\sof\sthe\ssame\spair.
+D 2023-10-12T19:46:58.003
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -94,7 +94,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
 F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6
 F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
-F ext/fts5/fts5_index.c 5c3872a01ae519af9839b675408163aab8eb1e31917c8b3d0ce726b4ad11d942
+F ext/fts5/fts5_index.c a0f8e58e1c101d0b7959264f76b8c0c3c44914a999b29ec4264a6f55f1bccfe1
 F ext/fts5/fts5_main.c 1ea6636a1f528e9042cc6992d57b5a97c4a4e401ab1d2836c0cc805a1d27a982
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -204,7 +204,7 @@ F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd
 F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610
 F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97
 F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a
-F ext/fts5/test/fts5secure6.test 120feecc8c55b4774f858721e6c62c2094b059ecbcfd7fdc24bde886f55ef6ca
+F ext/fts5/test/fts5secure6.test 6f7fbb1f96910ccbdbfd2097f761f4c77c150c03734503331e2d428f77fa2a6e
 F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a
 F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b
 F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b
@@ -2128,8 +2128,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d3b983edf7164d30ddb8b4b745a3feff9737149d85b1c428d6ad488ac6a1960a
-R d66c3f0c591f0a5c79818d4ec908dabb
-U drh
-Z 715f0eac07de34228b29f32483ef26b6
+P f34c533b6c1f8ef3f69c75c1039406c12751cdde1fbdcb4d9776b24455facf8a
+R 042a80d2d5719fd8ae111fd1b613eb4b
+U dan
+Z a5f84e7c870f4185af46feb29cb8159d
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index f80a5cef581b804ba48560b00ca81f358adf9dc5..3ef20a66601747b32c6b1f213d35611e52be1d67 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f34c533b6c1f8ef3f69c75c1039406c12751cdde1fbdcb4d9776b24455facf8a
\ No newline at end of file
+579aea0c28e01a79620ac758edc02db3a01baaa073e7773b8f0b6f610479520b
\ No newline at end of file