From: dan <Dan Kennedy>
Date: Mon, 17 Apr 2023 18:32:22 +0000 (+0000)
Subject: Fix some problems with using fts5 options 'secure-delete' and detail=none together.
X-Git-Tag: version-3.42.0~135
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=68c0da6f66f6468aae09f7976ecf95cdeebca4e6;p=thirdparty%2Fsqlite.git

Fix some problems with using fts5 options 'secure-delete' and detail=none together.

FossilOrigin-Name: 4d3f27ba90aa59837e49ceaae9f36cad426af3a33707d85d1bb848fc1bda6fed
---

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index c51d82be99..9c14023889 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -1954,7 +1954,7 @@ static void fts5SegIterNext_None(
   iOff = pIter->iLeafOffset;
 
   /* Next entry is on the next page */
-  if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
+  while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
     fts5SegIterNextPage(p, pIter);
     if( p->rc || pIter->pLeaf==0 ) return;
     pIter->iRowid = 0;
@@ -4601,6 +4601,7 @@ static void fts5SecureDeleteOverflow(
   int iPgno,
   int *pbLastInDoclist
 ){
+  const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE);
   int pgno;
   Fts5Data *pLeaf = 0;
   assert( iPgno!=1 );
@@ -4627,9 +4628,12 @@ static void fts5SecureDeleteOverflow(
       /* The page contains no terms or rowids. Replace it with an empty
       ** page and move on to the right-hand peer.  */
       const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04}; 
-      fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty));
+      assert_nc( bDetailNone==0 || pLeaf->nn==4 );
+      if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty));
       fts5DataRelease(pLeaf);
       pLeaf = 0;
+    }else if( bDetailNone ){
+      break;
     }else{
       int nShift = iNext - 4;
       int nPg;
@@ -4678,6 +4682,245 @@ static void fts5SecureDeleteOverflow(
   fts5DataRelease(pLeaf);
 }
 
+/*
+** Completely remove the entry that pSeg currently points to from 
+** the database.
+*/
+static void fts5DoSecureDelete(
+  Fts5Index *p,
+  Fts5Structure *pStruct,
+  Fts5SegIter *pSeg
+){
+  const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE);
+  int iSegid = pSeg->pSeg->iSegid;
+  u8 *aPg = pSeg->pLeaf->p;
+  int nPg = pSeg->pLeaf->nn;
+  int iPgIdx = pSeg->pLeaf->szLeaf;
+
+  u64 iDelta = 0;
+  u64 iNextDelta = 0;
+  int iNextOff = 0;
+  int iOff = 0;
+  int nIdx = 0;
+  u8 *aIdx = 0;
+  int bLastInDoclist = 0;
+  int iIdx = 0;
+  int iStart = 0;
+  int iKeyOff = 0;
+  int iPrevKeyOff = 0;
+  int iDelKeyOff = 0;       /* Offset of deleted key, if any */
+
+  nIdx = nPg-iPgIdx;
+  aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16);
+  if( p->rc ) return;
+  memcpy(aIdx, &aPg[iPgIdx], nIdx);
+
+  /* At this point segment iterator pSeg points to the entry
+  ** this function should remove from the b-tree segment. 
+  **
+  ** In detail=full or detail=column mode, pSeg->iLeafOffset is the 
+  ** offset of the first byte in the position-list for the entry to 
+  ** remove. Immediately before this comes two varints that will also
+  ** need to be removed:
+  **
+  **     + the rowid or delta rowid value for the entry, and
+  **     + the size of the position list in bytes.
+  **
+  ** Or, in detail=none mode, there is a single varint prior to 
+  ** pSeg->iLeafOffset - the rowid or delta rowid value.
+  **
+  ** This block sets the following variables:
+  **
+  **   iStart:
+  **   iDelta:
+  */
+  {
+    int iSOP;
+    if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
+      iStart = pSeg->iTermLeafOffset;
+    }else{
+      iStart = fts5GetU16(&aPg[0]);
+    }
+
+    iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
+    assert_nc( iSOP<=pSeg->iLeafOffset );
+
+    if( bDetailNone ){
+      while( iSOP<pSeg->iLeafOffset ){
+        if( aPg[iSOP]==0x00 ) iSOP++;
+        if( aPg[iSOP]==0x00 ) iSOP++;
+        iStart = iSOP;
+        iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
+      }
+
+      iNextOff = iSOP;
+      if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
+      if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
+
+    }else{
+      int nPos = 0;
+      iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
+      while( iSOP<pSeg->iLeafOffset ){
+        iStart = iSOP + (nPos/2);
+        iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
+        iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
+      }
+      assert_nc( iSOP==pSeg->iLeafOffset );
+      iNextOff = pSeg->iLeafOffset + pSeg->nPos;
+    }
+  }
+
+  iOff = iStart;
+  if( iNextOff>=iPgIdx ){
+    int pgno = pSeg->iLeafPgno+1;
+    fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
+    iNextOff = iPgIdx;
+  }else{
+    /* Set bLastInDoclist to true if the entry being removed is the last
+    ** in its doclist.  */
+    for(iIdx=0, iKeyOff=0; iIdx<nIdx; /* no-op */){
+      u32 iVal = 0;
+      iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+      iKeyOff += iVal;
+      if( iKeyOff==iNextOff ){
+        bLastInDoclist = 1;
+      }
+    }
+  }
+
+  if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist||iNextOff==iPgIdx) ){
+    fts5PutU16(&aPg[0], 0);
+  }
+
+  if( bLastInDoclist==0 ){
+    if( iNextOff!=iPgIdx ){
+      iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta);
+      iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
+    }
+  }else if( 
+      iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno 
+  ){
+    /* The entry being removed was the only position list in its
+    ** doclist. Therefore the term needs to be removed as well. */
+    int iKey = 0;
+    for(iIdx=0, iKeyOff=0; iIdx<nIdx; iKey++){
+      u32 iVal = 0;
+      iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+      if( (iKeyOff+iVal)>iStart ) break;
+      iKeyOff += iVal;
+    }
+
+    iDelKeyOff = iOff = iKeyOff;
+    if( iNextOff!=iPgIdx ){
+      int nPrefix = 0;
+      int nSuffix = 0;
+      int nPrefix2 = 0;
+      int nSuffix2 = 0;
+
+      iDelKeyOff = iNextOff;
+      iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2);
+      iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2);
+
+      if( iKey!=1 ){
+        iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix);
+      }
+      iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix);
+
+      nPrefix = MIN(nPrefix, nPrefix2);
+      nSuffix = (nPrefix2 + nSuffix2) - nPrefix;
+
+      if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){
+        p->rc = FTS5_CORRUPT;
+      }else{
+        if( iKey!=1 ){
+          iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix);
+        }
+        iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix);
+        if( nPrefix2>nPrefix ){
+          memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix);
+          iOff += (nPrefix2-nPrefix);
+        }
+        memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2);
+        iOff += nSuffix2;
+        iNextOff += nSuffix2;
+      }
+    }
+  }else if( iStart==4 ){
+      assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno );
+      /* The entry being removed may be the only position list in
+      ** its doclist. */
+      int iPgno = pSeg->iLeafPgno-1;
+
+      for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
+        Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno));
+        int bEmpty = (pPg && pPg->nn==4);
+        fts5DataRelease(pPg);
+        if( bEmpty==0 ) break;
+      }
+
+      if( iPgno==pSeg->iTermLeafPgno ){
+        i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno);
+        Fts5Data *pTerm = fts5DataRead(p, iId);
+        if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
+          u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
+          int nTermIdx = pTerm->nn - pTerm->szLeaf;
+          int iTermIdx = 0;
+          int iTermOff = 0;
+
+          while( 1 ){
+            u32 iVal = 0;
+            int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal);
+            iTermOff += iVal;
+            if( (iTermIdx+nByte)>=nTermIdx ) break;
+            iTermIdx += nByte;
+          }
+          nTermIdx = iTermIdx;
+
+          memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
+          fts5PutU16(&pTerm->p[2], iTermOff);
+
+          fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
+          if( nTermIdx==0 ){
+            fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
+          }
+        }
+        fts5DataRelease(pTerm);
+      }
+    }
+
+    if( p->rc==SQLITE_OK ){
+      const int nMove = nPg - iNextOff;
+      int nShift = 0;
+
+      memmove(&aPg[iOff], &aPg[iNextOff], nMove);
+      iPgIdx -= (iNextOff - iOff);
+      nPg = iPgIdx;
+      fts5PutU16(&aPg[2], iPgIdx);
+
+      nShift = iNextOff - iOff;
+      for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){
+        u32 iVal = 0;
+        iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+        iKeyOff += iVal;
+        if( iKeyOff!=iDelKeyOff ){
+          if( iKeyOff>iOff ){
+            iKeyOff -= nShift;
+            nShift = 0;
+          }
+          nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
+          iPrevKeyOff = iKeyOff;
+        }
+      }
+
+      if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
+        fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
+      }
+
+      assert_nc( nPg>4 || fts5GetU16(aPg)==0 );
+      fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg);
+    }
+    sqlite3_free(aIdx);
+}
 
 /*
 ** This is called as part of flushing a delete to disk in 'secure-delete'
@@ -4706,213 +4949,7 @@ static void fts5FlushSecureDelete(
      && iRowid==fts5MultiIterRowid(pIter)
     ){
       Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
-      int iSegid = pSeg->pSeg->iSegid;
-      u8 *aPg = pSeg->pLeaf->p;
-      int nPg = pSeg->pLeaf->nn;
-      int iPgIdx = pSeg->pLeaf->szLeaf;
-
-      u64 iDelta = 0;
-      u64 iNextDelta = 0;
-      int iNextOff = 0;
-      int iOff = 0;
-      int nMove = 0;
-      int nIdx = 0;
-
-      u8 *aIdx = 0;
-
-      nIdx = nPg-iPgIdx;
-      aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16);
-      if( aIdx ){
-        int bLastInDoclist = 0;
-        int iIdx = 0;
-        int iStart = 0;
-        int iKeyOff = 0;
-        int iPrevKeyOff = 0;
-        int nShift = 0;
-        int iDelKeyOff = 0;       /* Offset of deleted key, if any */
-        memcpy(aIdx, &aPg[iPgIdx], nIdx);
-
-        /* At this point segment iterator pSeg points to the entry
-        ** this function should remove from the b-tree segment. 
-        **
-        ** More specifically, pSeg->iLeafOffset is the offset of the
-        ** first byte in the position-list for the entry to remove.
-        ** Immediately before this comes two varints that will also
-        ** need to be removed:
-        **
-        **     + the rowid or delta rowid value for the entry, and
-        **     + the size of the position list in bytes.
-        */
-        {
-          int iSOP;
-          int nPos = 0;
-          if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
-            iStart = pSeg->iTermLeafOffset;
-          }else{
-            iStart = fts5GetU16(&aPg[0]);
-          }
-
-          iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
-          iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
-          assert_nc( iSOP<=pSeg->iLeafOffset );
-          while( iSOP<pSeg->iLeafOffset ){
-            iStart = iSOP + (nPos/2);
-            iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
-            iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
-          }
-          assert_nc( iSOP==pSeg->iLeafOffset );
-        }
-
-        iOff = iStart;
-        iNextOff = pSeg->iLeafOffset + pSeg->nPos;
-        if( iNextOff>=iPgIdx ){
-          int pgno = pSeg->iLeafPgno+1;
-          fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
-          iNextOff = iPgIdx;
-        }else{
-          /* Set bLastInDoclist to true if the entry being removed is the last
-          ** in its doclist.  */
-          for(iIdx=0, iKeyOff=0; iIdx<nIdx; /* no-op */){
-            u32 iVal = 0;
-            iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
-            iKeyOff += iVal;
-            if( iKeyOff==iNextOff ){
-              bLastInDoclist = 1;
-            }
-          }
-        }
-
-        if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist||iNextOff==iPgIdx) ){
-          fts5PutU16(&aPg[0], 0);
-        }
-
-        if( bLastInDoclist==0 ){
-          if( iNextOff!=iPgIdx ){
-            iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta);
-            iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
-          }
-        }else if( 
-          iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno 
-        ){
-          /* The entry being removed was the only position list in its
-          ** doclist. Therefore the term needs to be removed as well. */
-          int iKey = 0;
-          for(iIdx=0, iKeyOff=0; iIdx<nIdx; iKey++){
-            u32 iVal = 0;
-            iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
-            if( (iKeyOff+iVal)>iStart ) break;
-            iKeyOff += iVal;
-          }
-
-          iDelKeyOff = iOff = iKeyOff;
-          if( iNextOff!=iPgIdx ){
-            int nPrefix = 0;
-            int nSuffix = 0;
-            int nPrefix2 = 0;
-            int nSuffix2 = 0;
-
-            iDelKeyOff = iNextOff;
-            iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2);
-            iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2);
-
-            if( iKey!=1 ){
-              iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix);
-            }
-            iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix);
-
-            nPrefix = MIN(nPrefix, nPrefix2);
-            nSuffix = (nPrefix2 + nSuffix2) - nPrefix;
-
-            if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){
-              p->rc = FTS5_CORRUPT;
-            }else{
-              if( iKey!=1 ){
-                iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix);
-              }
-              iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix);
-              if( nPrefix2>nPrefix ){
-                memcpy(&aPg[iOff], &zTerm[nPrefix], nPrefix2-nPrefix);
-                iOff += (nPrefix2-nPrefix);
-              }
-              memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2);
-              iOff += nSuffix2;
-              iNextOff += nSuffix2;
-            }
-          }
-        }else if( iStart==4 ){
-          assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno );
-          /* The entry being removed may be the only position list in
-          ** its doclist. */
-          int iPgno = pSeg->iLeafPgno-1;
-
-          for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
-            Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno));
-            int bEmpty = (pPg && pPg->nn==4);
-            fts5DataRelease(pPg);
-            if( bEmpty==0 ) break;
-          }
-
-          if( iPgno==pSeg->iTermLeafPgno ){
-            i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno);
-            Fts5Data *pTerm = fts5DataRead(p, iId);
-            if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
-              u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
-              int nTermIdx = pTerm->nn - pTerm->szLeaf;
-              int iTermIdx = 0;
-              int iTermOff = 0;
-
-              while( 1 ){
-                u32 iVal = 0;
-                int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal);
-                iTermOff += iVal;
-                if( (iTermIdx+nByte)>=nTermIdx ) break;
-                iTermIdx += nByte;
-              }
-              nTermIdx = iTermIdx;
-
-              memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
-              fts5PutU16(&pTerm->p[2], iTermOff);
-
-              fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
-              if( nTermIdx==0 ){
-                fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
-              }
-            }
-            fts5DataRelease(pTerm);
-          }
-        }
-
-        if( p->rc==SQLITE_OK ){
-          nMove = nPg - iNextOff;
-          memmove(&aPg[iOff], &aPg[iNextOff], nMove);
-          iPgIdx -= (iNextOff - iOff);
-          nPg = iPgIdx;
-          fts5PutU16(&aPg[2], iPgIdx);
-
-          nShift = iNextOff - iOff;
-          for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){
-            u32 iVal = 0;
-            iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
-            iKeyOff += iVal;
-            if( iKeyOff!=iDelKeyOff ){
-              if( iKeyOff>iOff ){
-                iKeyOff -= nShift;
-                nShift = 0;
-              }
-              nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
-              iPrevKeyOff = iKeyOff;
-            }
-          }
-
-          if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
-            fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
-          }
-
-          assert_nc( nPg>4 || fts5GetU16(aPg)==0 );
-          fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg);
-        }
-        sqlite3_free(aIdx);
-      }
+      fts5DoSecureDelete(p, pStruct, pSeg);
     }
   }
 
@@ -4998,11 +5035,24 @@ static void fts5FlushOneHash(Fts5Index *p){
           /* If in secure delete mode, and if this entry in the poslist is
           ** in fact a delete, then edit the existing segments directly
           ** using fts5FlushSecureDelete().  */
-          if( bSecureDelete && (pDoclist[iOff] & 0x01) ){
-            fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
-            if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){
-              iOff++;
-              continue;
+          if( bSecureDelete ){
+            if( eDetail==FTS5_DETAIL_NONE ){
+              if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
+                fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
+                iOff++;
+                if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
+                  iOff++;
+                  nDoclist = 0;
+                }else{
+                  continue;
+                }
+              }
+            }else if( (pDoclist[iOff] & 0x01) ){
+              fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
+              if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){
+                iOff++;
+                continue;
+              }
             }
           }
 
diff --git a/ext/fts5/test/fts5secure4.test b/ext/fts5/test/fts5secure4.test
index 7cfc5a73e9..7588a34683 100644
--- a/ext/fts5/test/fts5secure4.test
+++ b/ext/fts5/test/fts5secure4.test
@@ -90,18 +90,20 @@ do_catchsql_test 1.12 {
 foreach {tn d} {1 full 2 none 3 column} {
   reset_db
   do_execsql_test 2.$tn.1 "
-    CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='1,2');
+    CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='10,20');
     INSERT INTO x1(x1, rank) VALUES('pgsz', 64);
+    INSERT INTO x1(x1, rank) VALUES('secure-delete', 1);
   "
 
   do_execsql_test 2.$tn.2 {
     BEGIN;
-      INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
-      INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
-      INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
-      INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
-      INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
+      INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c');
     COMMIT;
+    INSERT INTO x1(x1) VALUES('integrity-check');
   }
 
   do_execsql_test 2.$tn.3 {
diff --git a/ext/fts5/test/fts5secure5.test b/ext/fts5/test/fts5secure5.test
new file mode 100644
index 0000000000..ca8570211c
--- /dev/null
+++ b/ext/fts5/test/fts5secure5.test
@@ -0,0 +1,129 @@
+# 2023 April 14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+return_if_no_fts5
+set ::testprefix fts5secure5
+return_if_no_fts5
+
+proc dump {} {
+  execsql_pp {
+    SELECT id, quote(block), fts5_decode_none(id, block) FROM ft1_data
+  }
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none);
+  INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 1.1 {
+  BEGIN;
+    INSERT INTO ft1(rowid, a) VALUES(1, 'abcd');
+    INSERT INTO ft1(rowid, a) VALUES(2, 'abcd');
+    INSERT INTO ft1(rowid, a) VALUES(3, 'abcd');
+  COMMIT;
+}
+do_execsql_test 1.2 {
+  DELETE FROM ft1 WHERE rowid=1;
+}
+do_execsql_test 1.3 {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+do_execsql_test 1.4 {
+  DELETE FROM ft1 WHERE rowid=3;
+}
+do_execsql_test 1.5 {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+do_execsql_test 1.6 {
+  DELETE FROM ft1 WHERE rowid=3;
+}
+do_execsql_test 1.7 {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none);
+  INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 2.1 {
+  BEGIN;
+    INSERT INTO ft1(rowid, a) VALUES(1, 'abcd one');
+    INSERT INTO ft1(rowid, a) VALUES(2, 'abcd two');
+    INSERT INTO ft1(rowid, a) VALUES(3, 'abcd two');
+    INSERT INTO ft1(rowid, a) VALUES(4, 'abcd two');
+    INSERT INTO ft1(rowid, a) VALUES(5, 'abcd three');
+  COMMIT;
+}
+
+do_execsql_test 2.2a {
+  DELETE FROM ft1 WHERE rowid=3;
+}
+do_execsql_test 2.2b {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+do_execsql_test 2.3a {
+  DELETE FROM ft1 WHERE rowid=2;
+}
+do_execsql_test 2.3b {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+do_execsql_test 2.4a {
+  DELETE FROM ft1 WHERE rowid=4;
+}
+do_execsql_test 2.4b {
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none, prefix=1);
+  INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
+  INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
+}
+do_execsql_test 3.1 {
+  BEGIN;
+    INSERT INTO ft1(a) VALUES('c');
+  COMMIT;
+}
+do_execsql_test 3.2 {
+  DELETE FROM ft1 WHERE rowid IN (1);
+  INSERT INTO ft1(ft1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none);
+  INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1);
+  INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
+
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500
+  )
+  INSERT INTO ft1 SELECT 'abcdefg' FROM s;
+}
+
+do_test 4.1 {
+  for {set i 500} {$i > 0} {incr i -1} {
+    execsql { DELETE FROM ft1 WHERE rowid=$i }
+    execsql { INSERT INTO ft1(ft1) VALUES('integrity-check') }
+  }
+} {}
+
+finish_test
+
diff --git a/manifest b/manifest
index 26932a3817..010a7b6132 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sproblems\shandling\scorrupt\srecords\sin\snew\sfts5\scode.
-D 2023-04-17T11:41:42.438
+C Fix\ssome\sproblems\swith\susing\sfts5\soptions\s'secure-delete'\sand\sdetail=none\stogether.
+D 2023-04-17T18:32:22.329
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd
 F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989
 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982
-F ext/fts5/fts5_index.c 13c4ea17f9e6b155d3c6d78b54ae4e5ed22e1692041958f3ec71a5712172d915
+F ext/fts5/fts5_index.c 2a59602a7ea830ae40b742277f4d89cae60a646bb2fb9c2ce895aea5e7ca89ce
 F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d
 F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -190,7 +190,8 @@ F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e33
 F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc77157d874154bda64
 F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc
 F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610
-F ext/fts5/test/fts5secure4.test 58638eab7e80571649d08673e7924d85ec293566377a9e7f49160857f001d529
+F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97
+F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a
 F ext/fts5/test/fts5securefault.test 7208daed4171de8a54a293ef800411e3cec1ffc6e9d34209b53e56a70ee554da
 F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b
 F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0
@@ -2057,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e8de2774463c07b76cd6b9a44ab32a583fe973f42a21b21af46fe82ab177c1b8
-R e73db0b2fec49cf21caca2b40145b9f4
+P 2f63d4a8bac6dc72197df32efd84c148adbc4794943088dff08da57b15dd7900
+R 94f7d6e08b83dc7ea2e9a235119af696
 U dan
-Z 8d23b3253d2a52bc5cc6c258ff86d3c2
+Z 7915ab356b23565aa915e2ae0ad516a8
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 5bab2e43f5..f88d210269 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2f63d4a8bac6dc72197df32efd84c148adbc4794943088dff08da57b15dd7900
\ No newline at end of file
+4d3f27ba90aa59837e49ceaae9f36cad426af3a33707d85d1bb848fc1bda6fed
\ No newline at end of file