From: dan <Dan Kennedy>
Date: Tue, 5 Sep 2023 15:43:05 +0000 (+0000)
Subject: Add debugging routines to check each page for corruption after it it is manipulated... 
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fheads%2Ffts5-secure-delete-debug;p=thirdparty%2Fsqlite.git

Add debugging routines to check each page for corruption after it it is manipulated by the fts5 secure-delete code, and to log said page via sqlite3_log() if it is found to be corrupt.

FossilOrigin-Name: da4b7385a0d866e61cd5b037a8873acdb5f20e1535e0d5e6b189c7d0e3389d52
---

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index aa16b0174e..4c1910676a 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4929,6 +4929,155 @@ static void fts5SecureDeleteIdxEntry(
   }
 }
 
+/*
+** Buffer aData[], which is nData bytes in size, contains an fts5 leaf page.
+** Call sqlite3_log() with error code SQLITE_CORRUPT a bunch of times to log
+** the entire contents of the page.
+*/
+static void fts5SDLogPage(const u8 *aData, int nData){
+  int iOff;
+  for(iOff=0; iOff<nData; iOff+=16){
+    int iChar;
+    sqlite3_str *pStr = sqlite3_str_new(0);
+    char *zStr = 0;
+
+    sqlite3_str_appendf(pStr, "0x%.4x: ", iOff);
+    for(iChar=iOff; iChar<iOff+16; iChar++){
+      if( iChar<nData ){
+        sqlite3_str_appendf(
+            pStr, "%.2X%s", (int)aData[iChar], (iChar&0x01?" ":"")
+        );
+      }else{
+        sqlite3_str_appendf(pStr, "%s%s", "  ", (iChar&0x01?" ":""));
+      }
+    }
+    for(iChar=iOff; iChar<MIN(nData, iOff+16); iChar++){
+      u8 c = aData[iChar];
+      if( c>=0x7F || c<=0x20 ){
+        c = 0x2E; /* '.' */
+      }
+      sqlite3_str_appendf(pStr, "%c", (char)c);
+    }
+
+    zStr = sqlite3_str_finish(pStr);
+    sqlite3_log(SQLITE_CORRUPT, "%s", zStr);
+    sqlite3_free(zStr);
+  }
+}
+
+/*
+** Log both the new version (buffer pData, nData bytes in size) and the
+** old version (rowid iRowid on disk) via sqlite3_log(). And a header
+** line with information from parameters pSeg and eCall.
+*/
+static void fts5SDLogCorruptPage(
+  Fts5Index *p, 
+  Fts5SegIter *pSeg,
+  int eCall,
+  i64 iRowid, 
+  const u8 *pData,                /* Buffer containing new, corrupt, page */
+  int nData                       /* Size of buffer in bytes */
+){
+  char *zStr = 0;
+  sqlite3_str *pStr = 0;
+  Fts5Data *pOrig = 0;
+  pOrig = fts5LeafRead(p, iRowid);
+
+  pStr = sqlite3_str_new(0);
+
+  sqlite3_str_appendf(pStr, "fts5 secure-delete corruption(%d)", eCall);
+  if( pSeg ){
+    sqlite3_str_appendf(
+        pStr, "(iLeafOffset=%d, nPos=%d)", pSeg->iLeafOffset, pSeg->nPos
+    );
+  }
+  zStr = sqlite3_str_finish(pStr);
+  sqlite3_log(SQLITE_CORRUPT, "%s", zStr);
+  sqlite3_free(zStr);
+  
+  sqlite3_log(SQLITE_CORRUPT, "Original Page:");
+  fts5SDLogPage(pOrig->p, pOrig->nn);
+
+  sqlite3_log(SQLITE_CORRUPT, "Corrupt Page:");
+  fts5SDLogPage(pData, nData);
+}
+
+/*
+** Check that the position list that begins at offset iPos of buffer
+** aData[] looks like it might end at offset iKeyOff. If so, return 0.
+** Otherwise, if the position-list cannot end at offset iKeyOff, 
+** return 1.
+*/
+static int fts5SDCheckPoslistEnds(const u8 *aData, int iPos, int iKeyOff){
+  u64 iDelta = 0;
+  u32 iVal = 0;
+  int iOff = iPos;
+
+  while( iOff<iKeyOff ){
+    iOff += fts5GetVarint(&aData[iOff], &iDelta);
+    iOff += fts5GetVarint32(&aData[iOff], iVal);
+    iOff += (iVal/2);
+    if( iOff>iKeyOff ){
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+/*
+** Debugging wrapper around fts5DataWrite() used by secure-delete code when
+** modifying leaf pages. It checks whether or not the page about to be written
+** looks corrupt, and calls sqlite3_log() with lots of diagnostic information
+** if it does.
+*/
+static void fts5SDDataWrite(
+  Fts5Index *p, 
+  Fts5SegIter *pSeg,
+  int eCall,
+  i64 iRowid, 
+  const u8 *pData, 
+  int nData
+){
+  if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){
+    int bNotFirst = 0;
+    int iPos = 0;
+    int iPgidx = 0;
+    const u8 *aIdx = 0;
+    int nIdx = 0;
+    int iIdx = 0;
+    int iKeyOff = 0;
+
+    iPos = fts5GetU16(&pData[0]);
+    iPgidx = fts5GetU16(&pData[2]);
+
+    nIdx = nData - iPgidx;
+    aIdx = &pData[iPgidx];
+
+    while( iIdx<nIdx ){
+      u32 iVal = 0;
+      iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+      iKeyOff += iVal;
+      if( iPos ){
+        if( fts5SDCheckPoslistEnds(pData, iPos, iKeyOff) ){
+          fts5SDLogCorruptPage(p, pSeg, eCall, iRowid, pData, nData);
+          p->rc = FTS5_CORRUPT;
+          return;
+        }
+      }
+      iPos = iKeyOff;
+      if( bNotFirst ){
+        iPos += fts5GetVarint32(&pData[iKeyOff], iVal);
+      }
+      iPos += fts5GetVarint32(&pData[iPos], iVal);
+      iPos += iVal;
+      bNotFirst = 1;
+    }
+  }
+
+  fts5DataWrite(p, iRowid, pData, nData);
+}
+
 /*
 ** This is called when a secure-delete operation removes a position-list
 ** that overflows onto segment page iPgno of segment pSeg. This function
@@ -5026,13 +5175,14 @@ static void fts5SecureDeleteOverflow(
 
       /* Write the new page to disk and exit the loop */
       assert( nPg>4 || fts5GetU16(aPg)==0 );
-      fts5DataWrite(p, iRowid, aPg, nPg);
+      fts5SDDataWrite(p, 0, 0, iRowid, aPg, nPg);
       break;
     }
   }
   fts5DataRelease(pLeaf);
 }
 
+
 /*
 ** Completely remove the entry that pSeg currently points to from 
 ** the database.
@@ -5262,7 +5412,7 @@ static void fts5DoSecureDelete(
         memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
         fts5PutU16(&pTerm->p[2], iTermOff);
 
-        fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
+        fts5SDDataWrite(p, pSeg, 1, iId, pTerm->p, iTermOff+nTermIdx);
         if( nTermIdx==0 ){
           fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
         }
@@ -5301,7 +5451,7 @@ static void fts5DoSecureDelete(
     }
 
     assert_nc( nPg>4 || fts5GetU16(aPg)==0 );
-    fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nPg);
+    fts5SDDataWrite(p, pSeg, 2, FTS5_SEGMENT_ROWID(iSegid, pSeg->iLeafPgno), aPg, nPg);
   }
   sqlite3_free(aIdx);
 }
diff --git a/manifest b/manifest
index 24fefe807a..d65f1032ca 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sfew\sSQLITE_MISUSE\sreturns\sso\sthat\sthey\scall\ssqlite3MisuseError().
-D 2023-09-05T15:03:23.029
+C Add\sdebugging\sroutines\sto\scheck\seach\spage\sfor\scorruption\safter\sit\sit\sis\smanipulated\sby\sthe\sfts5\ssecure-delete\scode,\sand\sto\slog\ssaid\spage\svia\ssqlite3_log()\sif\sit\sis\sfound\sto\sbe\scorrupt.
+D 2023-09-05T15:43:05.156
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -94,7 +94,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
 F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6
 F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
-F ext/fts5/fts5_index.c 47c290589fa38f6a1860b4fc26716ed30d79ddc283b38813d1c8c1d702108ab8
+F ext/fts5/fts5_index.c df64d8f5ccfdd5ef3fe1b2f5c5ccec5fba49e569453fb044cdb59c9147c68550
 F ext/fts5/fts5_main.c 7070031993ba5b5d89b13206ec4ef624895f2f7c0ec72725913d301e4d382445
 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
@@ -2116,8 +2116,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b12afff4efe4de84388d102060e8c312abd2f9eca8270b3c0f01ac6d1f2e329a
-R c33ff09bfe39aecbb90b639f6158696c
-U drh
-Z 799fbc495b6d502c0a1ef040c8c3f7ec
+P 93f74490faf8cc07e107afdab6737c6e5141ae1f01a05142bfcede2dd1b2ba4e
+R dc5f859705ed941ece53becdd24b1f2d
+T *branch * fts5-secure-delete-debug
+T *sym-fts5-secure-delete-debug *
+T -sym-trunk *
+U dan
+Z dd8b7a29381f854b929481e9966446fa
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 98d9d4265c..7b0c609ff1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-93f74490faf8cc07e107afdab6737c6e5141ae1f01a05142bfcede2dd1b2ba4e
\ No newline at end of file
+da4b7385a0d866e61cd5b037a8873acdb5f20e1535e0d5e6b189c7d0e3389d52
\ No newline at end of file