More improvements to shadow table corruption detection in FTS3.

FossilOrigin-Name: 51525f9c3235967bc00a090e84c70a6400698c897aa4742e817121c725b8c99d

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index f90c7a56f0b83ca3280189e36b5d959fec6b1d52..e8e909a97ba2b41e3ac29defc1edb6f2a006fca9 100644 (file)
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -1486,6 +1486,10 @@ static int fts3InitVtab(
   fts3DatabasePageSize(&rc, p);
   p->nNodeSize = p->nPgsz-35;
 
+#if defined(SQLITE_DEBUG)||defined(SQLITE_TEST)
+  p->nMergeCount = FTS3_MERGE_COUNT;
+#endif
+
   /* Declare the table schema to SQLite. */
   fts3DeclareVtab(&rc, p);
 

diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h
index 8aebfcd38765ad6ab8f871da040841fa02e0a127..3c58e49583d2600c119ff927d6dbe737fcb04582 100644 (file)
--- a/ext/fts3/fts3Int.h
+++ b/ext/fts3/fts3Int.h
@@ -301,9 +301,19 @@ struct Fts3Table {
   /* True to disable the incremental doclist optimization. This is controled
   ** by special insert command 'test-no-incr-doclist'.  */
   int bNoIncrDoclist;
+
+  /* Number of segments in a level */
+  int nMergeCount;
 #endif
 };
 
+/* Macro to find the number of segments to merge */
+#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
+# define MergeCount(P) ((P)->nMergeCount)
+#else
+# define MergeCount(P) FTS3_MERGE_COUNT
+#endif
+
 /*
 ** When the core wants to read from the virtual table, it creates a
 ** virtual table cursor (an instance of the following structure) using

diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index 658d0d214f82c3fb5a84e633346f5c582b07392c..7d6f365b17ecdf0903e1b756be7fcf4a52573b0c 100644 (file)
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -1153,7 +1153,7 @@ static int fts3AllocateSegdirIdx(
     ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
     ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
     */
-    if( iNext>=FTS3_MERGE_COUNT ){
+    if( iNext>=MergeCount(p) ){
       fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel));
       rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel);
       *piIdx = 0;
@@ -4280,6 +4280,10 @@ static int fts3IncrmergeLoad(
       int i;
       int nHeight = (int)aRoot[0];
       NodeWriter *pNode;
+      if( nHeight<1 || nHeight>FTS_MAX_APPENDABLE_HEIGHT ){
+        sqlite3_reset(pSelect);
+        return FTS_CORRUPT_VTAB;
+      }
 
       pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT;
       pWriter->iStart = iStart;
@@ -5048,7 +5052,7 @@ static int fts3DoIncrmerge(
   const char *zParam              /* Nul-terminated string containing "A,B" */
 ){
   int rc;
-  int nMin = (FTS3_MERGE_COUNT / 2);
+  int nMin = (MergeCount(p) / 2);
   int nMerge = 0;
   const char *z = zParam;
 
@@ -5093,7 +5097,7 @@ static int fts3DoAutoincrmerge(
   int rc = SQLITE_OK;
   sqlite3_stmt *pStmt = 0;
   p->nAutoincrmerge = fts3Getint(&zParam);
-  if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){
+  if( p->nAutoincrmerge==1 || p->nAutoincrmerge>MergeCount(p) ){
     p->nAutoincrmerge = 8;
   }
   if( !p->bHasStat ){
@@ -5384,6 +5388,10 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
     }else if( nVal>21 && 0==sqlite3_strnicmp(zVal,"test-no-incr-doclist=",21) ){
       p->bNoIncrDoclist = atoi(&zVal[21]);
       rc = SQLITE_OK;
+    }else if( nVal>11 && 0==sqlite3_strnicmp(zVal,"mergecount=",11) ){
+      v = atoi(&zVal[11]);
+      if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v;
+      rc = SQLITE_OK;
     }
 #endif
   }

diff --git a/manifest b/manifest
index 2585239f616f0cb46f91a7d048cdc47f6905d004..5ac9aeea34f5a96cd1bce5271bf4c46bdd7a1aa0 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\sdetection\sof\scorrupt\sshadow\stables\sin\sFTS3.\s\sEnable\sthe\sdebugging\nspecial-inserts\sfor\sFTS3\sfor\sboth\sSQLITE_DEBUG\sand\sSQLITE_TEST.
-D 2019-11-16T21:40:49.550
+C More\simprovements\sto\sshadow\stable\scorruption\sdetection\sin\sFTS3.
+D 2019-11-16T23:47:40.269
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -81,9 +81,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 6184d5748b51301f401eda0d3e5ad0caa19b4b95c0520cdd1f83a74e3a1c97dc
+F ext/fts3/fts3.c ad760e49a68d13c11f611d7532d7f2277924df6d4a43cbaf8acf7693e1e83aab
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h b879664f543ce352815461cacd828f2606716e46e47323875eddfe100d8b7711
+F ext/fts3/fts3Int.h 31f2ceee083d93254d84ab7a69eddb253e079bfe7e35c15413c0132f8947d6b6
 F ext/fts3/fts3_aux.c 96708c8b3a7d9b8ca1b68ea2b7e503e283f20e95f145becadedfad096dbd0f34
 F ext/fts3/fts3_expr.c b132af223e90e35b9f9efa9fe63d6ae737d34153a3b6066736086df8abc78a1f
 F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7
@@ -99,7 +99,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d
 F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f
-F ext/fts3/fts3_write.c f7d443ebe2cdf0a8515edb7588c8aab77108cc28ede9ced8baa416c95aa810cb
+F ext/fts3/fts3_write.c 84f7b1390b85e6630d46fc1162f6138ce08aabcf9d57289db8fa1e04602cd3e9
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
@@ -1016,7 +1016,7 @@ F test/fuzzdata4.db b502c7d5498261715812dd8b3c2005bad08b3a26e6489414bd13926cd3e4
 F test/fuzzdata5.db e35f64af17ec48926481cfaf3b3855e436bd40d1cfe2d59a9474cb4b748a52a5
 F test/fuzzdata6.db 92a80e4afc172c24f662a10a612d188fb272de4a9bd19e017927c95f737de6d7
 F test/fuzzdata7.db e7a86fd83dda151d160445d542e32e5c6019c541b3a74c2a525b6ac640639711
-F test/fuzzdata8.db 053981588014b4e1e5bc8beaf7034671e133cee853a0f2dd9c8d444be1a97fe5
+F test/fuzzdata8.db 1490d840640e5ace4ec8e10e252681497fd7d5ffbb6b783bd3899fa789698226
 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8
 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14
 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536
@@ -1849,7 +1849,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bae76a5c40703871e5ce4cd23d6fae5a3836606f524a63b01ac828c7a602c5e9
-R b5d91364fca42dbea07f58830b5ec589
+P 04b2873be5aedeb1c4325cf36c4b5d180f929a641caf1e3829c03778adb29c8e
+R c4135a72f080b6416943cb56e5f0bed9
 U drh
-Z baf295687228d1300e4112b17540730f
+Z 628c9e8e0d5353ed3a948a9f5a3cad9a

diff --git a/manifest.uuid b/manifest.uuid
index 43f64260cac8df4c1e6b127728abc58da99074a6..40ad12833714117092526020bfefbe106cf9bbfd 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-04b2873be5aedeb1c4325cf36c4b5d180f929a641caf1e3829c03778adb29c8e
\ No newline at end of file
+51525f9c3235967bc00a090e84c70a6400698c897aa4742e817121c725b8c99d
\ No newline at end of file

diff --git a/test/fuzzdata8.db b/test/fuzzdata8.db
index 52385464373000b833e8408fa906f902aa67446b..7b4a3d38545776438502730d78b2e37afc8217aa 100644 (file)
Binary files a/test/fuzzdata8.db and b/test/fuzzdata8.db differ