In fts3, avoid making a very large memory allocation if the merge-hint record is...

FossilOrigin-Name: 9add58fe9688d5c1f675736e7f1c6ff7831d2ff6fdeb884a2e62c1bf9536ee50

diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index 203b48e2d1b6d52cde2228f04e4f4f566cacfb19..197aebdd499754fa9e4ece381f08d9e61e3b164f 100644 (file)
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -335,7 +335,7 @@ static int fts3SqlStmt(
 ** returns zero rows.  */
 /* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' "
          "  GROUP BY level HAVING cnt>=?"
-         "  ORDER BY (level %% 1024) ASC LIMIT 1",
+         "  ORDER BY (level %% 1024) ASC, 2 DESC LIMIT 1",
 
 /* Estimate the upper limit on the number of leaf nodes in a new segment
 ** created by merging the oldest :2 segments from absolute level :1. See 
@@ -4931,8 +4931,14 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
 
       rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg);
       if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){
+        /* Based on the scan in the block above, it is known that there
+        ** are no levels with a relative level smaller than that of
+        ** iAbsLevel with more than nSeg segments, or if nSeg is -1, 
+        ** no levels with more than nMin segments. Use this to limit the
+        ** value of nHintSeg to avoid a large memory allocation in case the 
+        ** merge-hint is corrupt*/
         iAbsLevel = iHintAbsLevel;
-        nSeg = nHintSeg;
+        nSeg = MIN(MAX(nMin,nSeg), nHintSeg);
         bUseHint = 1;
         bDirtyHint = 1;
       }else{
@@ -4945,7 +4951,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
     /* If nSeg is less that zero, then there is no level with at least
     ** nMin segments and no hint in the %_stat table. No work to do.
     ** Exit early in this case.  */
-    if( nSeg<0 ) break;
+    if( nSeg<=0 ) break;
 
     /* Open a cursor to iterate through the contents of the oldest nSeg 
     ** indexes of absolute level iAbsLevel. If this cursor is opened using 

diff --git a/manifest b/manifest
index 5b6e4d66c320ba784098dc5de531319511b8ab2b..1e35dea297c8f298333672ce0fbfc2e8adf65960 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sNEVER\smacros\sto\serror-detection\sbranches\sthat\swere\smade\sunreachable\n(as\sfar\sas\sI\scan\stell)\sby\scheck-in\s[ceacc28b03580334].
-D 2020-01-03T17:40:30.845
+C In\sfts3,\savoid\smaking\sa\svery\slarge\smemory\sallocation\sif\sthe\smerge-hint\srecord\sis\scorrupt.
+D 2020-01-03T20:16:32.201
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -99,7 +99,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d
 F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f
-F ext/fts3/fts3_write.c 51e0a4e3782ee17b6dd5e89949b7095fc98e36e87725c53de631734535507498
+F ext/fts3/fts3_write.c 6f9dd5d774003ea81b8b32daa7d0819f9aaf01bf2b5f33498a69aab096094ed3
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
@@ -940,7 +940,7 @@ F test/fts3conf.test c84bbaec81281c1788aa545ac6e78a6bd6cde2bdbbce2da261690e3659f
 F test/fts3corrupt.test ce7f7b5eaeee5f1804584d061b978d85e64abf2af9adaa7577589fac6f7eae01
 F test/fts3corrupt2.test bf55c3fa0b0dc8ea1c0fe5543623bd27714585da6a129038fd6999fe3b0d25f3
 F test/fts3corrupt3.test 0d5b69a0998b4adf868cc301fc78f3d0707745f1d984ce044c205cdb764b491f
-F test/fts3corrupt4.test 50f8e119b688c8b8aa46c1543a022e0e1a931f5820c7da099b34e9acea7ece3b
+F test/fts3corrupt4.test e407c7b4f4cd3335080833aff3a8855d520e531b79f84dcc77be4623af2342d4
 F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5
 F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf
 F test/fts3d.test 2bd8c97bcb9975f2334147173b4872505b6a41359a4f9068960a36afe07a679f
@@ -1853,7 +1853,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 523e09bd228750fb7be1d4e68d18f6c1bf9db19293d66fbf30b86409321e9814
-R 9db325677ae78118554163eac3ed2eae
-U drh
-Z e0ea28a1a8dedd28fad05208e3f0151a
+P 7b62555e285f32d354df7746ef8dce16856c6d9d53a02d895b74ba02362d2968
+R 7f27ee6576dc13efabadbee4643d1518
+U dan
+Z 162e3132aa65eb8605e2f1b28bf07203

diff --git a/manifest.uuid b/manifest.uuid
index e4bdd27cd6d944988b4f824c04bcba70cd39c7b5..778ba3f4cecf47ef9470830419932a1e867a60ee 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7b62555e285f32d354df7746ef8dce16856c6d9d53a02d895b74ba02362d2968
\ No newline at end of file
+9add58fe9688d5c1f675736e7f1c6ff7831d2ff6fdeb884a2e62c1bf9536ee50
\ No newline at end of file

diff --git a/test/fts3corrupt4.test b/test/fts3corrupt4.test
index c4cfd60d93df181394699b8bb6f76801dc75e892..7404ddeb2c0f5cc714b85be996266c5799456e2b 100644 (file)
--- a/test/fts3corrupt4.test
+++ b/test/fts3corrupt4.test
@@ -5812,4 +5812,13 @@ do_catchsql_test 35.1 {
   INSERT INTO f(f) VALUES ('integrity-check');
 } {1 {database disk image is malformed}}
 
+reset_db
+do_catchsql_test 36.0 {
+  CREATE VIRTUAL TABLE f USING fts3(a,tokenize=porter);
+  CREATE TABLE 'f_stat'(id INTEGER PRIMARY KEY, value BLOB);
+  INSERT INTO f VALUES (1);
+  INSERT INTO f_stat VALUES (1,x'00000000000101010119013d00ffff0400fa83717b71a69297979701f63d010101010101010101010101190000000000000000fa83717b71a601f63d01010101010101010101010119013d00ffffff0400fa83717b71a69297979701f63d010101010101010101010101190000000000000000fa83717b71a69201f63d010101f63d01010101010101010101010119013d00ffffff0400fa83717b71a6929797010101010101010101010119013d00ffff01f63d01010101010101010101010119013d00ffffff0400fa83717b71a69297979701f63d00fa03ffffffa69297979701f63d010101000000000101010101197e9797976567656565ffa63535354e');
+  INSERT INTO f(f) VALUES ('merge=53,216');
+} {0 {}}
+
 finish_test