From: drh <>
Date: Mon, 8 Jun 2026 19:23:56 +0000 (+0000)
Subject: Fix four obscure yet separate problems in FTS5.
X-Git-Tag: release~46
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=78a50f9e743a7cfe4846992102707f909edd8708;p=thirdparty%2Fsqlite.git

Fix four obscure yet separate problems in FTS5.

FossilOrigin-Name: a6eb055c35f091a8e864c1de3387d389e2bd3025c32596c0b7a78ebdf091898b
---

diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index ee43ca6cca..c9894f9fc3 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -426,7 +426,7 @@ static void fts5SnippetFunction(
   int rc = SQLITE_OK;             /* Return code */
   int iCol;                       /* 1st argument to snippet() */
   const char *zEllips;            /* 4th argument to snippet() */
-  int nToken;                     /* 5th argument to snippet() */
+  i64 nToken;                     /* 5th argument to snippet() */
   int nInst = 0;                  /* Number of instance matches this row */
   int i;                          /* Used to iterate through instances */
   int nPhrase;                    /* Number of phrases in query */
@@ -451,7 +451,7 @@ static void fts5SnippetFunction(
   ctx.zClose = fts5ValueToText(apVal[2]);
   ctx.iRangeEnd = -1;
   zEllips = fts5ValueToText(apVal[3]);
-  nToken = sqlite3_value_int(apVal[4]);
+  nToken = (int)(MIN( MAX(sqlite3_value_int64(apVal[4]), 0), 64));
 
   iBestCol = (iCol>=0 ? iCol : 0);
   nPhrase = pApi->xPhraseCount(pFts);
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 8ecaca34fe..8dc01bc0c2 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -807,7 +807,7 @@ static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){
       i64 iPos = a[i].reader.iPos;
       Fts5PoslistWriter *pWriter = &a[i].writer;
       if( a[i].pOut->n==0 || iPos!=pWriter->iPrev ){
-        sqlite3Fts5PoslistWriterAppend(a[i].pOut, pWriter, iPos);
+        sqlite3Fts5PoslistSafeAppend(a[i].pOut, &pWriter->iPrev, iPos);
       }
     }
 
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index ce55337022..db865379e0 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -2122,7 +2122,7 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
   while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){
     Fts5Data *pNew;
     pIter->iLeafPgno--;
-    pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(
+    pNew = fts5LeafRead(p, FTS5_SEGMENT_ROWID(
           pIter->pSeg->iSegid, pIter->iLeafPgno
     ));
     if( pNew ){
@@ -8570,7 +8570,7 @@ static void fts5IndexIntegrityCheckSegment(
         /* Check any rowid-less pages that occur before the current leaf. */
         for(iPg=iPrevLeaf+1; iPg<fts5DlidxIterPgno(pDlidx); iPg++){
           iKey = FTS5_SEGMENT_ROWID(iSegid, iPg);
-          pLeaf = fts5DataRead(p, iKey);
+          pLeaf = fts5LeafRead(p, iKey);
           if( pLeaf ){
             if( fts5LeafFirstRowidOff(pLeaf)!=0 ) FTS5_CORRUPT_ROWID(p, iKey);
             fts5DataRelease(pLeaf);
@@ -8581,7 +8581,7 @@ static void fts5IndexIntegrityCheckSegment(
         /* Check that the leaf page indicated by the iterator really does
         ** contain the rowid suggested by the same. */
         iKey = FTS5_SEGMENT_ROWID(iSegid, iPrevLeaf);
-        pLeaf = fts5DataRead(p, iKey);
+        pLeaf = fts5LeafRead(p, iKey);
         if( pLeaf ){
           i64 iRowid;
           int iRowidOff = fts5LeafFirstRowidOff(pLeaf);
diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test
index 9c95ef2daa..34a9d767e9 100644
--- a/ext/fts5/test/fts5af.test
+++ b/ext/fts5/test/fts5af.test
@@ -191,6 +191,29 @@ do_execsql_test 5.6 {
   SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x OR ' || x'DB');
 } {{[x a a a a a...}}
 
+do_execsql_test 5.7.1 {
+  SELECT snippet(p1, 0, '[', NULL, '...', -2147483647) FROM p1('x OR ' || x'DB')
+} {{[x...}}
+do_execsql_test 5.7.2 {
+  SELECT snippet(p1, 0, '[', NULL, '...', -2147483648) FROM p1('x OR ' || x'DB')
+} {{[x...}}
+do_execsql_test 5.7.3 {
+  SELECT snippet(p1, 0, '[', NULL, '...', -2147483649) FROM p1('x OR ' || x'DB')
+} {{[x...}}
+do_execsql_test 5.7.4 {
+  SELECT snippet(p1, 0, '[', NULL, '...', 0) FROM p1('x OR ' || x'DB')
+} {{[x...}}
+
+do_execsql_test 5.8.1 {
+  SELECT snippet(p1, 0, '[', NULL, '...', 2147483647) FROM p1('x OR ' || x'DB')
+} {{[x a a a a a a a a a a}}
+do_execsql_test 5.8.2 {
+  SELECT snippet(p1, 0, '[', NULL, '...', 2147483648) FROM p1('x OR ' || x'DB')
+} {{[x a a a a a a a a a a}}
+do_execsql_test 5.8.3 {
+  SELECT snippet(p1, 0, '[', NULL, '...', 2147483649) FROM p1('x OR ' || x'DB')
+} {{[x a a a a a a a a a a}}
+
 } ;# foreach_detail_mode 
 
 reset_db
diff --git a/ext/fts5/test/fts5corruptA.test b/ext/fts5/test/fts5corruptA.test
index dc6a2df40b..f44e9a9457 100644
--- a/ext/fts5/test/fts5corruptA.test
+++ b/ext/fts5/test/fts5corruptA.test
@@ -124,5 +124,122 @@ do_test 3.1 {
   set {} {}
 } {}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(content);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  BEGIN;
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<5000
+    ) 
+    INSERT INTO t1(content) SELECT 'xyzterm' FROM s;
+  COMMIT;
+  INSERT INTO t1(t1) VALUES('optimize');
+}
+
+do_test 4.1 {
+  db eval {
+    SELECT rowid AS a, hex(block) AS block FROM t1_data 
+    WHERE ((rowid>>36) & 0x01)==1
+      AND ((rowid>>31) & 0x1F)==0
+  } {
+    set n [string length $block]
+    set block [string range $block 0 11]
+    append block [string repeat "00" [expr ($n/2)-7-2]]
+    append block "8080"
+    db eval {
+      UPDATE t1_data SET block = unhex($block) WHERE rowid = $a
+    }
+  }
+} {}
+
+do_catchsql_test 4.2 {
+  SELECT count(*) FROM (
+      SELECT rowid FROM t1 WHERE t1 MATCH 'xyzterm' ORDER BY rowid ASC
+  )
+} {0 5000}
+
+do_catchsql_test 4.3 {
+  SELECT count(*) FROM (
+      SELECT rowid FROM t1 WHERE t1 MATCH 'xyzterm' ORDER BY rowid DESC
+  )
+} {0 4987}
+
+do_test 4.4 {
+  db eval {
+    SELECT rowid AS a, hex(block) AS block FROM t1_data 
+    WHERE ((rowid>>36) & 0x01)==1
+      AND ((rowid>>31) & 0x1F)==0
+  } {
+    set n [string length $block]
+    set block [string range $block 0 $n-5]
+    append block [string repeat "00" 2]
+    db eval {
+      UPDATE t1_data SET block = unhex($block) WHERE rowid = $a
+    }
+  }
+} {}
+
+do_catchsql_test 4.5 {
+  SELECT count(*) FROM (
+      SELECT rowid FROM t1 WHERE t1 MATCH 'xyzterm' ORDER BY rowid ASC
+  )
+} {0 5000}
+
+do_catchsql_test 4.6 {
+  SELECT count(*) FROM (
+      SELECT rowid FROM t1 WHERE t1 MATCH 'xyzterm' ORDER BY rowid DESC
+  )
+} {0 4879}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE t USING fts5(x);
+  INSERT INTO t(t,rank) VALUES('pgsz', 64);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<80
+  )
+  INSERT INTO t SELECT 'shared word' || (i%10) || ' shared shared' FROM s;
+  INSERT INTO t(t) VALUES('optimize');
+}
+
+set lLeaf [db eval { SELECT id FROM t_data }]
+db_save_and_close
+
+foreach leaf $lLeaf {
+  db_restore_and_reopen
+  set leaf [expr $leaf]
+  set hex [db one { 
+    SELECT hex(block) FROM t_data WHERE id=$leaf 
+  }]
+
+  # Replace the first 4 bytes of each leaf page with the size of the leaf in
+  # bytes plus 50 as a 2 byte integer, followed by 0x7FFF.
+  #
+  set nn [expr [string length $hex]/2]
+  set first "[format %.4x [expr $nn+50]]7FFF"
+  set hex [string replace $hex 0 7 $first]
+
+  db eval { UPDATE t_data SET block=unhex($hex) WHERE id=$leaf }
+  do_test 5.1.$leaf {
+    catchsql {
+      SELECT rowid FROM t WHERE t MATCH 'shared' ORDER BY rowid DESC;
+    }
+    set {} {}
+  } {}
+
+  do_test 5.1.$leaf.2 {
+    catchsql {
+      PRAGMA integrity_check;
+    }
+    set {} {}
+  } {}
+}
+
 sqlite3_fts5_may_be_corrupt 0
 finish_test
+
+
diff --git a/ext/fts5/test/fts5near.test b/ext/fts5/test/fts5near.test
index 318a169488..82b92fde0c 100644
--- a/ext/fts5/test/fts5near.test
+++ b/ext/fts5/test/fts5near.test
@@ -66,5 +66,18 @@ do_near_test 1.23 "a b c d e f g h i" { NEAR(a+b+c+d i b+c, 4) } 0
 do_near_test 1.24 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 5) } 1
 do_near_test 1.25 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 4) } 0
 
+#-------------------------------------------------------------------------
+# Check that https://sqlite.org/bugs/forumpost/16bb36f7e8 is fixed.
+#
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t USING fts5(x, tokenize=trigram);
+  INSERT INTO t 
+  VALUES('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa');
+}
+
+do_execsql_test 2.1 {
+  SELECT count(*) FROM t WHERE t MATCH 'NEAR(aaa aaa, 4)';
+} {1}
 
 finish_test
diff --git a/manifest b/manifest
index 585dd41899..087d9f1f96 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Disable\sthe\svector-IN-SELECT\soptimization\sif\sthe\snumber\sof\scolumns\sis\sso\nlarge\sthat\sit\swould\scause\sthe\sWhereTerm.nChild\scolumn\sto\swrap.
-D 2026-06-04T16:58:53.414
+C Fix\sfour\sobscure\syet\sseparate\sproblems\sin\sFTS5.
+D 2026-06-08T19:23:56.065
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -108,12 +108,12 @@ F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a0
 F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15
 F ext/fts5/fts5.h ff5d3cc88b29e41612bfb29eb723e29e38973de62ca75ba3e8f94ccb67f5b5f2
 F ext/fts5/fts5Int.h 8d98f8e180fe28d6067e240ed45b9011735d29d5cfb5bac194e1e376baa7c708
-F ext/fts5/fts5_aux.c 042da27e97d38071312c111cf18f3cb7983b75ba5e724aa1c3164e61e90f428a
+F ext/fts5/fts5_aux.c 27af933e1a052d9f12d62a45bc60e0b65023997e0cea8f0476ef3cf66e724599
 F ext/fts5/fts5_buffer.c dcc3f0352339fe79c9d8abbc1c2009bc3469206467880bf43558447ef4f846fb
 F ext/fts5/fts5_config.c bfba970fe1e4eed18ee57c8d51458e226db9a960ddf775c5e50e3d76603a667e
-F ext/fts5/fts5_expr.c 71d48e8cf0358deace4949276647d317ff7665db6db09f40b81e2e7fe6664c7c
+F ext/fts5/fts5_expr.c 20e41452e4f83899a3a1bc66d018701186a0bbbc3a1a524f8cae447e0b150f05
 F ext/fts5/fts5_hash.c d5871df92ce3fa210a650cf419ee916b87c29977e86084d06612edf772bff6f5
-F ext/fts5/fts5_index.c 1627d9c184dec6be913647ba1e31a906e825b8460c9575e1c367ee27fe847f0c
+F ext/fts5/fts5_index.c fa1d5f1fd88bc9e2e3d4caaf6a1c7614cb475fa823dfae53cfe764d4b6876db7
 F ext/fts5/fts5_main.c b0fed47b3b4420ba6810373480a75bc28a9c0b7d16478d19a396436fb3ff17d7
 F ext/fts5/fts5_storage.c 19bc7c4cbe1e6a2dd9849ef7d84b5ca1fcbf194cefc3e386b901e00e08bf05c2
 F ext/fts5/fts5_tcl.c 2be6cc14f9448f720fd4418339cd202961a0801ea9424cb3d9de946f8f5a051c
@@ -131,7 +131,7 @@ F ext/fts5/test/fts5ab.test c7e5c1519afb20366cb40d0179897a3c39d9fc06ba6b9c286d79
 F ext/fts5/test/fts5ac.test 4a73626de86f3d17c95738034880c4f0de8d54741fb943d819b528373657e59b
 F ext/fts5/test/fts5ad.test 058e616612964e61d19f70295f0e6eaedceb4b29b1fbf4f859615ef7e779dc22
 F ext/fts5/test/fts5ae.test 3d49edbd50bb0684199a2e7568aeb30d1d29718f5c0f61751983740fa836d15f
-F ext/fts5/test/fts5af.test ae81f08b8da4c5f9b3ec1ef538a4ab6b7c278e92fa9058d6dc5d842c5d9771b9
+F ext/fts5/test/fts5af.test 06f05ef35e8c66d8200204379e03d386673cf0c6f052bd67ddc3091dac3d32e9
 F ext/fts5/test/fts5ag.test 6667807b5d3fbf460892e756763fbe3d87a2fffe345a06514ba010ca6f6641f7
 F ext/fts5/test/fts5ah.test e1f01314b35745a30e1b494b46045b82005d71cae74f1ebd9f1338566b77f9fc
 F ext/fts5/test/fts5ai.test cbe26d78030998f535bc103f37915350b137a822c71a9db439a077d7666a3539
@@ -169,7 +169,7 @@ F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66
 F ext/fts5/test/fts5corrupt7.test 9664c15360e8b649ad76f457a0bbf5a7271b8eff1a8ee141ea039bc63240c934
 F ext/fts5/test/fts5corrupt8.test 0b10750caf8aa23fa1c379ca4caf6130d41454505e4d5315590f4061eedcbe44
 F ext/fts5/test/fts5corrupt9.test 4253b9b59f33effac8b67da72ec34309c738aca2d5e8e2656bfbbd6a489a1dfe
-F ext/fts5/test/fts5corruptA.test 7b31551444569420903d34ae50a55a1227d16969264f0b50de2dc812bc0b3414
+F ext/fts5/test/fts5corruptA.test 469571adb09d10c7a68d84b73ab7b4ea9e7f119d9b754a85b802e33976b62ea7
 F ext/fts5/test/fts5corruptbig.test 9f95b40fa36e292feceab02b2ef06e21878bfa1ac7afefa138aae05518b51774
 F ext/fts5/test/fts5delete.test 2a5008f8b1174ef41d1974e606928c20e4f9da77d9f8347aed818994d89cced4
 F ext/fts5/test/fts5detail.test 54015e9c43ec4ba542cfb93268abdf280e0300f350efd08ee411284b03595cc4
@@ -214,7 +214,7 @@ F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd731
 F ext/fts5/test/fts5misc.test 83d6c5101a092c5db8fb631cfdd69a6482e20528b2750427641ac9050d9d0381
 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581
 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45
-F ext/fts5/test/fts5near.test 33d60867581066e5db7016deb5d651628125d7ff4e0233a88175aa5b65874c74
+F ext/fts5/test/fts5near.test b173a56a3c45ac9fdd1626db2ddf923c42632489a4ccedfe68a806c1a0734286
 F ext/fts5/test/fts5onepass.test b56d4109e841c2bc83555c162515748780ea6e0c455c54cf4afd4bd940d14b84
 F ext/fts5/test/fts5optimize.test 264b9101721c17d06d1d174feb743fda3ddc89fad41dee980fef821428258e47
 F ext/fts5/test/fts5optimize2.test 795d4ae5f66a7239cf8d5aef4c2ea96aeb8bcd907bd9be0cfe22064fc71a44ed
@@ -2200,9 +2200,12 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee
 F tool/warnings.sh a554d13f6e5cf3760f041b87939e3d616ec6961859c3245e8ef701d1eafc2ca2
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
 F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c
-P 3be41e17529d34d083463d967bcb7e07515ce0d6ba616a4a04a68f162ba0cf7f
-Q +bb49dfc948048779a30deff9a3d1bb39846f6f72c26a2bdcdd0e386251720fe6
-R eecd4914eaa0e5e8749eebe152c1746a
+P 10d9da4f48623419f9abed17cd05cfa08743377926ca96cf6b457dc01ab062e2
+Q +4af1d9b3e54a7c42552e61284456bbd7089e525d4aa55e580f7518956d8521bb
+Q +8ba5acabf9a0273ffbc2ba0bacc7f0b000690db91081858e4e563d6cbeeb4e7a
+Q +9c018b02dbfb071c748d540ad679a4dbdc0fb88a62988e02cb51a3403509febe
+Q +b07441cfc06b8e6b47a4f4a6e0f5c261da580d5afe82444cc7f42a9f39ea7026
+R ea3611acda6a6b16df1fd44ffb81b67a
 U drh
-Z 9d3e723303e4f4e74ffbd4ad45cb65f6
+Z c1a37a10d986b5e345b2f754c1a3465c
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index a869c9540b..6ebefbad36 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-10d9da4f48623419f9abed17cd05cfa08743377926ca96cf6b457dc01ab062e2
+a6eb055c35f091a8e864c1de3387d389e2bd3025c32596c0b7a78ebdf091898b