Fix bug in xInstToken() causing the wrong token to be returned.

FossilOrigin-Name: da78d07e77cbc783fbc725758911c230fd6a1c1885d9576125de955dcc2bd37f

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index df08ca21278bca4e549ccf8208d051ba49dda41a..095b945838aeeac02437be681f10fb317b6f2937 100644 (file)
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -6607,6 +6607,7 @@ struct Fts5TokenDataIter {
   Fts5TokenDataMap *aMap;
 
   Fts5PoslistReader *aPoslistReader;
+  int *aPoslistToIter;
   Fts5Iter *apIter[1];
 };
 
@@ -6794,16 +6795,19 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){
 
       /* Allocate array of iterators if they are not already allocated. */
       if( pT->aPoslistReader==0 ){
-        pT->aPoslistReader = sqlite3Fts5MallocZero(
-            &pIter->pIndex->rc, sizeof(Fts5PoslistReader) * pT->nIter
+        int nByte = pT->nIter * (sizeof(Fts5PoslistReader) + sizeof(int));
+        pT->aPoslistReader = (Fts5PoslistReader*)sqlite3Fts5MallocZero(
+            &pIter->pIndex->rc, nByte
         );
         if( pT->aPoslistReader==0 ) return;
+        pT->aPoslistToIter = (int*)&pT->aPoslistReader[pT->nIter];
       }
 
       /* Populate an iterator for each poslist that will be merged */
       for(ii=0; ii<pT->nIter; ii++){
         Fts5Iter *p = pT->apIter[ii];
         if( iRowid==p->base.iRowid ){
+          pT->aPoslistToIter[nReader] = ii;
           sqlite3Fts5PoslistReaderInit(
               p->base.pData, p->base.nData, &pT->aPoslistReader[nReader++]
           );
@@ -6855,7 +6859,7 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){
 
         if( eDetail==FTS5_DETAIL_FULL ){
           pT->aMap[pT->nMap].iPos = iMinPos;
-          pT->aMap[pT->nMap].iIter = iMin;
+          pT->aMap[pT->nMap].iIter = pT->aPoslistToIter[iMin];
           pT->aMap[pT->nMap].iRowid = iRowid;
           pT->nMap++;
         }

diff --git a/ext/fts5/test/fts5origintext5.test b/ext/fts5/test/fts5origintext5.test
index a48dfbb3abe7043716f1bfcfa2cf34a7c846edbe..9421758f8873fd920d95e074f6d0d3182c34dd11 100644 (file)
--- a/ext/fts5/test/fts5origintext5.test
+++ b/ext/fts5/test/fts5origintext5.test
@@ -65,10 +65,10 @@ proc term {} {
   select_one $::expanded_vocab
 }
 
-# Return a document - between 3 and 6 terms.
+# Return a document - between 3 and 10 terms.
 #
 proc document {} {
-  set nTerm [expr [random 3] + 3]
+  set nTerm [expr [random 3] + 7]
   set doc ""
   for {set ii 0} {$ii < $nTerm} {incr ii} {
     lappend doc [term]
@@ -82,11 +82,39 @@ db func document document
 set NDOC  200
 set NLOOP 100
 
-
 sqlite3_fts5_register_origintext db
+
+proc tokens {cmd} { 
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    if {$txt==""} break
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+proc ctrl_tokens {doc term} {
+  set ret [list]
+  set term [string tolower $term]
+  foreach a $doc {
+    if {[string tolower $a]==$term} {
+      if {$a==$term} {
+        lappend ret $a
+      } else {
+        lappend ret [string tolower $a].$a
+      }
+    }
+  }
+  set ret
+}
+db func ctrl_tokens ctrl_tokens
+
+
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE ft USING fts5(
-      x, tokenize="origintext unicode61", contentless_delete=1, content=,
+      x, tokenize="origintext unicode61", 
       tokendata=1
   );
 
@@ -105,13 +133,20 @@ do_test 1.1 {
 
 proc do_all_vocab_test {tn} {
   foreach ::v [vocab] {
-    set answer [execsql {SELECT id FROM ctrl WHERE x LIKE '%' || $::v || '%'}]
+    set answer [execsql {
+      SELECT id, ctrl_tokens(x, $::v) FROM ctrl WHERE x LIKE '%' || $::v || '%'
+    }]
     do_execsql_test $tn.$::v {
-      SELECT rowid FROM ft($::v)
+      SELECT rowid, tokens(ft) FROM ft($::v)
     } $answer
   }
 }
 
+#execsql_pp { SELECT * FROM ctrl }
+#execsql_pp { SELECT * FROM ft }
+#fts5_aux_test_functions db
+#execsql_pp { SELECT rowid, tokens(ft), fts5_test_poslist(ft) FROM ft('ghi'); }
+
 do_all_vocab_test 1.2
 
 for {set ii 0} {$ii < $NLOOP} {incr ii} {

diff --git a/manifest b/manifest
index 61b1c976fd5912ceffb537cfc2bf4297bdc04f5b..4ecc82d692c82139a9eaf87af6df251f157c5622 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stests\sfor\susing\stokendata=1\sand\scontentless_delete=1\stogether.
-D 2023-12-04T15:08:21.125
+C Fix\sbug\sin\sxInstToken()\scausing\sthe\swrong\stoken\sto\sbe\sreturned.
+D 2023-12-04T17:05:37.721
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292
 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
 F ext/fts5/fts5_expr.c 5619c3fab45a78eb5ed3021e3b40ec3b435ef3669293e8700354aa8dd3e6c796
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
-F ext/fts5/fts5_index.c b31bf4f0fb51a15cc1aa54c2f337197740f4f8898347266781ca6970ca751302
+F ext/fts5/fts5_index.c 072666814be04485445c07e0e75362d13245cef6c66e07aa2060c532190d0c10
 F ext/fts5/fts5_main.c 075995302198fe6f591fdbbedd415dfac564a9bfc20aea81e6fa0503b2d94af0
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
 F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
@@ -194,7 +194,7 @@ F ext/fts5/test/fts5origintext.test 6574e8d2121460cda72866afe3e582693d9992f150b0
 F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969
 F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a
 F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871
-F ext/fts5/test/fts5origintext5.test 5b9fa1b7d2f8c5f933076000c30aea5b104c00c3f1b767334b87b76d46492e59
+F ext/fts5/test/fts5origintext5.test f4377b67debb10e3731030ce51245b9843ffb31f85725615b3b8820bd5912702
 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
 F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
 F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
@@ -2150,8 +2150,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3a623cfa173b4035c759cb84985d11d8727053beb383648503987d6ab15c0ef0
-R 977c39d056cdc1226d7f3117e124e785
+P a2506b8c9718054912270055638204753c4156bbc115e55194e6df9d7e76cb10
+R 135aeb3f5184e1f0131c446ddc8eab37
 U dan
-Z 8168409535c63203870031d6c1a58cc5
+Z 33215e49f8892bbcd31004428ffced69
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index 0187da7a6e3af3283b05bb3c597d3b9e31f2a4cb..60beff1eed862fdcae2147d43dd717e48f09f93a 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a2506b8c9718054912270055638204753c4156bbc115e55194e6df9d7e76cb10
\ No newline at end of file
+da78d07e77cbc783fbc725758911c230fd6a1c1885d9576125de955dcc2bd37f
\ No newline at end of file