From 24b845791168e481e9c6870f8a41f7123d8fb101 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 27 Oct 2015 17:48:57 +0000
Subject: [PATCH] Fix problems with prefix queries in fts5.

FossilOrigin-Name: 3b5758c647530bd5c2e68d0ee3e9f58a96347ca4
---
 ext/fts5/fts5_index.c            | 27 +++++++----
 ext/fts5/test/fts5integrity.test | 49 ++++++++++++++++++-
 ext/fts5/test/fts5query.test     | 83 ++++++++++++++++++++++++++++++++
 ext/fts5/tool/showfts5.tcl       | 13 +++++
 manifest                         | 20 ++++----
 manifest.uuid                    |  2 +-
 6 files changed, 172 insertions(+), 22 deletions(-)
 create mode 100644 ext/fts5/test/fts5query.test

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index e70bc85351..b3f4372965 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -1499,6 +1499,7 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
     pIter->bDel = (nSz & 0x0001);
     pIter->nPos = nSz>>1;
     pIter->iLeafOffset = iOff;
+    assert_nc( pIter->nPos>=0 );
   }
 }
 
@@ -1672,12 +1673,13 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
     if( pNew ){
       /* iTermLeafOffset may be equal to szLeaf if the term is the last
       ** thing on the page - i.e. the first rowid is on the following page.
-      ** In this case leaf pIter->pLeaf==0, this iterator is at EOF. */
-      if( pIter->iLeafPgno==pIter->iTermLeafPgno 
-       && pIter->iTermLeafOffset<pNew->szLeaf 
-      ){
-        pIter->pLeaf = pNew;
-        pIter->iLeafOffset = pIter->iTermLeafOffset;
+      ** In this case leave pIter->pLeaf==0, this iterator is at EOF. */
+      if( pIter->iLeafPgno==pIter->iTermLeafPgno ){
+        assert( pIter->pLeaf==0 );
+        if( pIter->iTermLeafOffset<pNew->szLeaf ){
+          pIter->pLeaf = pNew;
+          pIter->iLeafOffset = pIter->iTermLeafOffset;
+        }
       }else{
         int iRowidOff;
         iRowidOff = fts5LeafFirstRowidOff(pNew);
@@ -1851,6 +1853,7 @@ static void fts5SegIterNext(
           fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
           pIter->bDel = (nSz & 0x0001);
           pIter->nPos = nSz>>1;
+          assert_nc( pIter->nPos>=0 );
         }
       }
     }
@@ -2059,11 +2062,14 @@ static void fts5LeafSeek(
       if( pIter->pLeaf==0 ) return;
       a = pIter->pLeaf->p;
       if( fts5LeafIsTermless(pIter->pLeaf)==0 ){
-        fts5GetVarint32(&pIter->pLeaf->p[pIter->pLeaf->szLeaf], iOff);
+        iPgidx = pIter->pLeaf->szLeaf;
+        iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff);
         if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){
           p->rc = FTS5_CORRUPT;
         }else{
           nKeep = 0;
+          iTermOff = iOff;
+          n = pIter->pLeaf->nn;
           iOff += fts5GetVarint32(&a[iOff], nNew);
           break;
         }
@@ -4375,7 +4381,7 @@ static void fts5SetupPrefixIter(
     Fts5IndexIter *p1 = 0;     /* Iterator used to gather data from index */
     Fts5Data *pData;
     Fts5Buffer doclist;
-    int bNewTerm = 0;
+    int bNewTerm = 1;
 
     memset(&doclist, 0, sizeof(doclist));
     for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1);
@@ -5582,8 +5588,9 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
   }
   while( iOff<n ){
     int nPos;
-    int bDummy;
-    iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy);
+    int bDel;
+    iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDel);
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " nPos=%d%s", nPos, bDel?"*":"");
     iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
     if( iOff<n ){
       i64 iDelta;
diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test
index 478e790d6b..84490e0cb4 100644
--- a/ext/fts5/test/fts5integrity.test
+++ b/ext/fts5/test/fts5integrity.test
@@ -9,7 +9,7 @@
 #
 #***********************************************************************
 #
-# This file containst tests focused on the integrity-check procedure.
+# This file contains tests focused on the integrity-check procedure.
 #
 
 source [file join [file dirname [info script]] fts5_common.tcl]
@@ -102,6 +102,53 @@ do_catchsql_test 4.5 {
 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM zz_data} {puts $r}
 #exit
 
+execsql { ROLLBACK }
+
+
+#-------------------------------------------------------------------------
+# Test that integrity-check works on a reasonably large db with many
+# different terms.
+
+# Document generator command.
+proc rnddoc {n} {
+  set doc [list]
+  for {set i 0} {$i<$n} {incr i} {
+    lappend doc [format %.5d [expr int(rand()*10000)]]
+  }
+  return $doc
+}
+db func rnddoc rnddoc
+
+expr srand(0)
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE gg USING fts5(a, prefix="1,2,3");
+  INSERT INTO gg(gg, rank) VALUES('pgsz', 256);
+  INSERT INTO gg VALUES(rnddoc(20));
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+  INSERT INTO gg SELECT rnddoc(20) FROM gg;
+}
+
+do_execsql_test 5.1 {
+  INSERT INTO gg(gg) VALUES('integrity-check');
+}
+
+do_execsql_test 5.2 {
+  INSERT INTO gg(gg) VALUES('optimize');
+}
+
+breakpoint
+do_execsql_test 5.3 {
+  INSERT INTO gg(gg) VALUES('integrity-check');
+}
 
 finish_test
 
diff --git a/ext/fts5/test/fts5query.test b/ext/fts5/test/fts5query.test
new file mode 100644
index 0000000000..570abf46ba
--- /dev/null
+++ b/ext/fts5/test/fts5query.test
@@ -0,0 +1,83 @@
+# 2015 October 27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5query
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} {
+  reset_db
+  do_test 1.$tn.1 {
+    execsql {
+      CREATE VIRTUAL TABLE t1 USING fts5(x);
+      INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
+      BEGIN;
+    }
+    foreach x [list aaa bbb ccc ddd eee fff ggg hhh iii jjj] {
+      set doc [string repeat "$x " 30]
+      execsql { INSERT INTO t1 VALUES($doc) }
+    }
+    execsql COMMIT
+  } {}
+
+  do_execsql_test 1.$tn.2 {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+
+  set ret 1
+  foreach x [list a b c d e f g h i j] {
+    do_execsql_test 1.$tn.3.$ret {
+      SELECT rowid FROM t1 WHERE t1 MATCH $x || '*';
+    } $ret
+    incr ret
+  }
+}
+
+for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} {
+  reset_db
+  do_test 2.$tn.1 {
+    execsql {
+      CREATE VIRTUAL TABLE t1 USING fts5(x);
+      INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
+      BEGIN;
+    }
+    foreach x [list bbb ddd fff hhh jjj lll nnn ppp rrr ttt] {
+      set doc [string repeat "$x " 30]
+      execsql { INSERT INTO t1 VALUES($doc) }
+    }
+    execsql COMMIT
+  } {}
+
+  do_execsql_test 1.$tn.2 {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+
+  set ret 1
+  foreach x [list a c e g i k m o q s u] {
+    do_execsql_test 2.$tn.3.$ret {
+      SELECT rowid FROM t1 WHERE t1 MATCH $x || '*';
+    } {}
+    incr ret
+  }
+}
+
+
+finish_test
+
+
diff --git a/ext/fts5/tool/showfts5.tcl b/ext/fts5/tool/showfts5.tcl
index d9af5f38eb..75ac0f1c8f 100644
--- a/ext/fts5/tool/showfts5.tcl
+++ b/ext/fts5/tool/showfts5.tcl
@@ -8,11 +8,13 @@ proc usage {} {
   puts stderr "usage: $::argv0 ?OPTIONS? database table"
   puts stderr ""
   puts stderr "  -nterm                (count number of terms in each segment)"
+  puts stderr "  -segments             (output segment contents)"
   puts stderr ""
   exit 1
 }
 
 set O(nterm) 0
+set O(segments) 0
 
 if {[llength $argv]<2} usage
 foreach a [lrange $argv 0 end-2] {
@@ -21,6 +23,10 @@ foreach a [lrange $argv 0 end-2] {
       set O(nterm) 1
     }
 
+    -segments {
+      set O(segments) 1
+    }
+
     default {
       usage
     }
@@ -78,6 +84,13 @@ db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" {
   }
 }
 
+if {$O(segments)} {
+  puts ""
+  db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id>10" {
+    puts $d
+  }
+}
+
 
 
 
diff --git a/manifest b/manifest
index 980451cf38..9057c85e7b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Provide\shints\sto\sthe\sstorage\sengine\susing\sthe\ssqlite3BtreeCursorHint()\ninterface\swhen\scompiling\swith\sSQLITE_ENABLE_CURSOR_HINTS.
-D 2015-10-27T13:35:14.170
+C Fix\sproblems\swith\sprefix\squeries\sin\sfts5.
+D 2015-10-27T17:48:57.642
 F Makefile.in 2ea961bc09e441874eb3d1bf7398e04feb24f3ee
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 702d3e98f3afc6587a78481257f3c4c900efc3a4
@@ -109,7 +109,7 @@ F ext/fts5/fts5_buffer.c 1e49512a535045e621246dc7f4f65f3593fa0fc2
 F ext/fts5/fts5_config.c 88a77f5d5e4dfbb2355b8f6cc9969b7f02d94685
 F ext/fts5/fts5_expr.c 28b15c9ae296204bc0a2e5cf7a667d840a9d2900
 F ext/fts5/fts5_hash.c a9d4c1efebc2a91d26ad7ebdfcbf2678ceac405f
-F ext/fts5/fts5_index.c 2bd3cb65b36160f7b411e70c4360d0619c965e4e
+F ext/fts5/fts5_index.c 356481ce027cc2ede8e462c316b578260cda29d2
 F ext/fts5/fts5_main.c 520a29136ba07448331f73bdc36d0ffa1e9dcfef
 F ext/fts5/fts5_storage.c 8038a54a88d3beb94dc7f9db6428a3bc08b718bb
 F ext/fts5/fts5_tcl.c 3bf445e66de32137d4693694ff7b1fd6074e32bd
@@ -157,7 +157,7 @@ F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda
 F ext/fts5/test/fts5fault7.test 01be274bfc8d9bf22451a3bf5892e9399d044f1b
 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
 F ext/fts5/test/fts5hash.test 7cf4607b8657c383f0b520668a99971e95d8b139
-F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145
+F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65
 F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2
 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367
 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc
@@ -168,6 +168,7 @@ F ext/fts5/test/fts5plan.test 6a55ecbac9890765b0e16f8c421c7e0888cfe436
 F ext/fts5/test/fts5porter.test 7cdc07bef301d70eebbfa75dcaf45c3680e1d0e1
 F ext/fts5/test/fts5porter2.test 2e65633d58a1c525d5af0f6c01e5a59155bb3487
 F ext/fts5/test/fts5prefix.test 7ccbdf180ed561a912acef520519e85af8642239
+F ext/fts5/test/fts5query.test f5ec25f5f2fbb70033424113cdffc101b1985a40
 F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
@@ -184,7 +185,7 @@ F ext/fts5/test/fts5vocab.test c88a5554d0409494da95ba647bbdb4879b2624b0
 F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84
 F ext/fts5/tool/loadfts5.tcl 58e90407cc5c2b1770460119488fd7c0090d4dd3
 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45
-F ext/fts5/tool/showfts5.tcl 9eaf6c3df352f98a2ab5ce1921dd94128ab1381d
+F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c b2732aef0b076e4276d9b39b5a33cec7a05e1413
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
@@ -1393,8 +1394,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b4e70dd08e79decaa09f7b99e2dc7e2df92f26ee b3ec9a0d62c5543e91d4be2cd634ec4a3d6dca11
-R 4646a6990e647428ec4d0d5814dac399
-T +closed b3ec9a0d62c5543e91d4be2cd634ec4a3d6dca11
-U drh
-Z 02443220badb4130f058a2b797287082
+P 45d3539e152a44629639723f18ce70d9ef01f31a
+R fbe2860d8975b512a1715e460bd017e7
+U dan
+Z 8f449fae999a2e5acc0c44c5783d72e3
diff --git a/manifest.uuid b/manifest.uuid
index 1e5f93daa3..36934c91ef 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-45d3539e152a44629639723f18ce70d9ef01f31a
\ No newline at end of file
+3b5758c647530bd5c2e68d0ee3e9f58a96347ca4
\ No newline at end of file
-- 
2.47.2