From: dan <dan@noemail.net>
Date: Thu, 31 Dec 2015 18:39:14 +0000 (+0000)
Subject: Fix the fts5 integrity-check code so that it works with detail=none tables.
X-Git-Tag: version-3.11.0~157^2~15
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=811501e23741a3eb5b04bbb8613d3ca2a1d8bab0;p=thirdparty%2Fsqlite.git

Fix the fts5 integrity-check code so that it works with detail=none tables.

FossilOrigin-Name: 3a1df69e58e5830da1dff158eedbe3817743f58f
---

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 3f342195f0..ef27d73c31 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -5199,6 +5199,7 @@ static int fts5QueryCksum(
   int flags,                      /* Flags for Fts5IndexQuery */
   u64 *pCksum                     /* IN/OUT: Checksum value */
 ){
+  int eDetail = p->pConfig->eDetail;
   u64 cksum = *pCksum;
   Fts5IndexIter *pIdxIter = 0;
   Fts5Buffer buf = {0, 0, 0};
@@ -5206,17 +5207,24 @@ static int fts5QueryCksum(
 
   while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
     i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
-    rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
-    if( rc==SQLITE_OK ){
-      Fts5PoslistReader sReader;
-      for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
-          sReader.bEof==0;
-          sqlite3Fts5PoslistReaderNext(&sReader)
-      ){
-        int iCol = FTS5_POS2COLUMN(sReader.iPos);
-        int iOff = FTS5_POS2OFFSET(sReader.iPos);
-        cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
+
+    if( eDetail==FTS5_DETAIL_NONE ){
+      cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n);
+    }else{
+      rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
+      if( rc==SQLITE_OK ){
+        Fts5PoslistReader sReader;
+        for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
+            sReader.bEof==0;
+            sqlite3Fts5PoslistReaderNext(&sReader)
+           ){
+          int iCol = FTS5_POS2COLUMN(sReader.iPos);
+          int iOff = FTS5_POS2OFFSET(sReader.iPos);
+          cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
+        }
       }
+    }
+    if( rc==SQLITE_OK ){
       rc = sqlite3Fts5IterNext(pIdxIter);
     }
   }
@@ -5523,6 +5531,7 @@ static void fts5IndexIntegrityCheckSegment(
 ** occurs.
 */
 int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
+  int eDetail = p->pConfig->eDetail;
   u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
   Fts5Buffer poslist = {0,0,0};   /* Buffer used to hold a poslist */
   Fts5IndexIter *pIter;           /* Used to iterate through entire index */
@@ -5574,12 +5583,16 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
     /* If this is a new term, query for it. Update cksum3 with the results. */
     fts5TestTerm(p, &term, z, n, cksum2, &cksum3);
 
-    poslist.n = 0;
-    fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist);
-    while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
-      int iCol = FTS5_POS2COLUMN(iPos);
-      int iTokOff = FTS5_POS2OFFSET(iPos);
-      cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
+    if( eDetail==FTS5_DETAIL_NONE ){
+      cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n);
+    }else{
+      poslist.n = 0;
+      fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist);
+      while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
+        int iCol = FTS5_POS2COLUMN(iPos);
+        int iTokOff = FTS5_POS2OFFSET(iPos);
+        cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
+      }
     }
   }
   fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index fe3ce829a1..f047fd99dc 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -853,8 +853,23 @@ static int fts5StorageIntegrityCallback(
     pCtx->szCol++;
   }
 
-  iPos = pTermset ? pCtx->iCol : pCtx->szCol-1;
-  iCol = pTermset ? 0 : pCtx->iCol;
+  switch( pCtx->pConfig->eDetail ){
+    case FTS5_DETAIL_FULL:
+      iPos = pCtx->szCol-1;
+      iCol = pCtx->iCol;
+      break;
+
+    case FTS5_DETAIL_COLUMNS:
+      iPos = pCtx->iCol;
+      iCol = 0;
+      break;
+
+    default:
+      assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE );
+      iPos = 0;
+      iCol = 0;
+      break;
+  }
 
   rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
   if( rc==SQLITE_OK && bPresent==0 ){
@@ -912,11 +927,14 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
       if( pConfig->bColumnsize ){
         rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
       }
+      if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){
+        rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
+      }
       for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
         if( pConfig->abUnindexed[i] ) continue;
         ctx.iCol = i;
         ctx.szCol = 0;
-        if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
+        if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
           rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
         }
         if( rc==SQLITE_OK ){
@@ -932,9 +950,14 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
           rc = FTS5_CORRUPT;
         }
         aTotalSize[i] += ctx.szCol;
-        sqlite3Fts5TermsetFree(ctx.pTermset);
-        ctx.pTermset = 0;
+        if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
+          sqlite3Fts5TermsetFree(ctx.pTermset);
+          ctx.pTermset = 0;
+        }
       }
+      sqlite3Fts5TermsetFree(ctx.pTermset);
+      ctx.pTermset = 0;
+
       if( rc!=SQLITE_OK ) break;
     }
     rc2 = sqlite3_reset(pScan);
diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test
index 259aff0953..f99ea6112e 100644
--- a/ext/fts5/test/fts5simple2.test
+++ b/ext/fts5/test/fts5simple2.test
@@ -109,35 +109,59 @@ do_execsql_test 7.0 {
   INSERT INTO t1 VALUES('a1 b2');
   COMMIT;
 }
-do_execsql_test 7.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1}
-do_execsql_test 7.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1}
+do_execsql_test 7.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1}
+do_execsql_test 7.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1}
 
 #-------------------------------------------------------------------------
 #
 reset_db
-do_execsql_test 7.0 {
+do_execsql_test 8.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
   INSERT INTO t1 VALUES('a1 b1 c1');
   INSERT INTO t1 VALUES('a2 b2 c2');
   INSERT INTO t1 VALUES('a1 b1 c1');
 }
-do_execsql_test 7.0.1 { SELECT rowid FROM t1('b*') } {1 2 3}
-do_execsql_test 7.0.2 { SELECT rowid FROM t1('a1') } {1 3}
-do_execsql_test 7.0.3 { SELECT rowid FROM t1('c2') } {2}
+do_execsql_test 8.0.1 { SELECT rowid FROM t1('b*') } {1 2 3}
+do_execsql_test 8.0.2 { SELECT rowid FROM t1('a1') } {1 3}
+do_execsql_test 8.0.3 { SELECT rowid FROM t1('c2') } {2}
+
+do_execsql_test 8.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1}
+do_execsql_test 8.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1}
+do_execsql_test 8.0.8 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2}
+
+do_execsql_test 8.1.0 { INSERT INTO t1(t1) VALUES('optimize') }
 
-do_execsql_test 7.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1}
-do_execsql_test 7.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1}
-do_execsql_test 7.0.7 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2}
+do_execsql_test 8.1.1 { SELECT rowid FROM t1('b*') } {1 2 3}
+do_execsql_test 8.1.2 { SELECT rowid FROM t1('a1') } {1 3}
+do_execsql_test 8.1.3 { SELECT rowid FROM t1('c2') } {2}
 
-do_execsql_test 7.1.0 { INSERT INTO t1(t1) VALUES('optimize') }
+do_execsql_test 8.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1}
+do_execsql_test 8.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1}
+do_execsql_test 8.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2}
 
-do_execsql_test 7.1.1 { SELECT rowid FROM t1('b*') } {1 2 3}
-do_execsql_test 7.1.2 { SELECT rowid FROM t1('a1') } {1 3}
-do_execsql_test 7.1.3 { SELECT rowid FROM t1('c2') } {2}
+#--------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 9.0.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+  INSERT INTO t1 VALUES('a1 b1 c1');
+  INSERT INTO t1 VALUES('a2 b2 c2');
+  INSERT INTO t1 VALUES('a1 b1 c1');
+}
+do_execsql_test 9.0.1 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} {}
 
-do_execsql_test 7.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1}
-do_execsql_test 7.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1}
-do_execsql_test 7.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2}
+reset_db
+do_execsql_test 9.1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none);
+  INSERT INTO t1 VALUES('a1 b1 c1', 'x y z');
+  INSERT INTO t1 VALUES('a2 b2 c2', '1 2 3');
+  INSERT INTO t1 VALUES('a1 b1 c1', 'x 2 z');
+}
+do_execsql_test 9.2.1 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} {}
 
 finish_test
 
diff --git a/manifest b/manifest
index dc56eb5d2c..c173c4014d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\ssome\sproblems\swith\sfts5\sdetail=none\stables.\sSome\sstill\sremain.
-D 2015-12-31T17:36:58.906
+C Fix\sthe\sfts5\sintegrity-check\scode\sso\sthat\sit\sworks\swith\sdetail=none\stables.
+D 2015-12-31T18:39:14.423
 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751
@@ -103,9 +103,9 @@ F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1
 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6
 F ext/fts5/fts5_expr.c 08ee4f342a2b1fd82a5dccd0b9b3cde4921a8be5
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
-F ext/fts5/fts5_index.c 40eac312bdcb67552fe6dc2e0a4bad67d9273e3d
+F ext/fts5/fts5_index.c a7cfe8e88621b473d749268f918b54b81a067c2e
 F ext/fts5/fts5_main.c 23f3912ff44172859c771eb55cb57778fd662e89
-F ext/fts5/fts5_storage.c 076a3356536a8831eb6e554195171d5c249cd179
+F ext/fts5/fts5_storage.c 771dd0fda3ee513e32937a386dc2a4aa2d9ea64e
 F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb
 F ext/fts5/fts5_test_mi.c c9c8cf455c49ded156a234e0de1c8ba9be85e5c3
 F ext/fts5/fts5_tokenize.c 618efe033bceb80c521b1e9ddfd9fee85fb5946e
@@ -170,7 +170,7 @@ F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821
 F ext/fts5/test/fts5simple.test a599b7577bc3827a9a678add3b43d8b818b93456
-F ext/fts5/test/fts5simple2.test 9b1e4c44fc63bd0cbb3ea639cd0ad9fa764849bf
+F ext/fts5/test/fts5simple2.test a1a9f9c443f8852f412ee9658ef667086ccb74d7
 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
@@ -1407,7 +1407,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P ac8f4cf0cede6bcbe47eeefb85d80a27e9278212
-R 4d80cd94292f49ff63b2b342f7b40c82
+P 6a6f7bc40d6b4c8a4a254a9098c9d2f31fde69ca
+R 70fdbbf97fa363362bfe1572d2790ec2
 U dan
-Z 84b26c5644e7f6dfb95a54fd72f26ecc
+Z 9907fb7300f9e53b05e209bde8721940
diff --git a/manifest.uuid b/manifest.uuid
index bcbd354a20..7093688530 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6a6f7bc40d6b4c8a4a254a9098c9d2f31fde69ca
\ No newline at end of file
+3a1df69e58e5830da1dff158eedbe3817743f58f
\ No newline at end of file