Add tests for fts5. Fix a crash that can occur in fts5 if the database content is...

FossilOrigin-Name: acaf426449bf6fd3140fd63141750ff69d1119a5

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index ee953c399cd4ab4c0b2a19a90aea294d4ad4f557..b2f0d6c34e6b4b1bc5635225f518b48889aafabd 100644 (file)
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -225,13 +225,13 @@ int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
 typedef struct Fts5Buffer Fts5Buffer;
 struct Fts5Buffer {
   u8 *p;
-  int n;
-  int nSpace;
+  u32 n;
+  u32 nSpace;
 };
 
-int sqlite3Fts5BufferSize(int*, Fts5Buffer*, int);
+int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
 void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
-void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
+void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
 void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
 void sqlite3Fts5BufferFree(Fts5Buffer*);
 void sqlite3Fts5BufferZero(Fts5Buffer*);

diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c
index 8558687d8a6a39a59569f5f382658c6dd89dda0d..94591188d340e04bc1887db9d6339de5d52583fd 100644 (file)
--- a/ext/fts5/fts5_buffer.c
+++ b/ext/fts5/fts5_buffer.c
@@ -15,8 +15,8 @@
 
 #include "fts5Int.h"
 
-int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, int nByte){
-  int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
+int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
+  u32 nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
   u8 *pNew;
   while( nNew<nByte ){
     nNew = nNew * 2;
@@ -61,10 +61,10 @@ int sqlite3Fts5Get32(const u8 *aBuf){
 void sqlite3Fts5BufferAppendBlob(
   int *pRc,
   Fts5Buffer *pBuf, 
-  int nData, 
+  u32 nData, 
   const u8 *pData
 ){
-  assert( *pRc || nData>=0 );
+  assert_nc( *pRc || nData>=0 );
   if( fts5BufferGrow(pRc, pBuf, nData) ) return;
   memcpy(&pBuf->p[pBuf->n], pData, nData);
   pBuf->n += nData;

diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 7a433b69a0a5cf0d80abf7b7f011d04b938a762e..c51ed7939ca15d5e7224d79f883208392e12fae6 100644 (file)
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -834,7 +834,7 @@ static int fts5ExprTokenTest(
   assert( pPhrase->aTerm[0].pSynonym==0 );
 
   rc = sqlite3Fts5IterPoslist(pIter, pColset, 
-      (const u8**)&pPhrase->poslist.p, &pPhrase->poslist.n, &pNode->iRowid
+      (const u8**)&pPhrase->poslist.p, (int*)&pPhrase->poslist.n, &pNode->iRowid
   );
   pNode->bNomatch = (pPhrase->poslist.n==0);
   return rc;

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 3d4488aedfe167d127f6f2c66d043869ebe8ed6d..fc11a234138aa15e1413a0ea6540cc0ff78c7b54 100644 (file)
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -6014,7 +6014,7 @@ static void fts5DecodeFunction(
 
     /* Decode any entries that occur before the first term. */
     if( szLeaf<n ){
-      fts5FastGetVarint32(a, iPgidxOff, iTermOff);
+      iPgidxOff += fts5GetVarint32(&a[iPgidxOff], iTermOff);
     }else{
       iTermOff = szLeaf;
     }
@@ -6025,7 +6025,7 @@ static void fts5DecodeFunction(
       int nAppend;
 
       /* Read the term data for the next term*/
-      fts5FastGetVarint32(a, iOff, nAppend);
+      iOff += fts5GetVarint32(&a[iOff], nAppend);
       term.n = nKeep;
       fts5BufferAppendBlob(&rc, &term, nAppend, &a[iOff]);
       sqlite3Fts5BufferAppendPrintf(
@@ -6036,7 +6036,7 @@ static void fts5DecodeFunction(
       /* Figure out where the doclist for this term ends */
       if( iPgidxOff<n ){
         int nIncr;
-        fts5FastGetVarint32(a, iPgidxOff, nIncr);
+        iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nIncr);
         iTermOff += nIncr;
       }else{
         iTermOff = szLeaf;
@@ -6045,7 +6045,7 @@ static void fts5DecodeFunction(
       fts5DecodeRowidList(&rc, &s, &a[iOff], iTermOff-iOff);
       iOff = iTermOff;
       if( iOff<szLeaf ){
-        fts5FastGetVarint32(a, iOff, nKeep);
+        iOff += fts5GetVarint32(&a[iOff], nKeep);
       }
     }
 

diff --git a/ext/fts5/test/fts5merge2.test b/ext/fts5/test/fts5merge2.test
new file mode 100644 (file)
index 0000000..87dbf14
--- /dev/null
+++ b/ext/fts5/test/fts5merge2.test
@@ -0,0 +1,59 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test that focus on incremental merges of segments.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5merge
+
+proc dump_structure {} {
+  db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
+    foreach lvl [lrange $t 1 end] {
+      set seg [string repeat . [expr [llength $lvl]-2]]
+      puts "[lrange $lvl 0 1] $seg"
+    }
+  }
+}
+
+foreach_detail_mode $testprefix {
+
+if {[detail_is_none]==0} continue
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  INSERT INTO t1(t1, rank) VALUES('crisismerge', 2);
+  INSERT INTO t1 VALUES('1 2 3 4');
+}
+
+expr srand(0)
+db func rnddoc fts5_rnddoc
+do_test 1.1 {
+  for {set i 0} {$i < 100} {incr i} {
+    execsql {
+      BEGIN;
+        DELETE FROM t1 WHERE rowid = 1;
+        INSERT INTO t1(rowid, x) VALUES(1, '1 2 3 4');
+        INSERT INTO t1 VALUES(rnddoc(10));
+      COMMIT;
+    }
+  }
+} {}
+
+do_execsql_test 1.2 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+}
+
+finish_test
+

diff --git a/ext/fts5/test/fts5rowid.test b/ext/fts5/test/fts5rowid.test
index 1db1d560034ad572d6ddcdb1d983bedd54559d57..13a075e0bcb386d1955cca157c3a9996e3a6a8ff 100644 (file)
--- a/ext/fts5/test/fts5rowid.test
+++ b/ext/fts5/test/fts5rowid.test
@@ -184,5 +184,32 @@ do_execsql_test 5.2 {
   SELECT count(fts5_decode(rowid, block)) FROM x4_data;
 } $res
 
+#-------------------------------------------------------------------------
+#
+
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE x5 USING fts5(x, detail=none);
+  INSERT INTO x5(x5, rank) VALUES('pgsz', 32);
+  INSERT INTO x5 VALUES('a b c d e f');
+  INSERT INTO x5 VALUES('a b c d e f');
+  INSERT INTO x5 VALUES('a b c d e f');
+  BEGIN;
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+    ) INSERT INTO x5 SELECT 'a b c d e f' FROM s;
+  COMMIT;
+  SELECT count(fts5_decode_none(rowid, block)) FROM x5_data;
+} {32}
+
+do_execsql_test 6.1 {
+  DELETE FROM x5 WHERE rowid=1;
+  UPDATE x5 SET x='a b c d e f' WHERE rowid=2;
+  SELECT count(fts5_decode_none(rowid, block)) FROM x5_data;
+} {36}
+
+#db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM x5_data} {puts $r}
+
+
+
 finish_test
 

diff --git a/manifest b/manifest
index 17430c54232466179bc198eedd671f2e893ec41a..cdd417e773ed9d5bda1ee1edae02bfc4adf6362a 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Avoid\sunnecessary\scalls\sto\smemset()\sfor\sa\ssmall\sperformance\simprovement.
-D 2016-01-18T13:18:54.021
+C Add\stests\sfor\sfts5.\sFix\sa\scrash\sthat\scan\soccur\sin\sfts5\sif\sthe\sdatabase\scontent\sis\scorrupted.
+D 2016-01-18T17:48:28.938
 F Makefile.in a476545d0c8626224d0bacac85c6e2967474af81
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 01e855f958932d0d3ed62ec675fc63e2cef61fcb
@@ -97,13 +97,13 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
-F ext/fts5/fts5Int.h 313e3276ac9e0245ee722ef803253857a68722b9
+F ext/fts5/fts5Int.h 5599703af9c13512900a9f22fec39d48078d619d
 F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e
-F ext/fts5/fts5_buffer.c ba59964c95f760bd5ff1dbe173b85f197a13864e
+F ext/fts5/fts5_buffer.c 7d3f6f01f8fdc45204e6a33925ef8478a67d28dd
 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238
-F ext/fts5/fts5_expr.c 3ba4c9588ebb2e4f852d807869af7130b6362e5a
+F ext/fts5/fts5_expr.c 4ab4504c54bbe24689c83411d8588f4ec99136e9
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
-F ext/fts5/fts5_index.c 5ba2a5efcd6035fd35363ad8d014e488ab9d87fc
+F ext/fts5/fts5_index.c 716c301835a122ba36910b4f821c87d26ae9a5d9
 F ext/fts5/fts5_main.c 833db0a3df10ab26e0221a9baa40cf871c450df3
 F ext/fts5/fts5_storage.c fb2eaec3aa954b680d43096dc539f8270bd6390e
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
@@ -159,6 +159,7 @@ F ext/fts5/test/fts5hash.test 00668f6fa9b9bffbd7c1be29f408aa2bdade0451
 F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65
 F ext/fts5/test/fts5matchinfo.test 86569026d20f1ed748236587ce798de8a96615f1
 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367
+F ext/fts5/test/fts5merge2.test c0cb66eb38a41c26cc5848fb9e50093e0f59ac93
 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc
 F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c
 F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5
@@ -171,7 +172,7 @@ F ext/fts5/test/fts5query.test f5ec25f5f2fbb70033424113cdffc101b1985a40
 F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
-F ext/fts5/test/fts5rowid.test 27a1e18c26d599a80bcabf6f687a4f1d8d29d2d2
+F ext/fts5/test/fts5rowid.test 42de27109d2f6091108bf27097b105cb7e34f492
 F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9
 F ext/fts5/test/fts5simple2.test 98377ae1ff7749a42c21fe1a139c1ed312522c46
 F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48
@@ -1417,7 +1418,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 8358af3658d888516cdef5f8c8d89e9bdee53f91
-R c9824fbd0634badf7e3856a23bc7d527
-U drh
-Z 3a50e6364ad5d43622d6dbbf0015cd45
+P 9e8c23acf74944a165c733682a956948b15bd401
+R 08e65f545f5743a4e11b0d8939a59f80
+U dan
+Z 3083046e1d22f6a7fb73cbf36ab37bf0

diff --git a/manifest.uuid b/manifest.uuid
index f39ceaa8a5712f7fe51f15f23b132aa67b79ea83..102b730d6e40e827fcf3a2197a6c7fc79211fc51 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9e8c23acf74944a165c733682a956948b15bd401
\ No newline at end of file
+acaf426449bf6fd3140fd63141750ff69d1119a5
\ No newline at end of file