From: dan <dan@noemail.net>
Date: Sat, 12 Dec 2009 16:04:32 +0000 (+0000)
Subject: Further fts3 coverage tests.
X-Git-Tag: version-3.7.2~707
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=55be7449f94119e6fb67a1530cc4e0187b1f48c4;p=thirdparty%2Fsqlite.git

Further fts3 coverage tests.

FossilOrigin-Name: d2a8c0f683271f5fb8c9badfb13e4e46fd78db71
---

diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index 1f82e61d31..1a48f385e0 100644
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -1959,7 +1959,8 @@ int sqlite3Fts3SegReaderIterate(
       nMerge++;
     }
 
-    if( nMerge==1 && !isIgnoreEmpty && !isColFilter && isRequirePos ){
+    assert( isIgnoreEmpty==0 || (isRequirePos && isColFilter==0) );
+    if( nMerge==1 && !isIgnoreEmpty ){
       Fts3SegReader *p0 = apSegment[0];
       rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist);
       if( rc!=SQLITE_OK ) goto finished;
@@ -2061,7 +2062,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
   int rc;                         /* Return code */
   int iIdx;                       /* Index of new segment */
   int iNewLevel;                  /* Level to create new segment at */
-  sqlite3_stmt *pStmt;
+  sqlite3_stmt *pStmt = 0;
   SegmentWriter *pWriter = 0;
   int nSegment = 0;               /* Number of segments being merged */
   Fts3SegReader **apSegment = 0;  /* Array of Segment iterators */
@@ -2074,12 +2075,11 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
     ** greatest segment level currently present in the database. The index
     ** of the new segment is always 0.
     */
-    rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pPending);
-    if( rc!=SQLITE_OK ){
-      return rc;
-    }
     iIdx = 0;
+    rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pPending);
+    if( rc!=SQLITE_OK ) goto finished;
     rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel);
+    if( rc!=SQLITE_OK ) goto finished;
     nSegment += (pPending!=0);
     if( nSegment<=1 ){
       return SQLITE_DONE;
@@ -2092,17 +2092,18 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
     */
     iNewLevel = iLevel+1;
     rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
-    if( rc!=SQLITE_OK ) return rc;
+    if( rc!=SQLITE_OK ) goto finished;
     rc = fts3SegmentCount(p, iLevel, &nSegment);
+    if( rc!=SQLITE_OK ) goto finished;
   }
-  if( rc!=SQLITE_OK ) return rc;
   assert( nSegment>0 );
   assert( iNewLevel>=0 );
 
   /* Allocate space for an array of pointers to segment iterators. */
   apSegment = (Fts3SegReader**)sqlite3_malloc(sizeof(Fts3SegReader *)*nSegment);
   if( !apSegment ){
-    return SQLITE_NOMEM;
+    rc = SQLITE_NOMEM;
+    goto finished;
   }
   memset(apSegment, 0, sizeof(Fts3SegReader *)*nSegment);
 
@@ -2123,6 +2124,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
   rc = sqlite3_reset(pStmt);
   if( pPending ){
     apSegment[i] = pPending;
+    pPending = 0;
   }
   pStmt = 0;
   if( rc!=SQLITE_OK ) goto finished;
@@ -2148,6 +2150,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
     }
     sqlite3_free(apSegment);
   }
+  sqlite3Fts3SegReaderFree(p, pPending);
   sqlite3_reset(pStmt);
   return rc;
 }
diff --git a/manifest b/manifest
index a3d53d0ec8..7355ac2e41 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,5 @@
------BEGIN PGP SIGNED MESSAGE-----
-Hash: SHA1
-
-C Rename\stkt-d82e3f3721.txt\sto\suse\sthe\s(correct)\s.test\ssuffix.
-D 2009-12-12T13:58:19
+C Further\sfts3\scoverage\stests.
+D 2009-12-12T16:04:32
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -71,7 +68,7 @@ F ext/fts3/fts3_snippet.c 6c2eb6d872d66b2a9aa5663f2662e993f18a6496
 F ext/fts3/fts3_tokenizer.c 1a49ee3d79cbf0b9386250370d9cbfe4bb89c8ff
 F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c
 F ext/fts3/fts3_tokenizer1.c 11a604a53cff5e8c28882727bf794e5252e5227b
-F ext/fts3/fts3_write.c 8fa73906993c46d1add03367d658b4b2f2214bb8
+F ext/fts3/fts3_write.c c2e0ef39df121b291fda42992ef1387f7938dea1
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33
 F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2
@@ -399,7 +396,7 @@ F test/fts3ao.test 0aa29dd4fc1c8d46b1f7cfe5926f7ac97551bea9
 F test/fts3atoken.test 25c2070e1e8755d414bf9c8200427b277a9f99fa
 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
-F test/fts3cov.test ce49852ac727c74cfce8e5a4cc017c7edfab7b7e
+F test/fts3cov.test f2d27d29628941e4814473345603b20421b93c78
 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
 F test/fts3expr.test 05dab77387801e4900009917bb18f556037d82da
@@ -782,14 +779,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P eee921a99e69a9cd868a89de620bf47c4e26e4b5
-R 5d65e226c3987e06982591864693cc96
-U drh
-Z b6b0937ebc754fff562e435d0e81894c
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.6 (GNU/Linux)
-
-iD8DBQFLI6F/oxKgR168RlERAv3/AKCGxXDpk/BocjexCfeSDuJTQRc0JwCeOdSJ
-ZSbD17MSKkOO16NEgcCVaWk=
-=0ws+
------END PGP SIGNATURE-----
+P 68cccd62b71f7b55bf7f2f56dc8507cbe80229ba
+R 24dafc1291b25c037b068505da647d63
+U dan
+Z f50ab64261ebba22ef3131cb9e897960
diff --git a/manifest.uuid b/manifest.uuid
index 35de3d6e5c..521448761d 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-68cccd62b71f7b55bf7f2f56dc8507cbe80229ba
\ No newline at end of file
+d2a8c0f683271f5fb8c9badfb13e4e46fd78db71
\ No newline at end of file
diff --git a/test/fts3cov.test b/test/fts3cov.test
index 8537afdc82..dab208bd54 100644
--- a/test/fts3cov.test
+++ b/test/fts3cov.test
@@ -18,6 +18,7 @@ source $testdir/tester.tcl
 #
 ifcapable !fts3 { finish_test ; return }
 source $testdir/fts3_common.tcl
+source $testdir/malloc_common.tcl
 
 set DO_MALLOC_TEST 0
 
@@ -173,6 +174,12 @@ do_write_test fts3cov-4.2 t3_content {
 # by inserting insert a document with some fairly large terms into a
 # full-text table with a very small node-size. 
 #
+# Test this handling of large terms in three contexts:
+#
+#   1. When flushing the pending-terms table.
+#   2. When optimizing the data structures using the INSERT syntax. 
+#   2. When optimizing the data structures using the deprecated SELECT syntax. 
+#
 do_test fts3cov-5.1 {
   execsql {
     CREATE VIRTUAL TABLE t4 USING fts3(x);
@@ -180,6 +187,8 @@ do_test fts3cov-5.1 {
   }
 } {}
 set DO_MALLOC_TEST 1
+
+# Test when flushing pending-terms table.
 do_write_test fts3cov-5.2 t4_content {
   INSERT INTO t4
     SELECT 'ItisanancientMarinerAndhestoppethoneofthreeAA' UNION ALL
@@ -189,14 +198,155 @@ do_write_test fts3cov-5.2 t4_content {
     SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstBB' UNION ALL
     SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstCC'
 }
-do_test fts3cov-5.3 {
-  execsql { INSERT INTO t4 VALUES('extra!') }
-} {}
-do_write_test fts3cov-5.2 t4_segments {
-  INSERT INTO t4(t4) VALUES('optimize')
+
+# Test when optimizing via INSERT.
+do_test fts3cov-5.3 { execsql { INSERT INTO t4 VALUES('extra!') } } {}
+do_write_test fts3cov-5.2 t4_segments { INSERT INTO t4(t4) VALUES('optimize') }
+
+# Test when optimizing via SELECT.
+do_test fts3cov-5.5 { execsql { INSERT INTO t4 VALUES('more extra!') } } {}
+do_write_test fts3cov-5.6 t4_segments {
+  SELECT * FROM (SELECT optimize(t4) FROM t4 LIMIT 1)
+  EXCEPT SELECT 'Index optimized'
+}
+
+#-------------------------------------------------------------------------
+# When merging all segments at a given level to create a single segment
+# at level+1, FTS3 runs a query of the form:
+#
+#   SELECT count(*) FROM %_segdir WHERE level = ?
+#
+# The query is compiled the first time this operation is required and
+# reused thereafter. This test aims to test the effects of an OOM while
+# preparing and executing this query for the first time.
+#
+# Then, keep inserting rows into the table so that the effects of an OOM
+# while re-executing the same query can also be tested.
+#
+do_test fts3cov-6.1 {
+  execsql { CREATE VIRTUAL TABLE t5 USING fts3(x) }
+  for {set i 0} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" }
+  execsql { SELECT count(*) FROM t5_segdir }
+} {16}
+
+# First time.
+db close
+sqlite3 db test.db
+do_write_test fts3cov-6.2 t5_content {
+  INSERT INTO t5 VALUES('segment number 16!');
+}
+
+# Second time.
+do_test fts3cov-6.3 {
+  for {set i 1} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" }
+  execsql { SELECT count(*) FROM t5_segdir }
+} {17}
+do_write_test fts3cov-6.4 t5_content {
+  INSERT INTO t5 VALUES('segment number 16!');
+}
+
+#-------------------------------------------------------------------------
+# Update the docid of a row. Test this in two scenarios:
+#
+#   1. When the row being updated is the only row in the table.
+#   2. When it is not.
+#
+# The two cases above take different paths because in case 1 all data 
+# structures can simply be emptied before inserting the new row record.
+# In case 2, the data structures actually have to be updated.
+#
+do_test fts3cov-7.1 {
+  execsql {
+    CREATE VIRTUAL TABLE t7 USING fts3(a, b, c);
+    INSERT INTO t7 VALUES('A', 'B', 'C');
+    UPDATE t7 SET docid = 5;
+    SELECT docid, * FROM t7;
+  }
+} {5 A B C}
+do_test fts3cov-7.2 {
+  execsql {
+    INSERT INTO t7 VALUES('D', 'E', 'F');
+    UPDATE t7 SET docid = 1 WHERE docid = 6;
+    SELECT docid, * FROM t7;
+  }
+} {1 D E F 5 A B C}
+
+#-------------------------------------------------------------------------
+# If a set of documents are modified within a transaction, the 
+# pending-terms table must be flushed each time a document with a docid
+# less than or equal to the previous docid is modified. 
+#
+# This test checks the effects of an OOM error occuring when the 
+# pending-terms table is flushed for this reason as part of a DELETE 
+# statement.
+#
+do_malloc_test fts3cov-8 -sqlprep {
+  BEGIN;
+    CREATE VIRTUAL TABLE t8 USING fts3;
+    INSERT INTO t8 VALUES('the output of each batch run');
+    INSERT INTO t8 VALUES('(possibly a day''s work)');
+    INSERT INTO t8 VALUES('was written to two separate disks');
+  COMMIT;
+} -sqlbody {
+  BEGIN;
+    DELETE FROM t8 WHERE rowid = 3;
+    DELETE FROM t8 WHERE rowid = 2;
+    DELETE FROM t8 WHERE rowid = 1;
+  COMMIT;
 }
 
+#-------------------------------------------------------------------------
+# Test some branches in the code that handles "special" inserts like:
+#
+#   INSERT INTO t1(t1) VALUES('optimize');
+#
+# Also test that an optimize (INSERT method) works on an empty table.
+#
+set DO_MALLOC_TEST 0
+do_test fts3cov-9.1 {
+  execsql { CREATE VIRTUAL TABLE xx USING fts3 }
+} {}
+do_error_test fts3cov-9.2 {
+  INSERT INTO xx(xx) VALUES('optimise');   -- British spelling
+} {SQL logic error or missing database}
+do_error_test fts3cov-9.3 {
+  INSERT INTO xx(xx) VALUES('short');
+} {SQL logic error or missing database}
+do_error_test fts3cov-9.4 {
+  INSERT INTO xx(xx) VALUES('waytoolongtobecorrect');
+} {SQL logic error or missing database}
+do_test fts3cov-9.5 {
+  execsql { INSERT INTO xx(xx) VALUES('optimize') }
+} {}
+
+#-------------------------------------------------------------------------
+# Test that a table can be optimized in the middle of a transaction when
+# the pending-terms table is non-empty. This case involves some extra
+# branches because data must be read not only from the database, but
+# also from the pending-terms table.
+#
+do_malloc_test fts3cov-10 -sqlprep {
+  CREATE VIRTUAL TABLE t10 USING fts3;
+  INSERT INTO t10 VALUES('Optimising images for the web is a tricky business');
+  BEGIN;
+    INSERT INTO t10 VALUES('You have to get the right balance between');
+} -sqlbody {
+  INSERT INTO t10(t10) VALUES('optimize');
+}
 
+#-------------------------------------------------------------------------
+# Test a full-text query for a term that was once in the index, but is
+# no longer.
+#
+do_test fts3cov-11.1 {
+  execsql { 
+    CREATE VIRTUAL TABLE xx USING fts3;
+    INSERT INTO xx VALUES('one two three');
+    INSERT INTO xx VALUES('four five six');
+    DELETE FROM xx WHERE docid = 1;
+  }
+  execsql { SELECT * FROM xx WHERE xx MATCH 'two' }
+} {}
 
 finish_test