From: dan Date: Sat, 12 Dec 2009 09:51:25 +0000 (+0000) Subject: Add coverage test cases for fts3. X-Git-Tag: version-3.7.2~710 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f5fff2a7b204bf8d6fbe859dd8d9e0c949f5d8a4;p=thirdparty%2Fsqlite.git Add coverage test cases for fts3. FossilOrigin-Name: 8fcb0478c82507403165719724b62a308cb83b57 --- diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index c87a3241ff..749922f180 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -621,6 +621,7 @@ static int fts3InitVtab( p->azColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; p->nNodeSize = 1000; + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; zCsr = (char *)&p->azColumn[nCol]; fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); @@ -2268,6 +2269,10 @@ int sqlite3Fts3Init(sqlite3 *db){ } } +#ifdef SQLITE_TEST + sqlite3Fts3ExprInitTestInterface(db); +#endif + /* Create the virtual table wrapper around the hash-table and overload ** the two scalar functions. If this is successful, register the ** module with sqlite. diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 769cda4f7a..64fd32b2c6 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -120,10 +120,11 @@ struct Fts3Table { /* The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. - ** When nPendingData exceeds FTS3_MAX_PENDING_DATA, the buffer is flushed + ** When nPendingData exceeds nMaxPendingData, the buffer is flushed ** automatically. Variable iPrevDocid is the docid of the most recently ** inserted record. */ + int nMaxPendingData; int nPendingData; sqlite_int64 iPrevDocid; Fts3Hash pendingTerms; diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 7ed589095a..c4b820df50 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -294,8 +294,8 @@ int sqlite3Fts3ReadBlock( *pnBlock = sqlite3_column_bytes(pStmt, 0); *pzBlock = (char *)sqlite3_column_blob(pStmt, 0); - if( !*pzBlock ){ - return SQLITE_NOMEM; + if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ + return SQLITE_CORRUPT; } } return SQLITE_OK; @@ -510,7 +510,7 @@ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ ** buffer was half empty, that would let the less frequent terms ** generate longer doclists. */ - if( iDocid<=p->iPrevDocid || p->nPendingData>FTS3_MAX_PENDING_DATA ){ + if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){ int rc = sqlite3Fts3PendingTermsFlush(p); if( rc!=SQLITE_OK ) return rc; } @@ -2221,6 +2221,9 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + p->nMaxPendingData = atoi(&zVal[11]); + rc = SQLITE_OK; #endif }else{ rc = SQLITE_ERROR; diff --git a/manifest b/manifest index 6d208602c0..bc7f7be386 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Additional\schanges\sto\sC-language\sinterface\sdocumentation. -D 2009-12-11T23:11:27 +C Add\scoverage\stest\scases\sfor\sfts3. +D 2009-12-12T09:51:25 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -59,9 +56,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c c059f8e743276b589cd07e311a42d9b699cc7bb7 +F ext/fts3/fts3.c e1828210a55b4b27df2b1f6fa78b1fb12058f56e F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 1419e2973b44ee78f0ae8f7e03abfa2bdaf14f54 +F ext/fts3/fts3Int.h 6fdd41b4f296e5bcc908444dc591397995d4ff5d F ext/fts3/fts3_expr.c fcf6812dbfd9cb9a2cabaf50e741411794f83e7e F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec @@ -71,7 +68,7 @@ F ext/fts3/fts3_snippet.c 6c2eb6d872d66b2a9aa5663f2662e993f18a6496 F ext/fts3/fts3_tokenizer.c 1a49ee3d79cbf0b9386250370d9cbfe4bb89c8ff F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c F ext/fts3/fts3_tokenizer1.c 11a604a53cff5e8c28882727bf794e5252e5227b -F ext/fts3/fts3_write.c 3f49f2cde3285c6e3559a756812c4a525abcfb25 +F ext/fts3/fts3_write.c a8f2eb04ac4cc869b7d7ffbfff63a4805e5559a9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33 F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2 @@ -190,7 +187,7 @@ F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 F src/test_config.c 220a67047af393756f55760fdf442d935d0d88f3 F src/test_devsym.c de3c9af2bb9a8b1e44525c449e4ec3f88e3d4110 F src/test_func.c 1c94388a23d4a9e7cd62ec79d612d1bae2451fa2 -F src/test_hexio.c 415adbdb88cd9388831ce10edff76cb9e73d8a0b +F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2 F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 @@ -380,7 +377,7 @@ F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682 F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a F test/fts3.test ae0433b09b12def08105640e57693726c4949338 -F test/fts3_common.tcl dbed2fca2162d695908895680071a2b7e0a2eb55 +F test/fts3_common.tcl 2a2044688ce3addb1dd58d3d846c574cf4b7bbcd F test/fts3aa.test 5327d4c1d9b6c61021696746cc9a6cdc5bf159c0 F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9 F test/fts3ac.test 356280144a2c92aa7b11474afadfe62a437fcd69 @@ -399,6 +396,7 @@ F test/fts3ao.test 0aa29dd4fc1c8d46b1f7cfe5926f7ac97551bea9 F test/fts3atoken.test 25c2070e1e8755d414bf9c8200427b277a9f99fa F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 +F test/fts3cov.test 1a21eb90d994bc8dee3fb810ee663902d0796fa7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 05dab77387801e4900009917bb18f556037d82da @@ -781,14 +779,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 87fc0ce151c89beb5a43b65c0b1706f340c0c173 -R b6b67451b1e0a7b9820d087eb26e450f -U drh -Z 763db5cb8f1e7f1ba88fe11712fa307e ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFLItGjoxKgR168RlERAhmMAJ9m9BZSx/FxFUNPj9G9NQ3Jc6mFwgCfaZ8y -dtfT2OxwqJGAKinsW/tudQE= -=r4RM ------END PGP SIGNATURE----- +P 1342916fd350d06e1c1f3d7d380249f0c9282c7b +R 513a9c1fbf0e1a704528face3e7e0567 +U dan +Z d3f9cc9a3767b7f1e5f0d0743941cc08 diff --git a/manifest.uuid b/manifest.uuid index 38b40e7299..32c41e7a82 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1342916fd350d06e1c1f3d7d380249f0c9282c7b \ No newline at end of file +8fcb0478c82507403165719724b62a308cb83b57 \ No newline at end of file diff --git a/src/test_hexio.c b/src/test_hexio.c index db1aa44eb6..02bd60c8aa 100644 --- a/src/test_hexio.c +++ b/src/test_hexio.c @@ -330,12 +330,12 @@ static int getFts3Varint(const char *p, sqlite_int64 *v){ /* -** USAGE: read_varint BLOB VARNAME +** USAGE: read_fts3varint BLOB VARNAME ** ** Read a varint from the start of BLOB. Set variable VARNAME to contain ** the interpreted value. Return the number of bytes of BLOB consumed. */ -static int read_varint( +static int read_fts3varint( void * clientData, Tcl_Interp *interp, int objc, @@ -373,7 +373,7 @@ int Sqlitetest_hexio_Init(Tcl_Interp *interp){ { "hexio_render_int16", hexio_render_int16 }, { "hexio_render_int32", hexio_render_int32 }, { "utf8_to_utf8", utf8_to_utf8 }, - { "read_varint", read_varint }, + { "read_fts3varint", read_fts3varint }, }; int i; for(i=0; i0} {incr iTest} { - set res [uplevel [list catchsql $sql]] - if {[lsearch -exact $answers $res]>=0} { - set res $str + if {$isRestart} { sqlite3 db test.db } + + sqlite3_memdebug_fail [expr $iFail-$iTest] -repeat $nRepeat + set res [uplevel [list catchsql $sql]] + if {[lsearch -exact $answers $res]>=0} { set res $str } + set testname "$name.$zName.$iFail" + do_test "$name.$zName.$iLimit.$iFail" [list set {} $res] $str + + set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] } - set testname "$name.$zName.$iFail" - if {$zName == ""} { set testname $name } - do_test $testname [list set {} $res] $str - set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] - if {$nFail==0} break } } } diff --git a/test/fts3cov.test b/test/fts3cov.test new file mode 100644 index 0000000000..6bcce50c0e --- /dev/null +++ b/test/fts3cov.test @@ -0,0 +1,168 @@ +# 2009 December 03 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file are structural coverage tests. They are designed +# to complement the tests in fts3rnd.test and fts3doc.test. Between them, +# the three files should provide full coverage of the fts3 extension code. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If this build does not include FTS3, skip the tests in this file. +# +ifcapable !fts3 { finish_test ; return } +source $testdir/fts3_common.tcl + +set DO_MALLOC_TEST 0 + +#-------------------------------------------------------------------------- +# When it first needs to read a block from the %_segments table, the FTS3 +# module compiles an SQL statement for that purpose. The statement is +# stored and reused each subsequent time a block is read. This test case +# tests the effects of an OOM error occuring while compiling the statement. +# +# Similarly, when FTS3 first needs to scan through a set of segment leaves +# to find a set of documents that matches a term, it allocates a string +# containing the text of the required SQL, and compiles one or more +# statements to traverse the leaves. This test case tests that OOM errors +# that occur while allocating this string and statement are handled correctly +# also. +# +do_test fts3cov-1.1 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts3(x); + INSERT INTO t1(t1) VALUES('nodesize=24'); + BEGIN; + INSERT INTO t1 VALUES('Is the night chilly and dark?'); + INSERT INTO t1 VALUES('The night is chilly, but not dark.'); + INSERT INTO t1 VALUES('The thin gray cloud is spread on high,'); + INSERT INTO t1 VALUES('It covers but not hides the sky.'); + COMMIT; + SELECT count(*)>0 FROM t1_segments; + } +} {1} + +set DO_MALLOC_TEST 1 +do_restart_select_test fts3cov-1.2 { + SELECT docid FROM t1 WHERE t1 MATCH 'chilly'; +} {1 2} +set DO_MALLOC_TEST 0 + +#-------------------------------------------------------------------------- +# When querying the full-text index, if an expected internal node block is +# missing from the %_segments table, or if a NULL value is stored in the +# %_segments table instead of a binary blob, database corruption should be +# reported. +# +# Even with tiny 24 byte nodes, it takes a fair bit of data to produce a +# segment b-tree that uses the %_segments table to store internal nodes. +# +do_test fts3cov-2.1 { + execsql { + INSERT INTO t1(t1) VALUES('nodesize=24'); + BEGIN; + INSERT INTO t1 VALUES('The moon is behind, and at the full;'); + INSERT INTO t1 VALUES('And yet she looks both small and dull.'); + INSERT INTO t1 VALUES('The night is chill, the cloud is gray:'); + INSERT INTO t1 VALUES('''T is a month before the month of May,'); + INSERT INTO t1 VALUES('And the Spring comes slowly up this way.'); + INSERT INTO t1 VALUES('The lovely lady, Christabel,'); + INSERT INTO t1 VALUES('Whom her father loves so well,'); + INSERT INTO t1 VALUES('What makes her in the wood so late,'); + INSERT INTO t1 VALUES('A furlong from the castle gate?'); + INSERT INTO t1 VALUES('She had dreams all yesternight'); + INSERT INTO t1 VALUES('Of her own betrothed knight;'); + INSERT INTO t1 VALUES('And she in the midnight wood will pray'); + INSERT INTO t1 VALUES('For the weal of her lover that''s far away.'); + COMMIT; + + INSERT INTO t1(t1) VALUES('optimize'); + SELECT substr(hex(root), 1, 2) FROM t1_segdir; + } +} {03} + +# Test the "missing entry" case: +do_test fts3cov-2.1 { + set root [db one {SELECT root FROM t1_segdir}] + read_fts3varint [string range $root 1 end] left_child + execsql { DELETE FROM t1_segments WHERE blockid = $left_child } +} {} +do_error_test fts3cov-2.2 { + SELECT * FROM t1 WHERE t1 MATCH 'c*' +} {database disk image is malformed} + +# Test the "replaced with NULL" case: +do_test fts3cov-2.3 { + execsql { INSERT INTO t1_segments VALUES($left_child, NULL) } +} {} +do_error_test fts3cov-2.4 { + SELECT * FROM t1 WHERE t1 MATCH 'cloud' +} {database disk image is malformed} + +#-------------------------------------------------------------------------- +# The following tests are to test the effects of OOM errors while storing +# terms in the pending-hash table. Specifically, while creating doclist +# blobs to store in the table. More specifically, to test OOM errors while +# appending column numbers to doclists. For example, if a doclist consists +# of: +# +# 0x01 +# +# The following tests check that malloc errors encountered while appending +# the "0x01 " data to the dynamically growable blob used to +# accumulate the doclist in memory are handled correctly. +# +do_test fts3cov-3.1 { + set cols [list] + set vals [list] + for {set i 0} {$i < 120} {incr i} { + lappend cols "col$i" + lappend vals "'word'" + } + execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])" +} {} +set DO_MALLOC_TEST 1 +do_write_test fts3cov-3.2 t2_content " + INSERT INTO t2(docid, [join $cols ,]) VALUES(1, [join $vals ,]) +" +do_write_test fts3cov-3.3 t2_content " + INSERT INTO t2(docid, [join $cols ,]) VALUES(200, [join $vals ,]) +" +do_write_test fts3cov-3.4 t2_content " + INSERT INTO t2(docid, [join $cols ,]) VALUES(60000, [join $vals ,]) +" + +#------------------------------------------------------------------------- +# If too much data accumulates in the pending-terms hash table, it is +# flushed to the database automatically, even if the transaction has not +# finished. The following tests check the effects of encountering an OOM +# while doing this. +# +do_test fts3cov-4.1 { + execsql { + CREATE VIRTUAL TABLE t3 USING fts3(x); + INSERT INTO t3(t3) VALUES('nodesize=24'); + INSERT INTO t3(t3) VALUES('maxpending=100'); + } +} {} +set DO_MALLOC_TEST 1 +do_write_test fts3cov-4.2 t3_content { + INSERT INTO t3(docid, x) + SELECT 1, 'Then Christabel stretched forth her hand,' UNION ALL + SELECT 3, 'And comforted fair Geraldine:' UNION ALL + SELECT 4, '''O well, bright dame, may you command' UNION ALL + SELECT 5, 'The service of Sir Leoline;' UNION ALL + SELECT 2, 'And gladly our stout chivalry' UNION ALL + SELECT 7, 'Will he send forth, and friends withal,' UNION ALL + SELECT 8, 'To guide and guard you safe and free' UNION ALL + SELECT 6, 'Home to your noble father''s hall.''' +} + +finish_test +