p->azColumn = (char **)&p[1];
p->pTokenizer = pTokenizer;
p->nNodeSize = 1000;
+ p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
zCsr = (char *)&p->azColumn[nCol];
fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
}
}
+#ifdef SQLITE_TEST
+ sqlite3Fts3ExprInitTestInterface(db);
+#endif
+
/* Create the virtual table wrapper around the hash-table and overload
** the two scalar functions. If this is successful, register the
** module with sqlite.
/* The following hash table is used to buffer pending index updates during
** transactions. Variable nPendingData estimates the memory size of the
** pending data, including hash table overhead, but not malloc overhead.
- ** When nPendingData exceeds FTS3_MAX_PENDING_DATA, the buffer is flushed
+ ** When nPendingData exceeds nMaxPendingData, the buffer is flushed
** automatically. Variable iPrevDocid is the docid of the most recently
** inserted record.
*/
+ int nMaxPendingData;
int nPendingData;
sqlite_int64 iPrevDocid;
Fts3Hash pendingTerms;
*pnBlock = sqlite3_column_bytes(pStmt, 0);
*pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
- if( !*pzBlock ){
- return SQLITE_NOMEM;
+ if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
+ return SQLITE_CORRUPT;
}
}
return SQLITE_OK;
** buffer was half empty, that would let the less frequent terms
** generate longer doclists.
*/
- if( iDocid<=p->iPrevDocid || p->nPendingData>FTS3_MAX_PENDING_DATA ){
+ if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){
int rc = sqlite3Fts3PendingTermsFlush(p);
if( rc!=SQLITE_OK ) return rc;
}
}else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
p->nNodeSize = atoi(&zVal[9]);
rc = SQLITE_OK;
+ }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
+ p->nMaxPendingData = atoi(&zVal[11]);
+ rc = SQLITE_OK;
#endif
}else{
rc = SQLITE_ERROR;
------BEGIN PGP SIGNED MESSAGE-----
-Hash: SHA1
-
-C Additional\schanges\sto\sC-language\sinterface\sdocumentation.
-D 2009-12-11T23:11:27
+C Add\scoverage\stest\scases\sfor\sfts3.
+D 2009-12-12T09:51:25
F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c c059f8e743276b589cd07e311a42d9b699cc7bb7
+F ext/fts3/fts3.c e1828210a55b4b27df2b1f6fa78b1fb12058f56e
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h 1419e2973b44ee78f0ae8f7e03abfa2bdaf14f54
+F ext/fts3/fts3Int.h 6fdd41b4f296e5bcc908444dc591397995d4ff5d
F ext/fts3/fts3_expr.c fcf6812dbfd9cb9a2cabaf50e741411794f83e7e
F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c
F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
F ext/fts3/fts3_tokenizer.c 1a49ee3d79cbf0b9386250370d9cbfe4bb89c8ff
F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c
F ext/fts3/fts3_tokenizer1.c 11a604a53cff5e8c28882727bf794e5252e5227b
-F ext/fts3/fts3_write.c 3f49f2cde3285c6e3559a756812c4a525abcfb25
+F ext/fts3/fts3_write.c a8f2eb04ac4cc869b7d7ffbfff63a4805e5559a9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33
F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2
F src/test_config.c 220a67047af393756f55760fdf442d935d0d88f3
F src/test_devsym.c de3c9af2bb9a8b1e44525c449e4ec3f88e3d4110
F src/test_func.c 1c94388a23d4a9e7cd62ec79d612d1bae2451fa2
-F src/test_hexio.c 415adbdb88cd9388831ce10edff76cb9e73d8a0b
+F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2
F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c
F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99
F src/test_intarray.h 489edb9068bb926583445cb02589344961054207
F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e
F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a
F test/fts3.test ae0433b09b12def08105640e57693726c4949338
-F test/fts3_common.tcl dbed2fca2162d695908895680071a2b7e0a2eb55
+F test/fts3_common.tcl 2a2044688ce3addb1dd58d3d846c574cf4b7bbcd
F test/fts3aa.test 5327d4c1d9b6c61021696746cc9a6cdc5bf159c0
F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9
F test/fts3ac.test 356280144a2c92aa7b11474afadfe62a437fcd69
F test/fts3atoken.test 25c2070e1e8755d414bf9c8200427b277a9f99fa
F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
+F test/fts3cov.test 1a21eb90d994bc8dee3fb810ee663902d0796fa7
F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
F test/fts3expr.test 05dab77387801e4900009917bb18f556037d82da
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 87fc0ce151c89beb5a43b65c0b1706f340c0c173
-R b6b67451b1e0a7b9820d087eb26e450f
-U drh
-Z 763db5cb8f1e7f1ba88fe11712fa307e
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.6 (GNU/Linux)
-
-iD8DBQFLItGjoxKgR168RlERAhmMAJ9m9BZSx/FxFUNPj9G9NQ3Jc6mFwgCfaZ8y
-dtfT2OxwqJGAKinsW/tudQE=
-=r4RM
------END PGP SIGNATURE-----
+P 1342916fd350d06e1c1f3d7d380249f0c9282c7b
+R 513a9c1fbf0e1a704528face3e7e0567
+U dan
+Z d3f9cc9a3767b7f1e5f0d0743941cc08
-1342916fd350d06e1c1f3d7d380249f0c9282c7b
\ No newline at end of file
+8fcb0478c82507403165719724b62a308cb83b57
\ No newline at end of file
/*
-** USAGE: read_varint BLOB VARNAME
+** USAGE: read_fts3varint BLOB VARNAME
**
** Read a varint from the start of BLOB. Set variable VARNAME to contain
** the interpreted value. Return the number of bytes of BLOB consumed.
*/
-static int read_varint(
+static int read_fts3varint(
void * clientData,
Tcl_Interp *interp,
int objc,
{ "hexio_render_int16", hexio_render_int16 },
{ "hexio_render_int32", hexio_render_int32 },
{ "utf8_to_utf8", utf8_to_utf8 },
- { "read_varint", read_varint },
+ { "read_fts3varint", read_fts3varint },
};
int i;
for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){
proc gobble_varint {varname} {
upvar $varname blob
- set n [read_varint $blob ret]
+ set n [read_fts3varint $blob ret]
set blob [string range $blob $n end]
return $ret
}
# match the expected results passed via parameter $result.
#
proc do_select_test {name sql result} {
- uplevel [list doPassiveTest $name $sql [list 0 $result]]
+ uplevel [list doPassiveTest 0 $name $sql [list 0 $result]]
+}
+
+proc do_restart_select_test {name sql result} {
+ uplevel [list doPassiveTest 1 $name $sql [list 0 $result]]
}
proc do_error_test {name sql error} {
- uplevel [list doPassiveTest $name $sql [list 1 $error]]
+ uplevel [list doPassiveTest 0 $name $sql [list 1 $error]]
}
-proc doPassiveTest {name sql catchres} {
+proc doPassiveTest {isRestart name sql catchres} {
if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }
- if {$::DO_MALLOC_TEST} {
- set answers [list {1 {out of memory}} $catchres]
- set modes [list 100000 transient 1 persistent]
- } else {
- set answers [list $catchres]
- set modes [list 0 ""]
+ switch $::DO_MALLOC_TEST {
+ 0 { # No malloc failures.
+ do_test $name [list catchsql $sql] $catchres
+ return
+ }
+ 1 { # Simulate transient failures.
+ set nRepeat 1
+ set zName "transient"
+ set nStartLimit 100000
+ set nBackup 1
+ }
+ 2 { # Simulate persistent failures.
+ set nRepeat 1
+ set zName "persistent"
+ set nStartLimit 100000
+ set nBackup 1
+ }
+ 3 { # Simulate transient failures with extra brute force.
+ set nRepeat 100000
+ set zName "ridiculous"
+ set nStartLimit 1
+ set nBackup 10
+ }
}
+
+ # The set of acceptable results from running [catchsql $sql].
+ #
+ set answers [list {1 {out of memory}} $catchres]
set str [join $answers " OR "]
- foreach {nRepeat zName} $modes {
- for {set iFail 1} 1 {incr iFail} {
- if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}
+ set nFail 1
+ for {set iLimit $nStartLimit} {$nFail} {incr iLimit} {
+ for {set iFail 1} {$nFail && $iFail<=$iLimit} {incr iFail} {
+ for {set iTest 0} {$iTest<$nBackup && ($iFail-$iTest)>0} {incr iTest} {
- set res [uplevel [list catchsql $sql]]
- if {[lsearch -exact $answers $res]>=0} {
- set res $str
+ if {$isRestart} { sqlite3 db test.db }
+
+ sqlite3_memdebug_fail [expr $iFail-$iTest] -repeat $nRepeat
+ set res [uplevel [list catchsql $sql]]
+ if {[lsearch -exact $answers $res]>=0} { set res $str }
+ set testname "$name.$zName.$iFail"
+ do_test "$name.$zName.$iLimit.$iFail" [list set {} $res] $str
+
+ set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
}
- set testname "$name.$zName.$iFail"
- if {$zName == ""} { set testname $name }
- do_test $testname [list set {} $res] $str
- set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
- if {$nFail==0} break
}
}
}
--- /dev/null
+# 2009 December 03
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The tests in this file are structural coverage tests. They are designed
+# to complement the tests in fts3rnd.test and fts3doc.test. Between them,
+# the three files should provide full coverage of the fts3 extension code.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# If this build does not include FTS3, skip the tests in this file.
+#
+ifcapable !fts3 { finish_test ; return }
+source $testdir/fts3_common.tcl
+
+set DO_MALLOC_TEST 0
+
+#--------------------------------------------------------------------------
+# When it first needs to read a block from the %_segments table, the FTS3
+# module compiles an SQL statement for that purpose. The statement is
+# stored and reused each subsequent time a block is read. This test case
+# tests the effects of an OOM error occuring while compiling the statement.
+#
+# Similarly, when FTS3 first needs to scan through a set of segment leaves
+# to find a set of documents that matches a term, it allocates a string
+# containing the text of the required SQL, and compiles one or more
+# statements to traverse the leaves. This test case tests that OOM errors
+# that occur while allocating this string and statement are handled correctly
+# also.
+#
+do_test fts3cov-1.1 {
+ execsql {
+ CREATE VIRTUAL TABLE t1 USING fts3(x);
+ INSERT INTO t1(t1) VALUES('nodesize=24');
+ BEGIN;
+ INSERT INTO t1 VALUES('Is the night chilly and dark?');
+ INSERT INTO t1 VALUES('The night is chilly, but not dark.');
+ INSERT INTO t1 VALUES('The thin gray cloud is spread on high,');
+ INSERT INTO t1 VALUES('It covers but not hides the sky.');
+ COMMIT;
+ SELECT count(*)>0 FROM t1_segments;
+ }
+} {1}
+
+set DO_MALLOC_TEST 1
+do_restart_select_test fts3cov-1.2 {
+ SELECT docid FROM t1 WHERE t1 MATCH 'chilly';
+} {1 2}
+set DO_MALLOC_TEST 0
+
+#--------------------------------------------------------------------------
+# When querying the full-text index, if an expected internal node block is
+# missing from the %_segments table, or if a NULL value is stored in the
+# %_segments table instead of a binary blob, database corruption should be
+# reported.
+#
+# Even with tiny 24 byte nodes, it takes a fair bit of data to produce a
+# segment b-tree that uses the %_segments table to store internal nodes.
+#
+do_test fts3cov-2.1 {
+ execsql {
+ INSERT INTO t1(t1) VALUES('nodesize=24');
+ BEGIN;
+ INSERT INTO t1 VALUES('The moon is behind, and at the full;');
+ INSERT INTO t1 VALUES('And yet she looks both small and dull.');
+ INSERT INTO t1 VALUES('The night is chill, the cloud is gray:');
+ INSERT INTO t1 VALUES('''T is a month before the month of May,');
+ INSERT INTO t1 VALUES('And the Spring comes slowly up this way.');
+ INSERT INTO t1 VALUES('The lovely lady, Christabel,');
+ INSERT INTO t1 VALUES('Whom her father loves so well,');
+ INSERT INTO t1 VALUES('What makes her in the wood so late,');
+ INSERT INTO t1 VALUES('A furlong from the castle gate?');
+ INSERT INTO t1 VALUES('She had dreams all yesternight');
+ INSERT INTO t1 VALUES('Of her own betrothed knight;');
+ INSERT INTO t1 VALUES('And she in the midnight wood will pray');
+ INSERT INTO t1 VALUES('For the weal of her lover that''s far away.');
+ COMMIT;
+
+ INSERT INTO t1(t1) VALUES('optimize');
+ SELECT substr(hex(root), 1, 2) FROM t1_segdir;
+ }
+} {03}
+
+# Test the "missing entry" case:
+do_test fts3cov-2.1 {
+ set root [db one {SELECT root FROM t1_segdir}]
+ read_fts3varint [string range $root 1 end] left_child
+ execsql { DELETE FROM t1_segments WHERE blockid = $left_child }
+} {}
+do_error_test fts3cov-2.2 {
+ SELECT * FROM t1 WHERE t1 MATCH 'c*'
+} {database disk image is malformed}
+
+# Test the "replaced with NULL" case:
+do_test fts3cov-2.3 {
+ execsql { INSERT INTO t1_segments VALUES($left_child, NULL) }
+} {}
+do_error_test fts3cov-2.4 {
+ SELECT * FROM t1 WHERE t1 MATCH 'cloud'
+} {database disk image is malformed}
+
+#--------------------------------------------------------------------------
+# The following tests are to test the effects of OOM errors while storing
+# terms in the pending-hash table. Specifically, while creating doclist
+# blobs to store in the table. More specifically, to test OOM errors while
+# appending column numbers to doclists. For example, if a doclist consists
+# of:
+#
+# <docid> <column 0 offset-list> 0x01 <column N> <column N offset-list>
+#
+# The following tests check that malloc errors encountered while appending
+# the "0x01 <column N>" data to the dynamically growable blob used to
+# accumulate the doclist in memory are handled correctly.
+#
+do_test fts3cov-3.1 {
+ set cols [list]
+ set vals [list]
+ for {set i 0} {$i < 120} {incr i} {
+ lappend cols "col$i"
+ lappend vals "'word'"
+ }
+ execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])"
+} {}
+set DO_MALLOC_TEST 1
+do_write_test fts3cov-3.2 t2_content "
+ INSERT INTO t2(docid, [join $cols ,]) VALUES(1, [join $vals ,])
+"
+do_write_test fts3cov-3.3 t2_content "
+ INSERT INTO t2(docid, [join $cols ,]) VALUES(200, [join $vals ,])
+"
+do_write_test fts3cov-3.4 t2_content "
+ INSERT INTO t2(docid, [join $cols ,]) VALUES(60000, [join $vals ,])
+"
+
+#-------------------------------------------------------------------------
+# If too much data accumulates in the pending-terms hash table, it is
+# flushed to the database automatically, even if the transaction has not
+# finished. The following tests check the effects of encountering an OOM
+# while doing this.
+#
+do_test fts3cov-4.1 {
+ execsql {
+ CREATE VIRTUAL TABLE t3 USING fts3(x);
+ INSERT INTO t3(t3) VALUES('nodesize=24');
+ INSERT INTO t3(t3) VALUES('maxpending=100');
+ }
+} {}
+set DO_MALLOC_TEST 1
+do_write_test fts3cov-4.2 t3_content {
+ INSERT INTO t3(docid, x)
+ SELECT 1, 'Then Christabel stretched forth her hand,' UNION ALL
+ SELECT 3, 'And comforted fair Geraldine:' UNION ALL
+ SELECT 4, '''O well, bright dame, may you command' UNION ALL
+ SELECT 5, 'The service of Sir Leoline;' UNION ALL
+ SELECT 2, 'And gladly our stout chivalry' UNION ALL
+ SELECT 7, 'Will he send forth, and friends withal,' UNION ALL
+ SELECT 8, 'To guide and guard you safe and free' UNION ALL
+ SELECT 6, 'Home to your noble father''s hall.'''
+}
+
+finish_test
+
projects / thirdparty / sqlite.git / commitdiff
