From: dan Date: Tue, 1 Dec 2009 12:00:22 +0000 (+0000) Subject: Fix a segfault that can occur when querying an empty FTS3 table. Also restore the... X-Git-Tag: version-3.7.2~778 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=612b1d5cd07f676c29bc9ce388b9736014897d5a;p=thirdparty%2Fsqlite.git Fix a segfault that can occur when querying an empty FTS3 table. Also restore the rowid/docid conflict handling to work as it did in version 3.6.20. FossilOrigin-Name: c022f66b5a65aa54d5ebd55cfe941118a2042280 --- diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 1e108d41a2..48ee0e790a 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -583,10 +583,16 @@ static int fts3InsertData( ** ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); ** - ** In FTS3, if a non-NULL docid value is specified, it is the value - ** inserted. Otherwise, the rowid value is used. + ** In FTS3, this is an error. It is an error to specify non-NULL values + ** for both docid and some other rowid alias. */ if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ + if( SQLITE_NULL==sqlite3_value_type(apVal[0]) + && SQLITE_NULL!=sqlite3_value_type(apVal[1]) + ){ + /* A rowid/docid conflict. */ + return SQLITE_ERROR; + } rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); if( rc!=SQLITE_OK ) return rc; } @@ -1769,6 +1775,11 @@ int sqlite3Fts3SegReaderIterate( int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX); + /* If there are zero segments, this function is a no-op. This scenario + ** comes about only when reading from an empty database. + */ + if( nSegment==0 ) goto finished; + /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of ** equal or greater value than the specified term. This prevents many diff --git a/manifest b/manifest index 5073230584..923976c617 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Test\scoverage\simprovements\sin\sthe\sFTS3\sporter\sstemmer. -D 2009-11-30T19:48:16 +C Fix\sa\ssegfault\sthat\scan\soccur\swhen\squerying\san\sempty\sFTS3\stable.\sAlso\srestore\sthe\srowid/docid\sconflict\shandling\sto\swork\sas\sit\sdid\sin\sversion\s3.6.20. +D 2009-12-01T12:00:22 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -71,7 +68,7 @@ F ext/fts3/fts3_snippet.c 39cf30a7916b2562867d52176e87b6d7de02aea0 F ext/fts3/fts3_tokenizer.c 36f78d1a43a29b0feaec1ced6da9e56b9c653d1f F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c F ext/fts3/fts3_tokenizer1.c 0a5bcc579f35de5d24a9345d7908dc25ae403ee7 -F ext/fts3/fts3_write.c 9b35ff9666b4867b406e63ca2277de6a81b53103 +F ext/fts3/fts3_write.c 3ca5d8b24cdf6fcf18676a629ea44ab6cbe9e868 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33 F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2 @@ -329,7 +326,7 @@ F test/descidx3.test 3394ad4d089335cac743c36a14129d6d931c316f F test/diskfull.test 0cede7ef9d8f415d9d3944005c76be7589bb5ebb F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_fkey.test fd1fcf89badd5f2773d7ac04775b5ff3488eda17 -F test/e_fts3.test 24bb398086eca6d8d3dab64d383d5bb2dd4ff681 +F test/e_fts3.test 2750245de8425f11f5b07463f0c6efdfad49a936 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 6d91a5286f59add0cfcbb2d0da913b76f2242398 F test/enc3.test 5c550d59ff31dccdba5d1a02ae11c7047d77c041 @@ -778,14 +775,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 498922cc356316a3ec59320529b685728e407746 -R 85fbd80c2eff94ee29818ce5daa5c593 -U drh -Z 82d8de7e4c4da95df35f9d6571d07bab ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFLFCGEoxKgR168RlERAqrhAJwKwdpq5MdPMYkvvsC2yzQUk6xomACfTaxo -OaNnIAAU9JPJsrBdHKg5uAc= -=vlNw ------END PGP SIGNATURE----- +P 6d112bfd53998b8f6693d3f2edbcd5ab4cdf5fb1 +R 85871751503ef5eb36e53f4cac58d258 +U dan +Z 01db46b86c86cc6015cde0a44e9285a0 diff --git a/manifest.uuid b/manifest.uuid index c554d375c2..bb8d2ef8cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d112bfd53998b8f6693d3f2edbcd5ab4cdf5fb1 \ No newline at end of file +c022f66b5a65aa54d5ebd55cfe941118a2042280 \ No newline at end of file diff --git a/test/e_fts3.test b/test/e_fts3.test index 0c23d225bb..4fc86599c3 100644 --- a/test/e_fts3.test +++ b/test/e_fts3.test @@ -16,10 +16,9 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !fts3 { - finish_test - return -} +# If this build does not include FTS3, skip the tests in this file. +# +ifcapable !fts3 { finish_test ; return } source $testdir/fts3_common.tcl # Procs used to make the tests in this file easier to read. @@ -34,8 +33,21 @@ proc read_test {tn sql result} { uplevel [list do_select_test e_fts3-$tn $sql $result] } + +#------------------------------------------------------------------------- +# The body of the following [foreach] block contains test cases to verify +# that the example code in fts3.html works as expected. The tests run three +# times, with different values for DO_MALLOC_TEST. +# +# DO_MALLOC_TEST=0: Run tests with no OOM errors. +# DO_MALLOC_TEST=1: Run tests with transient OOM errors. +# DO_MALLOC_TEST=2: Run tests with persistent OOM errors. +# foreach DO_MALLOC_TEST {0 1 2} { +# Reset the database and database connection. If this iteration of the +# [foreach] loop is testing with OOM errors, disable the lookaside buffer. +# db close file delete -force test.db test.db-journal sqlite3 db test.db @@ -45,23 +57,23 @@ if {$DO_MALLOC_TEST} { sqlite3_db_config_lookaside db 0 0 0 } # Test the example CREATE VIRTUAL TABLE statements in section 1.1 # of fts3.in. # -ddl_test 1.1.1 {CREATE VIRTUAL TABLE data USING fts3()} -read_test 1.1.2 {PRAGMA table_info(data)} {0 content {} 0 {} 0} +ddl_test 1.1.1.1 {CREATE VIRTUAL TABLE data USING fts3()} +read_test 1.1.1.2 {PRAGMA table_info(data)} {0 content {} 0 {} 0} -ddl_test 1.2.1 { +ddl_test 1.1.2.1 { CREATE VIRTUAL TABLE pages USING fts3(title, keywords, body) } -read_test 1.2.2 { +read_test 1.1.2.2 { PRAGMA table_info(pages) } {0 title {} 0 {} 0 1 keywords {} 0 {} 0 2 body {} 0 {} 0} -ddl_test 1.3.1 { +ddl_test 1.1.3.1 { CREATE VIRTUAL TABLE mail USING fts3( subject VARCHAR(256) NOT NULL, body TEXT CHECK(length(body)<10240) ) } -read_test 1.3.2 { +read_test 1.1.3.2 { PRAGMA table_info(mail) } {0 subject {} 0 {} 0 1 body {} 0 {} 0} @@ -69,40 +81,222 @@ read_test 1.3.2 { # table "mail" is enforced (it should not be - FTS3 tables do not support # constraints). set largetext [string repeat "the quick brown fox " 5000] -write_test 1.3.3 mail_content { INSERT INTO mail VALUES(NULL, $largetext) } -read_test 1.3.4 { +write_test 1.1.3.3 mail_content { INSERT INTO mail VALUES(NULL, $largetext) } +read_test 1.1.3.4 { SELECT subject IS NULL, length(body) FROM mail } [list 1 100000] -ddl_test 1.4.1 { +ddl_test 1.1.4.1 { CREATE VIRTUAL TABLE papers USING fts3(author, document, tokenize=porter) } -read_test 1.4.2 { +read_test 1.1.4.2 { PRAGMA table_info(papers) } {0 author {} 0 {} 0 1 document {} 0 {} 0} -ddl_test 1.5.1 { +ddl_test 1.1.5.1 { CREATE VIRTUAL TABLE simpledata USING fts3(tokenize=simple) } -read_test 1.5.2 { +read_test 1.1.5.2 { PRAGMA table_info(simpledata) } {0 content {} 0 {} 0} -ddl_test 1.6.1 {DROP TABLE data} -ddl_test 1.6.2 {DROP TABLE pages} -ddl_test 1.6.3 {DROP TABLE mail} -ddl_test 1.6.4 {DROP TABLE papers} -ddl_test 1.6.5 {DROP TABLE simpledata} -read_test 1.6.6 {SELECT * FROM sqlite_master} {} +ifcapable icu { + ddl_test 1.1.6.1 { + CREATE VIRTUAL TABLE names USING fts3(a, b, tokenize=icu en_AU) + } + read_test 1.1.6.2 { + PRAGMA table_info(names) + } {0 a {} 0 {} 0 1 b {} 0 {} 0} +} + +ddl_test 1.1.7.1 {DROP TABLE data} +ddl_test 1.1.7.2 {DROP TABLE pages} +ddl_test 1.1.7.3 {DROP TABLE mail} +ddl_test 1.1.7.4 {DROP TABLE papers} +ddl_test 1.1.7.5 {DROP TABLE simpledata} +read_test 1.1.7.6 {SELECT * FROM sqlite_master} {} # The following is not one of the examples in section 1.1. It tests # specifying an FTS3 table with no module arguments using a slightly # different syntax. -ddl_test 1.7.1 {CREATE VIRTUAL TABLE data USING fts3;} -read_test 1.7.2 {PRAGMA table_info(data)} {0 content {} 0 {} 0} -ddl_test 1.7.3 {DROP TABLE data} +ddl_test 1.1.8.1 {CREATE VIRTUAL TABLE data USING fts3;} +read_test 1.1.8.2 {PRAGMA table_info(data)} {0 content {} 0 {} 0} +ddl_test 1.1.8.3 {DROP TABLE data} + +########################################################################## + +########################################################################## +# Test the examples in section 1.2 (populating fts3 tables) +# +ddl_test 1.2.1.1 { + CREATE VIRTUAL TABLE pages USING fts3(title, body); +} +write_test 1.2.1.2 pages_content { + INSERT INTO pages(docid, title, body) + VALUES(53, 'Home Page', 'SQLite is a software...'); +} +read_test 1.2.1.3 { + SELECT docid, * FROM pages +} {53 {Home Page} {SQLite is a software...}} + +write_test 1.2.1.4 pages_content { + INSERT INTO pages(title, body) + VALUES('Download', 'All SQLite source code...'); +} +read_test 1.2.1.5 { + SELECT docid, * FROM pages +} {53 {Home Page} {SQLite is a software...} 54 Download {All SQLite source code...}} + +write_test 1.2.1.6 pages_content { + UPDATE pages SET title = 'Download SQLite' WHERE rowid = 54 +} +read_test 1.2.1.7 { + SELECT docid, * FROM pages +} {53 {Home Page} {SQLite is a software...} 54 {Download SQLite} {All SQLite source code...}} + +write_test 1.2.1.8 pages_content { DELETE FROM pages } +read_test 1.2.1.9 { SELECT docid, * FROM pages } {} + +do_error_test fts3-1.2.1.10 { + INSERT INTO pages(rowid, docid, title, body) VALUES(1, 2, 'A title', 'A document body'); +} {SQL logic error or missing database} + +# Test the optimize() function example: +if 0 { +ddl_test 1.2.2.1 { CREATE VIRTUAL TABLE docs USING fts3 } +write_test 1.2.2.2 docs_content { + INSERT INTO docs VALUES('Others translate the first clause as'); +} +write_test 1.2.2.3 docs_content { + INSERT INTO docs VALUES('"which is for Solomon," meaning that'); +} +write_test 1.2.2.4 docs_content { + INSERT INTO docs VALUES('the book is dedicated to Solomon.'); +} +read_test 1.2.2.5 { SELECT count(*) FROM docs_segdir } {3} +set DO_MALLOC_TEST 1 +write_test 1.2.2.6 docs_segdir { + INSERT INTO docs + SELECT * FROM (SELECT optimize(docs) FROM docs LIMIT 1) WHERE 0; +} +read_test 1.2.2.7 { SELECT count(*) FROM docs_segdir } {1} +} + +########################################################################## +# Test the examples in section 1.3 (querying FTS3 tables) +# +ddl_test 1.3.1.1 { CREATE VIRTUAL TABLE mail USING fts3(subject, body) } +read_test 1.3.1.2 { + SELECT * FROM mail WHERE rowid = 15; -- Fast. Rowid lookup. + SELECT * FROM mail WHERE body MATCH 'sqlite'; -- Fast. Full-text query. + SELECT * FROM mail WHERE mail MATCH 'search'; -- Fast. Full-text query. + SELECT * FROM mail WHERE rowid BETWEEN 15 AND 20; -- Slow. Linear scan. + SELECT * FROM mail WHERE subject = 'database'; -- Slow. Linear scan. + SELECT * FROM mail WHERE subject MATCH 'database'; -- Fast. Full-text query. +} {} +ddl_test 1.3.1.3 { DROP TABLE mail } + +ddl_test 1.3.2.1 { CREATE VIRTUAL TABLE mail USING fts3(subject, body) } + +write_test 1.3.2.2 mail_content { + INSERT INTO mail(docid, subject, body) + VALUES(1, 'software feedback', 'found it too slow') +} +write_test 1.3.2.3 mail_content { + INSERT INTO mail(docid, subject, body) + VALUES(2, 'software feedback', 'no feedback') +} +write_test 1.3.2.4 mail_content { + INSERT INTO mail(docid, subject, body) + VALUES(3, 'slow lunch order', 'was a software problem') +} +read_test 1.3.2.5 { + SELECT * FROM mail WHERE subject MATCH 'software' +} {{software feedback} {found it too slow} {software feedback} {no feedback}} +read_test 1.3.2.6 { + SELECT * FROM mail WHERE body MATCH 'feedback' +} {{software feedback} {no feedback}} +read_test 1.3.2.7 { + SELECT * FROM mail WHERE mail MATCH 'software' +} {{software feedback} {found it too slow} {software feedback} {no feedback} {slow lunch order} {was a software problem}} +read_test 1.3.2.7 { + SELECT * FROM mail WHERE mail MATCH 'slow' +} {{software feedback} {found it too slow} {slow lunch order} {was a software problem}} +ddl_test 1.3.2.8 { DROP TABLE mail } + +ddl_test 1.3.3.1 { CREATE VIRTUAL TABLE docs USING fts3(content) } +read_test 1.3.3.2 { SELECT * FROM docs WHERE docs MATCH 'sqlite' } {} +read_test 1.3.3.3 { SELECT * FROM docs WHERE docs.docs MATCH 'sqlite' } {} +read_test 1.3.3.4 { SELECT * FROM docs WHERE main.docs.docs MATCH 'sqlite' } {} +do_error_test e_fts3-1.3.3.5 { + SELECT * FROM docs WHERE main.docs MATCH 'sqlite' +} {no such column: main.docs} +ddl_test 1.3.2.8 { DROP TABLE docs } + +########################################################################## ########################################################################## +# Test the examples in section 3 (full-text index queries). +# +ddl_test 1.4.1.1 { CREATE VIRTUAL TABLE docs USING fts3(title, body) } +foreach {tn title body} { + 2 "linux driver" "a device" + 3 "driver" "linguistic trick" + 4 "problems" "linux problems" + 5 "linux" "big problems" + 6 "linux driver" "a device driver problem" + 7 "good times" "applications for linux" + 8 "not so good" "linux applications" + 9 "alternative" "linoleum appliances" + 10 "no L I N" "to be seen" +} { + write_test 1.4.1.$tn docs_content { INSERT INTO docs VALUES($title,$body) } + set R($tn) [list $title $body] +} + +read_test 1.4.1.11 { + SELECT * FROM docs WHERE docs MATCH 'linux' +} [concat $R(2) $R(4) $R(5) $R(6) $R(7) $R(8)] +read_test 1.4.1.12 { + SELECT * FROM docs WHERE docs MATCH 'lin*' +} [concat $R(2) $R(3) $R(4) $R(5) $R(6) $R(7) $R(8) $R(9)] +read_test 1.4.1.13 { + SELECT * FROM docs WHERE docs MATCH 'title:linux problems' +} [concat $R(5)] +read_test 1.4.1.14 { + SELECT * FROM docs WHERE body MATCH 'title:linux driver' +} [concat $R(6)] +read_test 1.4.1.15 { + SELECT * FROM docs WHERE docs MATCH '"linux applications"' +} [concat $R(8)] +read_test 1.4.1.16 { + SELECT * FROM docs WHERE docs MATCH '"lin* app*"' +} [concat $R(8) $R(9)] +ddl_test 1.4.1.17 { DROP TABLE docs } + +ddl_test 1.4.2.1 { CREATE VIRTUAL TABLE docs USING fts3() } +write_test 1.4.2.2 docs_content { + INSERT INTO docs VALUES( + 'SQLite is an ACID compliant embedded relational database management system') +} +foreach {tn query hit} { +3 {SELECT * FROM docs WHERE docs MATCH 'sqlite NEAR database'} 1 +4 {SELECT * FROM docs WHERE docs MATCH 'database NEAR/6 sqlite'} 1 +5 {SELECT * FROM docs WHERE docs MATCH 'database NEAR/5 sqlite'} 0 +6 {SELECT * FROM docs WHERE docs MATCH 'database NEAR/2 "ACID compliant"'} 1 +7 {SELECT * FROM docs WHERE docs MATCH '"ACID compliant" NEAR/2 sqlite'} 1 +8 {SELECT * FROM docs WHERE docs MATCH 'sqlite NEAR/2 acid NEAR/2 relational'} 1 +9 {SELECT * FROM docs WHERE docs MATCH 'acid NEAR/2 sqlite NEAR/2 relational'} 0 +} { + set res [db eval {SELECT * FROM docs WHERE $hit}] + read_test 1.4.2.$tn $query $res +} +ddl_test 1.4.2.10 { DROP TABLE docs } + +unset R + + +# TODO: Change numbering after here... ########################################################################## # Test the example in section 5 (custom tokenizers). @@ -120,6 +314,7 @@ write_test 2.2.2 porter_content { } read_test 2.2.3 {SELECT docid FROM porter WHERE porter MATCH 'Frustrated'} {1} read_test 2.2.4 {SELECT docid FROM porter WHERE porter MATCH 'Frustration'} {1} +########################################################################## ########################################################################## # Test the examples in section 4 (auxillary functions). @@ -163,5 +358,8 @@ read_test 3.2.4 { #break } +# End of tests of example code in fts3.html +#------------------------------------------------------------------------- + finish_test