From: shess Date: Tue, 1 May 2007 18:25:52 +0000 (+0000) Subject: Enable prefix-search in query-parsing and snippet generation. If the X-Git-Tag: version-3.4.0~188 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=290283fe696e1a21e38e31d4d28e19ee5ebf02a4;p=thirdparty%2Fsqlite.git Enable prefix-search in query-parsing and snippet generation. If the character immediately after the end of a term is '*', that term is marked for prefix matching. Modify term comparison in snippetOffsetsOfColumn() to respect isPrefix. fts2n.test runs prefix searching through some obvious test cases. (CVS 3893) FossilOrigin-Name: 7c4c65924035d9f260f6b64eb92c5c6cf6c04b7b --- diff --git a/ext/fts2/fts2.c b/ext/fts2/fts2.c index 8eaf28479c..1f0809a364 100644 --- a/ext/fts2/fts2.c +++ b/ext/fts2/fts2.c @@ -3056,8 +3056,10 @@ static void snippetOffsetsOfColumn( int iCol; iCol = aTerm[i].iColumn; if( iCol>=0 && iColnToken ) continue; + if( !aTerm[i].isPrefix && aTerm[i].nTerm1 && (prevMatch & (1<0 && pSegment[iBegin-1]=='-' ){ pQuery->pTerms[pQuery->nTerms-1].isNot = 1; } + if( iEndpTerms[pQuery->nTerms-1].isPrefix = 1; + } pQuery->pTerms[pQuery->nTerms-1].iPhrase = nTerm; if( inPhrase ){ nTerm++; diff --git a/manifest b/manifest index d715b33c0b..cabd8f7348 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C First\sapproximation\sof\sincremental\sblob\sIO\sAPI.\s(CVS\s3892) -D 2007-05-01T17:49:49 +C Enable\sprefix-search\sin\squery-parsing\sand\ssnippet\sgeneration.\s\sIf\sthe\ncharacter\simmediately\safter\sthe\send\sof\sa\sterm\sis\s'*',\sthat\sterm\sis\nmarked\sfor\sprefix\smatching.\s\sModify\sterm\scomparison\sin\nsnippetOffsetsOfColumn()\sto\srespect\sisPrefix.\s\sfts2n.test\sruns\sprefix\nsearching\sthrough\ssome\sobvious\stest\scases.\s(CVS\s3893) +D 2007-05-01T18:25:53 F Makefile.in 8cab54f7c9f5af8f22fd97ddf1ecfd1e1860de62 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -34,7 +34,7 @@ F ext/fts1/fulltext.h 08525a47852d1d62a0be81d3fc3fe2d23b094efd F ext/fts1/simple_tokenizer.c 1844d72f7194c3fd3d7e4173053911bf0661b70d F ext/fts1/tokenizer.h 0c53421b832366d20d720d21ea3e1f6e66a36ef9 F ext/fts2/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts2/fts2.c a6762b7a6cc173eb83a0aa9506c1b7be66f00786 +F ext/fts2/fts2.c 9e1f5942fc063f2d6778bc44372eba9b2b43eebd F ext/fts2/fts2.h 591916a822cfb6426518fdbf6069359119bc46eb F ext/fts2/fts2_hash.c b3f22116d4ef0bc8f2da6e3fdc435c86d0951a9b F ext/fts2/fts2_hash.h e283308156018329f042816eb09334df714e105e @@ -236,6 +236,7 @@ F test/fts2j.test f68d7611f76309bc8b94170f3740d9fbbc061d9b F test/fts2k.test 222d0b3bc8667753f18406aaea9906a6098ea016 F test/fts2l.test 4c53c89ce3919003765ff4fd8d98ecf724d97dd3 F test/fts2m.test 4b30142ead6f3ed076e880a2a464064c5ad58c51 +F test/fts2n.test a70357e72742681eaebfdbe9007b87ff3b771638 F test/func.test 8a3bc8e8365dc0053c826923c0f738645f50f2f5 F test/hook.test 7e7645fd9a033f79cce8fdff151e32715e7ec50a F test/in.test 369cb2aa1eab02296b4ec470732fe8c131260b1d @@ -468,7 +469,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5 -P 72c796307338c2751a91c30f6fb16989afbf3816 -R 30bd64cea77836fbaf98d3678ccdaa09 -U danielk1977 -Z 5cdc2c295a9fcd37c90d369f1b123096 +P c444836e7b690c16dd6acff571c613a23beb42dc +R 15961852c9b829c1a838faae38aeed43 +U shess +Z c808046b1e2602c778234ecda5713f7c diff --git a/manifest.uuid b/manifest.uuid index 9801a2e4d0..657800ab3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c444836e7b690c16dd6acff571c613a23beb42dc \ No newline at end of file +7c4c65924035d9f260f6b64eb92c5c6cf6c04b7b \ No newline at end of file diff --git a/test/fts2n.test b/test/fts2n.test new file mode 100644 index 0000000000..4bcbeef592 --- /dev/null +++ b/test/fts2n.test @@ -0,0 +1,196 @@ +# 2007 April 26 +# +# The author disclaims copyright to this source code. +# +#************************************************************************* +# This file implements tests for prefix-searching in the fts2 +# component of the SQLite library. +# +# $Id: fts2n.test,v 1.1 2007/05/01 18:25:53 shess Exp $ +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If SQLITE_ENABLE_FTS2 is defined, omit this file. +ifcapable !fts2 { + finish_test + return +} + +# A large string to prime the pump with. +set text { + Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Maecenas + iaculis mollis ipsum. Praesent rhoncus placerat justo. Duis non quam + sed turpis posuere placerat. Curabitur et lorem in lorem porttitor + aliquet. Pellentesque bibendum tincidunt diam. Vestibulum blandit + ante nec elit. In sapien diam, facilisis eget, dictum sed, viverra + at, felis. Vestibulum magna. Sed magna dolor, vestibulum rhoncus, + ornare vel, vulputate sit amet, felis. Integer malesuada, tellus at + luctus gravida, diam nunc porta nibh, nec imperdiet massa metus eu + lectus. Aliquam nisi. Nunc fringilla nulla at lectus. Suspendisse + potenti. Cum sociis natoque penatibus et magnis dis parturient + montes, nascetur ridiculus mus. Pellentesque odio nulla, feugiat eu, + suscipit nec, consequat quis, risus. +} + +db eval { + CREATE VIRTUAL TABLE t1 USING fts2(c); + + INSERT INTO t1(rowid, c) VALUES(1, $text); + INSERT INTO t1(rowid, c) VALUES(2, 'Another lovely row'); +} + +# Exact match +do_test fts2n-1.1 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lorem'" +} {1} + +# And a prefix +do_test fts2n-1.2 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lore*'" +} {1} + +# Prefix includes exact match +do_test fts2n-1.3 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lorem*'" +} {1} + +# Make certain everything isn't considered a prefix! +do_test fts2n-1.4 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lore'" +} {} + +# Prefix across multiple rows. +do_test fts2n-1.5 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lo*'" +} {1 2} + +# Likewise, with multiple hits in one document. +do_test fts2n-1.6 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'l*'" +} {1 2} + +# Prefix which should only hit one document. +do_test fts2n-1.7 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lov*'" +} {2} + +# * not at end is dropped. +do_test fts2n-1.8 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lo *'" +} {} + +# Stand-alone * is dropped. +do_test fts2n-1.9 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH '*'" +} {} + +# Phrase-query prefix. +do_test fts2n-1.10 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"lovely r*\"'" +} {2} +do_test fts2n-1.11 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"lovely r\"'" +} {} + +# Phrase query with multiple prefix matches. +do_test fts2n-1.12 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"a* l*\"'" +} {1 2} + +# Phrase query with multiple prefix matches. +do_test fts2n-1.13 { + execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"a* l* row\"'" +} {2} + + + + +# Test across updates (and, by implication, deletes). + +# Version of text without "lorem". +regsub -all {[Ll]orem} $text '' ntext + +db eval { + CREATE VIRTUAL TABLE t2 USING fts2(c); + + INSERT INTO t2(rowid, c) VALUES(1, $text); + INSERT INTO t2(rowid, c) VALUES(2, 'Another lovely row'); + UPDATE t2 SET c = $ntext WHERE rowid = 1; +} + +# Can't see lorem as an exact match. +do_test fts2n-2.1 { + execsql "SELECT rowid FROM t2 WHERE t2 MATCH 'lorem'" +} {} + +# Can't see a prefix of lorem, either. +do_test fts2n-2.2 { + execsql "SELECT rowid FROM t2 WHERE t2 MATCH 'lore*'" +} {} + +# Can see lovely in the other document. +do_test fts2n-2.3 { + execsql "SELECT rowid FROM t2 WHERE t2 MATCH 'lo*'" +} {2} + +# Can still see other hits. +do_test fts2n-2.4 { + execsql "SELECT rowid FROM t2 WHERE t2 MATCH 'l*'" +} {1 2} + +# Prefix which should only hit one document. +do_test fts2n-2.5 { + execsql "SELECT rowid FROM t2 WHERE t2 MATCH 'lov*'" +} {2} + + + +# Test with a segment which will have multiple levels in the tree. + +# Build a big document with lots of unique terms. +set bigtext $text +foreach c {a b c d e} { + regsub -all {[A-Za-z]+} $bigtext "&$c" t + append bigtext $t +} + +# Populate a table with many copies of the big document, so that we +# can test the number of hits found. Populate $ret with the expected +# hit counts for each row. offsets() returns 4 elements for every +# hit. We'll have 6 hits for row 1, 1 for row 2, and 6*(2^5)==192 for +# $bigtext. +set ret {6 1} +db eval { + BEGIN; + CREATE VIRTUAL TABLE t3 USING fts2(c); + + INSERT INTO t3(rowid, c) VALUES(1, $text); + INSERT INTO t3(rowid, c) VALUES(2, 'Another lovely row'); +} +for {set i 0} {$i<100} {incr i} { + db eval {INSERT INTO t3(rowid, c) VALUES(3+$i, $bigtext)} + lappend ret 192 +} +db eval {COMMIT;} + +# Test that we get the expected number of hits. +do_test fts2n-3.1 { + set t {} + db eval {SELECT offsets(t3) as o FROM t3 WHERE t3 MATCH 'l*'} { + set l [llength $o] + lappend t [expr {$l/4}] + } + set t +} $ret + +# TODO(shess) It would be useful to test a couple edge cases, but I +# don't know if we have the precision to manage it from here at this +# time. Prefix hits can cross leaves, which the code above _should_ +# hit by virtue of size. There are two variations on this. If the +# tree is 2 levels high, the code will find the leaf-node extent +# directly, but if it's higher, the code will have to follow two +# separate interior branches down the tree. Both should be tested. + +finish_test