From: dan Date: Wed, 24 Nov 2010 11:51:56 +0000 (+0000) Subject: Remove some unused code from fts3. Add tests to fts3matchinfo.test. X-Git-Tag: version-3.7.4~36^2~6 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ad3acbbfb3846bb062dfa103b202ebfb951a9590;p=thirdparty%2Fsqlite.git Remove some unused code from fts3. Add tests to fts3matchinfo.test. FossilOrigin-Name: ae40b34cf7c24c9601bdfb5cbe5b20f05a376ea8 --- diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 29a577bcd0..96a9736a9b 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2255,8 +2255,8 @@ static int fts3PhraseSelect( for(ii=0; iinToken; ii++){ Fts3PhraseToken *pTok; /* Token to find doclist for */ int iTok; /* The token being queried this iteration */ - char *pList; /* Pointer to token doclist */ - int nList; /* Size of buffer at pList */ + char *pList = 0; /* Pointer to token doclist */ + int nList = 0; /* Size of buffer at pList */ /* Select a token to process. If this is an xFilter() call, then tokens ** are processed in order from least to most costly. Otherwise, tokens @@ -3275,7 +3275,10 @@ static void fts3MatchinfoFunc( Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ assert( nVal==1 || nVal==2 ); if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ - const char *zArg = (nVal>1 ? sqlite3_value_text(apVal[1]) : 0); + const char *zArg = 0; + if( nVal>1 ){ + zArg = (const char *)sqlite3_value_text(apVal[1]); + } sqlite3Fts3Matchinfo(pContext, pCsr, zArg); } } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index e561663dff..5c4cbc181c 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -291,8 +291,6 @@ int sqlite3Fts3SegReaderIterate( ); int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); -int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*); -int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*); int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 34d0b0201a..4d0134c652 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -819,6 +819,12 @@ static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){ ** where X is the number of matches for phrase iPhrase is column iCol of all ** rows of the table. Y is the number of rows for which column iCol contains ** at least one instance of phrase iPhrase. +** +** If the phrase pExpr consists entirely of deferred tokens, then all X and +** Y values are set to nDoc, where nDoc is the number of documents in the +** file system. This is done because the full-text index doclist is required +** to calculate these values properly, and the full-text index doclist is +** not available for deferred tokens. */ static int fts3ExprGlobalHitsCb( Fts3Expr *pExpr, /* Phrase expression node */ @@ -903,13 +909,13 @@ static int fts3MatchinfoCheck( char cArg, char **pzErr ){ - if( cArg==FTS3_MATCHINFO_NPHRASE - || cArg==FTS3_MATCHINFO_NCOL - || cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat - || cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat - || cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize - || cArg==FTS3_MATCHINFO_LCS - || cArg==FTS3_MATCHINFO_HITS + if( (cArg==FTS3_MATCHINFO_NPHRASE) + || (cArg==FTS3_MATCHINFO_NCOL) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat) + || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) + || (cArg==FTS3_MATCHINFO_LCS) + || (cArg==FTS3_MATCHINFO_HITS) ){ return SQLITE_OK; } @@ -1182,7 +1188,7 @@ void sqlite3Fts3Snippet( ** columns of the FTS3 table. Otherwise, only column iCol is considered. */ for(iRead=0; iReadnColumn; iRead++){ - SnippetFragment sF; + SnippetFragment sF = {0, 0, 0, 0}; int iS; if( iCol>=0 && iRead!=iCol ) continue; diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 05f46f24e2..b18c429637 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -2488,75 +2488,6 @@ static void fts3DecodeIntArray( } } -/* -** Fill in the document size auxiliary information for the matchinfo -** structure. The auxiliary information is: -** -** N Total number of documents in the full-text index -** a0 Average length of column 0 over the whole index -** n0 Length of column 0 on the matching row -** ... -** aM Average length of column M over the whole index -** nM Length of column M on the matching row -** -** The fts3MatchinfoDocsizeLocal() routine fills in the nX values. -** The fts3MatchinfoDocsizeGlobal() routine fills in N and the aX values. -*/ -int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor *pCur, u32 *a){ - const char *pBlob; /* The BLOB holding %_docsize info */ - int nBlob; /* Size of the BLOB */ - sqlite3_stmt *pStmt; /* Statement for reading and writing */ - int i, j; /* Loop counters */ - sqlite3_int64 x; /* Varint value */ - int rc; /* Result code from subfunctions */ - Fts3Table *p; /* The FTS table */ - - p = (Fts3Table*)pCur->base.pVtab; - rc = fts3SqlStmt(p, SQL_SELECT_DOCSIZE, &pStmt, 0); - if( rc ){ - return rc; - } - sqlite3_bind_int64(pStmt, 1, pCur->iPrevId); - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - nBlob = sqlite3_column_bytes(pStmt, 0); - pBlob = (const char*)sqlite3_column_blob(pStmt, 0); - for(i=j=0; inColumn && jbase.pVtab; - rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); - if( rc ){ - return rc; - } - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - nBlob = sqlite3_column_bytes(pStmt, 0); - pBlob = (const char*)sqlite3_column_blob(pStmt, 0); - j = sqlite3Fts3GetVarint(pBlob, &x); - a[0] = nDoc = (u32)(x & 0xffffffff); - for(i=0; inColumn && jiPrevDocid. The sizes are encoded as diff --git a/manifest b/manifest index 5ef19efb6a..34f01f6978 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\schanges\sto\sfts3\sfunction\smatchinfo(). -D 2010-11-23T19:16:48 +C Remove\ssome\sunused\scode\sfrom\sfts3.\sAdd\stests\sto\sfts3matchinfo.test. +D 2010-11-24T11:51:56 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,19 +61,19 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 591b6fdb563dd00cd05b57ea27272f388e8156c8 +F ext/fts3/fts3.c 916091c7aeeefa8a7017865d97d10d13a5949b52 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 0f7b9f189bcb0eed43a9489d2251b99acf06b833 +F ext/fts3/fts3Int.h 52c818623c60943bc4ac4a22d77b2e8f28395e78 F ext/fts3/fts3_expr.c ee48b9278b8b2432a05a03320fbcacba151dbaa5 F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c 8df6f6efcc4e9e31f8bf73a4007c2e9abca1dfba -F ext/fts3/fts3_snippet.c aa0a76088024a693ab6a51d0d383e06f97e4fd8c +F ext/fts3/fts3_snippet.c 967ca2d3201fd6555062c7e929bcc2b89ef8dcb8 F ext/fts3/fts3_tokenizer.c 1301b0ee3ef414caae3257a702215925cc48cd9c F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c f5e7db79057ea46c1f3d15a4e2c8b658d306e37f +F ext/fts3/fts3_write.c 9b2db92b815fdd50b5531eb6db912c71feca6a70 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -446,7 +446,7 @@ F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test 81fd40ceb12f33f9d16c5637d0f8d95d4556c456 F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b -F test/fts3matchinfo.test 051bc3d09152998cedb7b9bdfd5b0730b5c0a939 +F test/fts3matchinfo.test cf3d0149f749abdb228c7bfe2350cf799b1809f4 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 707533ce943f490443ce5e696236bb1675a37635 @@ -889,10 +889,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 3df3e79b56821201b4f5ecd23f94d485745c48c3 -R d2823d2eb839cb4c673b301711f75566 -T *branch * fts3-experimental -T *sym-fts3-experimental * -T -sym-trunk * +P 9cf0f2b76bc68c168e3fa861b7235f384db21d38 +R 9d3e5478b917e1cb6e46d1d7c8efa28d U dan -Z cb34d18554c4ac084411d81cdabfad29 +Z 2747212062c29415eaa46cf5d82fabf7 diff --git a/manifest.uuid b/manifest.uuid index 1e10a20f1a..cea5ccd060 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9cf0f2b76bc68c168e3fa861b7235f384db21d38 \ No newline at end of file +ae40b34cf7c24c9601bdfb5cbe5b20f05a376ea8 \ No newline at end of file diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index 36b85aed5d..88166411c6 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -66,4 +66,141 @@ do_execsql_test 3.1 { SELECT mtchinfo FROM t3; } {{Beside the lake, beneath the trees}} + +#-------------------------------------------------------------------------- +# Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function. +# +# The first argument - $tn - is a test identifier. This may be either a +# full identifier (i.e. "fts3matchinfo-1.1") or, if global var $testprefix +# is set, just the numeric component (i.e. "1.1"). +# +# The second argument is the name of an FTSX table. The third is the +# full text of a WHERE/MATCH expression to query the table for +# (i.e. "t1 MATCH 'abc'"). The final argument - $results - should be a +# key-value list (serialized array) with matchinfo() format specifiers +# as keys, and the results of executing the statement: +# +# SELECT matchinfo($tbl, '$key') FROM $tbl WHERE $expr +# +# For example: +# +# CREATE VIRTUAL TABLE t1 USING fts4; +# INSERT INTO t1 VALUES('abc'); +# INSERT INTO t1 VALUES('def'); +# INSERT INTO t1 VALUES('abc abc'); +# +# do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" { +# n {3 3} +# p {1 1} +# c {1 1} +# x {{1 3 2} {2 3 2}} +# } +# +# If the $results list contains keys mapped to "-" instead of a matchinfo() +# result, then this command computes the expected results based on other +# mappings to test the matchinfo() function. For example, the command above +# could be changed to: +# +# do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" { +# n {3 3} p {1 1} c {1 1} x {{1 3 2} {2 3 2}} +# pcx - +# } +# +# And this command would compute the expected results for matchinfo(t1, 'pcx') +# based on the results of matchinfo(t1, 'p'), matchinfo(t1, 'c') and +# matchinfo(t1, 'x') in order to test 'pcx'. +# +proc do_matchinfo_test {tn tbl expr results} { + + foreach {fmt res} $results { + if {$res == "-"} continue + set resarray($fmt) $res + } + + set nRow 0 + foreach {fmt res} [array get resarray] { + if {[llength $res]>$nRow} { set nRow [llength $res] } + } + + # Construct expected results for any formats for which the caller + # supplied result is "-". + # + foreach {fmt res} $results { + if {$res == "-"} { + set res [list] + for {set iRow 0} {$iRow<$nRow} {incr iRow} { + set rowres [list] + foreach c [split $fmt ""] { + set rowres [concat $rowres [lindex $resarray($c) $iRow]] + } + lappend res $rowres + } + set resarray($fmt) $res + } + } + + # Test each matchinfo() request individually. + # + foreach {fmt res} [array get resarray] { + set sql "SELECT mit(matchinfo($tbl, '$fmt')) FROM $tbl WHERE $expr" + do_execsql_test $tn.$fmt $sql [normalize2 $res] + } + + # Test them all executed together (multiple invocations of matchinfo()). + # + set exprlist [list] + foreach {format res} [array get resarray] { + lappend exprlist "mit(matchinfo($tbl, '$format'))" + } + set allres [list] + for {set iRow 0} {$iRow<$nRow} {incr iRow} { + foreach {format res} [array get resarray] { + lappend allres [lindex $res $iRow] + } + } + set sql "SELECT [join $exprlist ,] FROM $tbl WHERE $expr" + do_execsql_test $tn.multi $sql [normalize2 $allres] +} +proc normalize2 {list_of_lists} { + set res [list] + foreach elem $list_of_lists { + lappend res [list {*}$elem] + } + return $res +} + + +do_execsql_test 4.1.0 { + CREATE VIRTUAL TABLE t4 USING fts4(x, y); + INSERT INTO t4 VALUES('a b c d e', 'f g h i j'); + INSERT INTO t4 VALUES('f g h i j', 'a b c d e'); +} +do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} { + p {3 3} + c {2 2} + x { + {1 1 1 0 1 1 1 1 1 0 1 1 1 1 1 0 1 1} + {0 1 1 1 1 1 0 1 1 1 1 1 0 1 1 1 1 1} + } + n {2 2} + l {{5 5} {5 5}} + a {{5 5} {5 5}} + + xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc - +} +do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} { + p {1 1} + c {2 2} + x { + {0 1 1 1 1 1} + {1 1 1 0 1 1} + } + n {2 2} + l {{5 5} {5 5}} + a {{5 5} {5 5}} + + xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc - +} + + finish_test