From: dan Date: Mon, 6 Jun 2011 14:51:50 +0000 (+0000) Subject: Modify fts3rnd.test to run tests for both "ORDER BY docid ASC" and "ORDER BY docid... X-Git-Tag: version-3.7.7~62^2~12 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5289b01524c329af93ae44dfb70c8303ed553203;p=thirdparty%2Fsqlite.git Modify fts3rnd.test to run tests for both "ORDER BY docid ASC" and "ORDER BY docid DESC" with both order=ASC and order=DESC FTS tables. Fixes for some bugs found. FossilOrigin-Name: 89f2f482e077241ac29a58eadf44a72a9c01f98c --- diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 317b2b5625..18d7e79b78 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1144,6 +1144,7 @@ fts3_init_out: pTokenizer->pModule->xDestroy(pTokenizer); } }else{ + assert( p->pSegments==0 ); *ppVTab = &p->base; } return rc; @@ -1245,6 +1246,7 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ } } + assert( p->pSegments==0 ); return SQLITE_OK; } @@ -1280,6 +1282,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); sqlite3_free(pCsr->aMatchinfo); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_free(pCsr); return SQLITE_OK; } @@ -2564,6 +2567,7 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ }else{ rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor); } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -2701,7 +2705,7 @@ static int fts3ColumnMethod( sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; Fts3Table *p = (Fts3Table *)pCursor->pVtab; @@ -2712,21 +2716,20 @@ static int fts3ColumnMethod( /* This call is a request for the "docid" column. Since "docid" is an ** alias for "rowid", use the xRowid() method to obtain the value. */ - sqlite3_int64 iRowid; - rc = fts3RowidMethod(pCursor, &iRowid); - sqlite3_result_int64(pContext, iRowid); + sqlite3_result_int64(pContext, pCsr->iPrevId); }else if( iCol==p->nColumn ){ /* The extra column whose name is the same as the table. ** Return a blob which is a pointer to the cursor. */ sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - rc = SQLITE_OK; }else{ rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); } } + + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -2760,6 +2763,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ static int fts3BeginMethod(sqlite3_vtab *pVtab){ UNUSED_PARAMETER(pVtab); TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + assert( p->pSegments==0 ); assert( p->nPendingData==0 ); assert( p->inTransaction!=1 ); TESTONLY( p->inTransaction = 1 ); @@ -2777,6 +2781,7 @@ static int fts3CommitMethod(sqlite3_vtab *pVtab){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); assert( p->nPendingData==0 ); assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); TESTONLY( p->inTransaction = 0 ); TESTONLY( p->mxSavepoint = -1; ); return SQLITE_OK; @@ -3047,7 +3052,7 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ assert( p->inTransaction ); assert( p->mxSavepoint < iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint ); - return sqlite3Fts3PendingTermsFlush(p); + return fts3SyncMethod(pVtab); } static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); @@ -3459,6 +3464,10 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ return rc; } +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. +*/ void sqlite3Fts3DoclistPrev( int bDescIdx, /* True if the doclist is desc */ char *aDoclist, /* Pointer to entire doclist */ @@ -3471,6 +3480,7 @@ void sqlite3Fts3DoclistPrev( char *p = *ppIter; int iMul = (bDescIdx ? -1 : 1); + assert( nDoclist>0 ); assert( *pbEof==0 ); assert( p || *piDocid==0 ); assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); @@ -3537,7 +3547,7 @@ static int fts3EvalPhraseNext( if( rc==SQLITE_OK && !pDL->pList ){ *pbEof = 1; } - }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->aAll ){ + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof ); @@ -3879,6 +3889,7 @@ static void fts3EvalNext( int *pRc ){ if( *pRc==SQLITE_OK ){ + assert( pExpr->bEof==0 ); pExpr->bStart = 1; switch( pExpr->eType ){ @@ -3901,7 +3912,7 @@ static void fts3EvalNext( fts3EvalNext(pCsr, pRight, pRc); while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - int iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); if( iDiff==0 ) break; if( iDiff<0 ){ fts3EvalNext(pCsr, pLeft, pRc); @@ -3919,20 +3930,20 @@ static void fts3EvalNext( case FTSQUERY_OR: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; - int iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - if( iCmp==0 ){ + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ fts3EvalNext(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ fts3EvalNext(pCsr, pRight, pRc); - }else if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - fts3EvalNext(pCsr, pLeft, pRc); }else{ + fts3EvalNext(pCsr, pLeft, pRc); fts3EvalNext(pCsr, pRight, pRc); } - + pExpr->bEof = (pLeft->bEof && pRight->bEof); iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ @@ -3972,6 +3983,12 @@ static void fts3EvalNext( fts3EvalFreeDeferredDoclist(pPhrase); *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); pExpr->iDocid = pPhrase->doclist.iDocid; +#if 0 + printf("token \"%.*s\" docid=%lld\n", + pPhrase->aToken[0].n, pPhrase->aToken[0].z, pExpr->iDocid + ); +#endif + break; } } @@ -4123,8 +4140,9 @@ int sqlite3Fts3EvalPhraseDoclist( assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); if( pExpr->bStart && !pExpr->bEof ){ pExpr->bStart = 0; - while( rc==SQLITE_OK && pExpr->bEof==0 && pExpr->iDocid!=iDocid ){ + while( rc==SQLITE_OK && (pExpr->bStart==0 || pExpr->iDocid!=iDocid) ){ fts3EvalNext(pCsr, pExpr, &rc); + assert( !pExpr->bEof ); } } } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index fd964739b2..a72556384c 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3159,7 +3159,7 @@ int sqlite3Fts3UpdateMethod( int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ - u32 *aSzIns; /* Sizes of inserted documents */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ int bInsertDone = 0; @@ -3174,12 +3174,16 @@ int sqlite3Fts3UpdateMethod( && sqlite3_value_type(apVal[0])==SQLITE_NULL && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ - return fts3SpecialInsert(p, apVal[p->nColumn+2]); + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; } /* Allocate space to hold the change in document sizes */ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); - if( aSzIns==0 ) return SQLITE_NOMEM; + if( aSzIns==0 ){ + rc = SQLITE_NOMEM; + goto update_out; + } aSzDel = &aSzIns[p->nColumn+1]; memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); @@ -3229,8 +3233,7 @@ int sqlite3Fts3UpdateMethod( } } if( rc!=SQLITE_OK ){ - sqlite3_free(aSzIns); - return rc; + goto update_out; } /* If this is a DELETE or UPDATE operation, remove the old record. */ @@ -3263,6 +3266,7 @@ int sqlite3Fts3UpdateMethod( fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } + update_out: sqlite3_free(aSzIns); sqlite3Fts3SegmentsClose(p); return rc; diff --git a/manifest b/manifest index 3294d4e1c4..e6e2af3449 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\sthe\scode\sfor\sprocessing\sFTS4\soptions\sa\sbit. -D 2011-06-06T06:55:38.365 +C Modify\sfts3rnd.test\sto\srun\stests\sfor\sboth\s"ORDER\sBY\sdocid\sASC"\sand\s"ORDER\sBY\sdocid\sDESC"\swith\sboth\sorder=ASC\sand\sorder=DESC\sFTS\stables.\sFixes\sfor\ssome\sbugs\sfound. +D 2011-06-06T14:51:50.541 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c ea03065c8fbffc7258777fd9e1bf1c40404ce14a +F ext/fts3/fts3.c aafc143b59710e85a6215e8843aa2802f38397bc F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f @@ -75,7 +75,7 @@ F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c d7eded6f5ee3032b41126047cc04b6720f59e6da +F ext/fts3/fts3_write.c ed525afd524d713abe7da174d56ad935dfc26008 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -475,7 +475,7 @@ F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 -F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad +F test/fts3rnd.test d88ec3dbe52e81e410cd1a39554d15941f86333c F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 650e1a79eda5a2134a1fbd305ab1f205a57c0892 -R d2808d37d9a0cf28002d639dd1fa255c +P 0425138a2365d23b07d88fda2b1f458f112f389d +R bea111e886e30c118babca4cc7623bc5 U dan -Z 406d2ac578f3de4e57d0cc24bb1a90c8 +Z 6f031e903f656ed90101feb3a80e60ca diff --git a/manifest.uuid b/manifest.uuid index f78050d023..118c979d31 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0425138a2365d23b07d88fda2b1f458f112f389d \ No newline at end of file +89f2f482e077241ac29a58eadf44a72a9c01f98c \ No newline at end of file diff --git a/test/fts3rnd.test b/test/fts3rnd.test index 0909cee614..0ad41d8ce4 100644 --- a/test/fts3rnd.test +++ b/test/fts3rnd.test @@ -162,7 +162,7 @@ proc simple_phrase {zPrefix} { # This [proc] is used to test the FTS3 matchinfo() function. # -proc simple_token_matchinfo {zToken} { +proc simple_token_matchinfo {zToken bDesc} { set nDoc(0) 0 set nDoc(1) 0 @@ -171,6 +171,8 @@ proc simple_token_matchinfo {zToken} { set nHit(1) 0 set nHit(2) 0 + set dir -inc + if {$bDesc} { set dir -dec } foreach key [array names ::t1] { set value $::t1($key) @@ -184,7 +186,7 @@ proc simple_token_matchinfo {zToken} { } set ret [list] - foreach docid [lsort -integer [array names a]] { + foreach docid [lsort -integer $dir [array names a]] { if { [lindex [lsort -integer $a($docid)] end] } { set matchinfo [list 1 3] foreach i {0 1 2} hit $a($docid) { @@ -262,17 +264,30 @@ proc mit {blob} { return $r } db func mit mit - set sqlite_fts3_enable_parentheses 1 -foreach nodesize {50 500 1000 2000} { +proc do_orderbydocid_test {tn sql res} { + uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res] + uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \ + [lsort -int -dec $res] + ] +} + +foreach {nodesize order} { + 50 DESC + 50 ASC + 500 ASC + 1000 DESC + 2000 ASC +} { catch { array unset ::t1 } + set testname "$nodesize/$order" # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows. # db transaction { catchsql { DROP TABLE t1 } - execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)" + execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)" execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')" for {set i 0} {$i < 100} {incr i} { insert_row $i } } @@ -286,6 +301,8 @@ foreach nodesize {50 500 1000 2000} { set DO_MALLOC_TEST 1 set nRep 2 } + + set ::testprefix fts3rnd-1.$testname.$iTest # Delete one row, update one row and insert one row. # @@ -307,7 +324,7 @@ foreach nodesize {50 500 1000 2000} { if {0==($iTest%2)} { execsql COMMIT } if {0==($iTest%2)} { - do_test fts3rnd-1.$nodesize.$iTest.0 { fts3_integrity_check t1 } ok + #do_test 0 { fts3_integrity_check t1 } ok } # Pick 10 terms from the vocabulary. Check that the results of querying @@ -317,9 +334,14 @@ foreach nodesize {50 500 1000 2000} { # for {set i 0} {$i < 10} {incr i} { set term [random_term] - do_select_test fts3rnd-1.$nodesize.$iTest.1.$i { + do_select_test 1.$i { + SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term + ORDER BY docid ASC + } [simple_token_matchinfo $term 0] + do_select_test 1.$i { SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term - } [simple_token_matchinfo $term] + ORDER BY docid DESC + } [simple_token_matchinfo $term 1] } # This time, use the first two characters of each term as a term prefix @@ -329,7 +351,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set prefix [string range [random_term] 0 end-1] set match "${prefix}*" - do_select_test fts3rnd-1.$nodesize.$iTest.2.$i { + do_orderbydocid_test 2.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $match] } @@ -339,7 +361,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set term [list [random_term] [random_term]] set match "\"$term\"" - do_select_test fts3rnd-1.$nodesize.$iTest.3.$i { + do_orderbydocid_test 3.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $term] } @@ -349,7 +371,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set term [list [random_term] [random_term] [random_term]] set match "\"$term\"" - do_select_test fts3rnd-1.$nodesize.$iTest.4.$i { + do_orderbydocid_test 4.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $term] } @@ -362,17 +384,19 @@ foreach nodesize {50 500 1000 2000} { append query "[string range [random_term] 0 end-1]*" set match "\"$query\"" - do_select_test fts3rnd-1.$nodesize.$iTest.5.$i { + do_orderbydocid_test 5.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $query] } - # A NEAR query with terms as the arguments. + # A NEAR query with terms as the arguments: + # + # ... MATCH '$term1 NEAR $term2' ... # for {set i 0} {$i < $nRep} {incr i} { set terms [list [random_term] [random_term]] set match [join $terms " NEAR "] - do_select_test fts3rnd-1.$nodesize.$iTest.6.$i { + do_orderbydocid_test 6.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_near $terms 10] } @@ -383,7 +407,7 @@ foreach nodesize {50 500 1000 2000} { set terms [list [random_term] [random_term] [random_term]] set nNear 11 set match [join $terms " NEAR/$nNear "] - do_select_test fts3rnd-1.$nodesize.$iTest.7.$i { + do_orderbydocid_test 7.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_near $terms $nNear] } @@ -399,7 +423,7 @@ foreach nodesize {50 500 1000 2000} { set term1 [random_term] set term2 [random_term] set match "$term1 $op $term2" - do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + do_orderbydocid_test $tn.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [$proc [simple_phrase $term1] [simple_phrase $term2]] } @@ -418,7 +442,7 @@ foreach nodesize {50 500 1000 2000} { set term3 [random_term] set term4 [random_term] set match "$term1 NEAR $term2 $op $term3 NEAR $term4" - do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + do_orderbydocid_test $tn.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [$proc \ [simple_near [list $term1 $term2] 10] \