From: dan Date: Thu, 7 Jan 2016 20:07:41 +0000 (+0000) Subject: Update the fts5vocab module to work with detail=col and detail=none tables. X-Git-Tag: version-3.11.0~157^2~6 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=2598b833f06bbf6f8184ea74110ccd427fc8795c;p=thirdparty%2Fsqlite.git Update the fts5vocab module to work with detail=col and detail=none tables. FossilOrigin-Name: eedd095dc1c81ce45df00093ba237dd7b3cdff3d --- diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index 860cfedb9b..2ed10b7cdb 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -379,7 +379,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ if( pTab->eType==FTS5_VOCAB_COL ){ for(pCsr->iCol++; pCsr->iColiCol++){ - if( pCsr->aCnt[pCsr->iCol] ) break; + if( pCsr->aDoc[pCsr->iCol] ) break; } } @@ -412,24 +412,52 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ i64 iPos = 0; /* 64-bit position read from poslist */ int iOff = 0; /* Current offset within position list */ - rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy); - if( rc==SQLITE_OK ){ - if( pTab->eType==FTS5_VOCAB_ROW ){ - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - pCsr->aCnt[0]++; + switch( pCsr->pConfig->eDetail ){ + case FTS5_DETAIL_FULL: + rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy); + if( rc==SQLITE_OK ){ + if( pTab->eType==FTS5_VOCAB_ROW ){ + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + pCsr->aCnt[0]++; + } + pCsr->aDoc[0]++; + }else{ + int iCol = -1; + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + int ii = FTS5_POS2COLUMN(iPos); + pCsr->aCnt[ii]++; + if( iCol!=ii ){ + pCsr->aDoc[ii]++; + iCol = ii; + } + } + } } - pCsr->aDoc[0]++; - }else{ - int iCol = -1; - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - int ii = FTS5_POS2COLUMN(iPos); - pCsr->aCnt[ii]++; - if( iCol!=ii ){ - pCsr->aDoc[ii]++; - iCol = ii; + break; + + case FTS5_DETAIL_COLUMNS: + if( pTab->eType==FTS5_VOCAB_ROW ){ + pCsr->aDoc[0]++; + }else{ + Fts5Buffer buf = {0, 0, 0}; + rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf); + if( rc==SQLITE_OK ){ + while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){ + assert_nc( iPos>=0 && iPosaDoc[iPos]++; + } } + sqlite3Fts5BufferFree(&buf); } - } + break; + + default: + assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE ); + pCsr->aDoc[0]++; + break; + } + + if( rc==SQLITE_OK ){ rc = sqlite3Fts5IterNextScan(pCsr->pIter); } @@ -445,7 +473,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ } if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ - while( pCsr->aCnt[pCsr->iCol]==0 ) pCsr->iCol++; + while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++; assert( pCsr->iColpConfig->nCol ); } return rc; @@ -525,30 +553,36 @@ static int fts5VocabColumnMethod( int iCol /* Index of column to read value from */ ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int eDetail = pCsr->pConfig->eDetail; + int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType; + i64 iVal = 0; if( iCol==0 ){ sqlite3_result_text( pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT ); - } - else if( ((Fts5VocabTable*)(pCursor->pVtab))->eType==FTS5_VOCAB_COL ){ + }else if( eType==FTS5_VOCAB_COL ){ assert( iCol==1 || iCol==2 || iCol==3 ); if( iCol==1 ){ - const char *z = pCsr->pConfig->azCol[pCsr->iCol]; - sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + if( eDetail!=FTS5_DETAIL_NONE ){ + const char *z = pCsr->pConfig->azCol[pCsr->iCol]; + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + } }else if( iCol==2 ){ - sqlite3_result_int64(pCtx, pCsr->aDoc[pCsr->iCol]); + iVal = pCsr->aDoc[pCsr->iCol]; }else{ - sqlite3_result_int64(pCtx, pCsr->aCnt[pCsr->iCol]); + iVal = pCsr->aCnt[pCsr->iCol]; } }else{ assert( iCol==1 || iCol==2 ); if( iCol==1 ){ - sqlite3_result_int64(pCtx, pCsr->aDoc[0]); + iVal = pCsr->aDoc[0]; }else{ - sqlite3_result_int64(pCtx, pCsr->aCnt[0]); + iVal = pCsr->aCnt[0]; } } + + if( iVal>0 ) sqlite3_result_int64(pCtx, iVal); return SQLITE_OK; } diff --git a/ext/fts5/test/fts5vocab.test b/ext/fts5/test/fts5vocab.test index dc5099c6e3..1f5fa91663 100644 --- a/ext/fts5/test/fts5vocab.test +++ b/ext/fts5/test/fts5vocab.test @@ -21,9 +21,44 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + +if {[detail_is_none]} continue + +proc null_list_entries {iFirst nInterval L} { + for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} { + lset L $i {} + } + return $L +} + +proc null_insert {iFirst nInterval L} { + for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} { + lset L $i {} + } + return $L +} + +proc star_from_row {L} { + if {[detail_is_full]==0} { + set L [null_list_entries 2 3 $L] + } + return $L +} + +proc star_from_col {L} { + if {[detail_is_col]} { + set L [null_list_entries 3 4 $L] + } + if {[detail_is_none]} { + set L [null_list_entries 1 4 $L] + set L [null_list_entries 3 4 $L] + } + return $L +} do_execsql_test 1.1.1 { - CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1); + CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%); CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row'); PRAGMA table_info = v1; } { @@ -52,32 +87,32 @@ do_execsql_test 1.3 { do_execsql_test 1.4.1 { SELECT * FROM v1; -} {x 2 4 y 1 1 z 1 1} +} [star_from_row {x 2 4 y 1 1 z 1 1}] do_execsql_test 1.4.2 { SELECT * FROM v2; -} {x one 2 4 y one 1 1 z one 1 1} +} [star_from_col {x one 2 4 y one 1 1 z one 1 1}] do_execsql_test 1.5.1 { BEGIN; INSERT INTO t1 VALUES('a b c'); SELECT * FROM v1 WHERE term<'d'; -} {a 1 1 b 1 1 c 1 1} +} [star_from_row {a 1 1 b 1 1 c 1 1}] do_execsql_test 1.5.2 { SELECT * FROM v2 WHERE term<'d'; COMMIT; -} {a one 1 1 b one 1 1 c one 1 1} +} [star_from_col {a one 1 1 b one 1 1 c one 1 1}] do_execsql_test 1.6 { DELETE FROM t1 WHERE one = 'a b c'; SELECT * FROM v1; -} {x 2 4 y 1 1 z 1 1} +} [star_from_row {x 2 4 y 1 1 z 1 1}] #------------------------------------------------------------------------- # do_execsql_test 2.0 { - CREATE VIRTUAL TABLE tt USING fts5(a, b); + CREATE VIRTUAL TABLE tt USING fts5(a, b, detail=%DETAIL%); INSERT INTO tt VALUES('d g b f d f', 'f c e c d a'); INSERT INTO tt VALUES('f a e a a b', 'e d c f d d'); INSERT INTO tt VALUES('b c a a a b', 'f f c c b c'); @@ -90,7 +125,12 @@ do_execsql_test 2.0 { INSERT INTO tt VALUES('c c a a c f', 'd g a e b g'); } -set res_col { +set res_row [star_from_row { + a 10 20 b 9 14 c 9 20 d 9 19 + e 8 13 f 10 20 g 7 14 x 1 1 + y 1 1 +}] +set res_col [star_from_col { a a 6 11 a b 7 9 b a 6 7 b b 7 7 c a 6 12 c b 5 8 @@ -99,12 +139,7 @@ set res_col { f a 9 10 f b 7 10 g a 5 7 g b 5 7 x a 1 1 y b 1 1 -} -set res_row { - a 10 20 b 9 14 c 9 20 d 9 19 - e 8 13 f 10 20 g 7 14 x 1 1 - y 1 1 -} +}] foreach {tn tbl resname} { 1 "fts5vocab(tt, 'col')" res_col @@ -153,9 +188,9 @@ reset_db forcedelete test.db2 do_execsql_test 5.0 { ATTACH 'test.db2' AS aux; - CREATE VIRTUAL TABLE t1 USING fts5(x); - CREATE VIRTUAL TABLE temp.t1 USING fts5(x); - CREATE VIRTUAL TABLE aux.t1 USING fts5(x); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE aux.t1 USING fts5(x, detail=%DETAIL%); INSERT INTO main.t1 VALUES('a b c'); INSERT INTO main.t1 VALUES('d e f'); @@ -178,18 +213,18 @@ do_execsql_test 5.1 { CREATE VIRTUAL TABLE temp.va USING fts5vocab(aux, t1, row); } -do_execsql_test 5.2 { SELECT * FROM vm } { +do_execsql_test 5.2 { SELECT * FROM vm } [star_from_row { a 2 2 b 1 1 c 2 2 d 1 1 e 2 2 f 1 1 -} -do_execsql_test 5.3 { SELECT * FROM vt1 } { +}] +do_execsql_test 5.3 { SELECT * FROM vt1 } [star_from_row { 1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1 -} -do_execsql_test 5.4 { SELECT * FROM vt2 } { +}] +do_execsql_test 5.4 { SELECT * FROM vt2 } [star_from_row { 1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1 -} -do_execsql_test 5.5 { SELECT * FROM va } { +}] +do_execsql_test 5.5 { SELECT * FROM va } [star_from_row { m 1 1 n 2 2 o 1 1 x 2 2 y 1 1 z 2 2 -} +}] #------------------------------------------------------------------------- # @@ -218,7 +253,7 @@ do_catchsql_test 6.2 { # constraints in the WHERE clause). # do_execsql_test 7.0 { - CREATE VIRTUAL TABLE tx USING fts5(one, two); + CREATE VIRTUAL TABLE tx USING fts5(one, two, detail=%DETAIL%); INSERT INTO tx VALUES('g a ggg g a b eee', 'cc d aa ff g ee'); INSERT INTO tx VALUES('dd fff i a i jjj', 'f fff hh jj e f'); INSERT INTO tx VALUES('ggg a f f fff dd aa', 'd ggg f f j gg ddd'); @@ -276,6 +311,9 @@ foreach {term} { if {[lindex $r2 2]==0} {set r2 [list]} set resc [concat $r1 $r2] + + set resc [star_from_col $resc] + set resr [star_from_row $resr] do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr } @@ -345,5 +383,7 @@ do_execsql_test 7.3.2 { WHERE txc.term = txc_c.term AND txc.col=txc_c.col; } {57} +} + finish_test diff --git a/manifest b/manifest index efcebbdb14..d860431936 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\sscript\sproblem\scaused\sby\sa\schange\sin\sconstraint\shandling\swithin\sthe\score. -D 2016-01-06T19:43:26.257 +C Update\sthe\sfts5vocab\smodule\sto\swork\swith\sdetail=col\sand\sdetail=none\stables. +D 2016-01-07T20:07:41.638 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -111,7 +111,7 @@ F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070 F ext/fts5/fts5_tokenize.c 504984ac6993323247221eebe3cd55bead01b5f8 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 -F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2 +F ext/fts5/fts5_vocab.c da64ecbd217625980a1721fbd588a1e4118a51b6 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl c9169fe40bf751e1b311271df31aec15732b26c0 @@ -179,7 +179,7 @@ F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e -F ext/fts5/test/fts5vocab.test c88a5554d0409494da95ba647bbdb4879b2624b0 +F ext/fts5/test/fts5vocab.test f4ff3ba4036a83678f2a8ba8b89eb303bebc122e F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84 F ext/fts5/tool/loadfts5.tcl 4cc2d6af43b58d4fac05bc4fdabd0e5862c3b2c1 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 @@ -1409,7 +1409,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e7dcd013516723941c2fb78ecdc96806d49d310b -R dce0c272fbae692a97296955b7b2c775 +P 625695b3d7d4eae7536b5f761b96cd7519ac6ada +R 276342437a8f737c3f32b7f1a4e61d27 U dan -Z 318fda862dd5c3483c784e4e425f0506 +Z 8c87f6adf59add172dd12da497a3ce1f diff --git a/manifest.uuid b/manifest.uuid index 298843ae92..46d1607a71 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -625695b3d7d4eae7536b5f761b96cd7519ac6ada \ No newline at end of file +eedd095dc1c81ce45df00093ba237dd7b3cdff3d \ No newline at end of file