*pbPresent = 0;
if( p ){
int i;
- int hash;
+ int hash = 13;
Fts5TermsetEntry *pEntry;
- /* Calculate a hash value for this term */
- hash = 104 + iIdx;
- for(i=0; i<nTerm; i++){
- hash += (hash << 3) + (int)pTerm[i];
+ /* Calculate a hash value for this term. This is the same hash checksum
+ ** used by the fts5_hash.c module. This is not important for correct
+ ** operation of the module, but is necessary to ensure that some tests
+ ** designed to produce hash table collisions really do work. */
+ for(i=nTerm-1; i>=0; i--){
+ hash = (hash << 3) ^ hash ^ pTerm[i];
}
+ hash = (hash << 3) ^ hash ^ iIdx;
hash = hash % ArraySize(p->apHash);
for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
p++;
}
- if( rc==SQLITE_OK && (nPre<=0 || nPre>=1000) ){
+ if( nPre<=0 || nPre>=1000 ){
*pzErr = sqlite3_mprintf("prefix length out of range (max 999)");
rc = SQLITE_ERROR;
break;
}
return sqlite3Fts5Tokenize(pConfig,
- FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
+ FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
);
}
}
static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
- if( pNode ){
- pNode->iRowid = iRowid;
- pNode->bEof = 0;
- switch( pNode->eType ){
- case FTS5_TERM:
- case FTS5_STRING:
- return (pNode->pNear->apPhrase[0]->poslist.n>0);
+ pNode->iRowid = iRowid;
+ pNode->bEof = 0;
+ switch( pNode->eType ){
+ case FTS5_TERM:
+ case FTS5_STRING:
+ return (pNode->pNear->apPhrase[0]->poslist.n>0);
- case FTS5_AND: {
- int i;
- for(i=0; i<pNode->nChild; i++){
- if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
- fts5ExprClearPoslists(pNode);
- return 0;
- }
+ case FTS5_AND: {
+ int i;
+ for(i=0; i<pNode->nChild; i++){
+ if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
+ fts5ExprClearPoslists(pNode);
+ return 0;
}
- break;
}
+ break;
+ }
- case FTS5_OR: {
- int i;
- int bRet = 0;
- for(i=0; i<pNode->nChild; i++){
- if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
- bRet = 1;
- }
- }
- if( bRet==0 ){
- fts5ExprClearPoslists(pNode);
+ case FTS5_OR: {
+ int i;
+ int bRet = 0;
+ for(i=0; i<pNode->nChild; i++){
+ if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
+ bRet = 1;
}
- return bRet;
}
+ return bRet;
+ }
- default: {
- assert( pNode->eType==FTS5_NOT );
- if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
- || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
+ default: {
+ assert( pNode->eType==FTS5_NOT );
+ if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
+ || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
){
- fts5ExprClearPoslists(pNode);
- return 0;
- }
- break;
+ fts5ExprClearPoslists(pNode);
+ return 0;
}
+ break;
}
}
return 1;
}
-static sqlite3_stmt *fts5PrepareStatement(
- int *pRc,
+static int fts5PrepareStatement(
+ sqlite3_stmt **ppStmt,
Fts5Config *pConfig,
const char *zFmt,
...
){
sqlite3_stmt *pRet = 0;
+ int rc;
+ char *zSql;
va_list ap;
- va_start(ap, zFmt);
- if( *pRc==SQLITE_OK ){
- int rc;
- char *zSql = sqlite3_vmprintf(zFmt, ap);
- if( zSql==0 ){
- rc = SQLITE_NOMEM;
- }else{
- rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
- if( rc!=SQLITE_OK ){
- *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
- }
- sqlite3_free(zSql);
+ va_start(ap, zFmt);
+ zSql = sqlite3_vmprintf(zFmt, ap);
+ if( zSql==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
+ if( rc!=SQLITE_OK ){
+ *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
}
- *pRc = rc;
+ sqlite3_free(zSql);
}
va_end(ap);
- return pRet;
+ *ppStmt = pRet;
+ return rc;
}
static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
Fts5Sorter *pSorter;
int nPhrase;
int nByte;
- int rc = SQLITE_OK;
+ int rc;
const char *zRank = pCsr->zRank;
const char *zRankArgs = pCsr->zRankArgs;
** table, saving it creates a circular reference.
**
** If SQLite a built-in statement cache, this wouldn't be a problem. */
- pSorter->pStmt = fts5PrepareStatement(&rc, pConfig,
+ rc = fts5PrepareStatement(&pSorter->pStmt, pConfig,
"SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s",
pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
(zRankArgs ? ", " : ""),
Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
+ Fts5Sorter *pSorter = pCsr->pSorter;
int n;
- rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
+ if( pSorter ){
+ int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
+ n = pSorter->aIdx[iPhrase] - i1;
+ pIter->a = &pSorter->aPoslist[i1];
+ }else{
+ rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
+ }
if( rc==SQLITE_OK ){
pIter->b = &pIter->a[n];
*piCol = 0;
zColvar = Tcl_GetString(objv[3]);
zOffvar = Tcl_GetString(objv[4]);
- for(p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
- iCol>=0;
- p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff)
- ){
+ rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
+ if( rc!=SQLITE_OK ){
+ Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+ return TCL_ERROR;
+ }
+ for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){
Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0);
rc = Tcl_EvalObjEx(interp, pScript, 0);
if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
zColvar = Tcl_GetString(objv[3]);
- for(p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
- iCol>=0;
- p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)
- ){
+ rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
+ if( rc!=SQLITE_OK ){
+ Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+ return TCL_ERROR;
+ }
+ for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
rc = Tcl_EvalObjEx(interp, pScript, 0);
if( rc==TCL_CONTINUE ) rc = TCL_OK;
set res
}
+proc fts5_collist_data {expr tbl {order ASC} {aDictVar ""}} {
+ set res [list]
+
+ if {$aDictVar!=""} {
+ upvar $aDictVar aDict
+ set dict aDict
+ } else {
+ set dict ""
+ }
+
+ foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
+ lappend res $rowid $collist
+ }
+ set res
+}
+
#-------------------------------------------------------------------------
#
list
}
+
+#-------------------------------------------------------------------------
+# Code for a simple Tcl tokenizer that supports synonyms at query time.
+#
+proc tclnum_tokenize {mode tflags text} {
+ foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+ sqlite3_fts5_token $w $iStart $iEnd
+ if {$tflags == $mode && [info exists ::tclnum_syn($w)]} {
+ foreach s $::tclnum_syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd }
+ }
+ }
+}
+
+proc tclnum_create {args} {
+ set mode query
+ if {[llength $args]} {
+ set mode [lindex $args 0]
+ }
+ if {$mode != "query" && $mode != "document"} { error "bad mode: $mode" }
+ return [list tclnum_tokenize $mode]
+}
+
+proc fts5_tclnum_register {db} {
+ foreach SYNDICT {
+ {zero 0}
+ {one 1 i}
+ {two 2 ii}
+ {three 3 iii}
+ {four 4 iv}
+ {five 5 v}
+ {six 6 vi}
+ {seven 7 vii}
+ {eight 8 viii}
+ {nine 9 ix}
+
+ {a1 a2 a3 a4 a5 a6 a7 a8 a9}
+ {b1 b2 b3 b4 b5 b6 b7 b8 b9}
+ {c1 c2 c3 c4 c5 c6 c7 c8 c9}
+ } {
+ foreach s $SYNDICT {
+ set o [list]
+ foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
+ set ::tclnum_syn($s) $o
+ }
+ }
+ sqlite3_fts5_create_tokenizer db tclnum tclnum_create
+}
+#
+# End of tokenizer code.
+#-------------------------------------------------------------------------
+
3 {prefix='$'}
4 {prefix='1,2,'}
5 {prefix=',1'}
+ 6 {prefix='1,2,3...'}
+ 7 {prefix='1,2,3xyz'}
} {
set res [list 1 {malformed prefix=... directive}]
do_catchsql_test 2.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
# 9.1.* 'pgsz' options.
# 9.2.* 'automerge' options.
# 9.3.* 'crisismerge' options.
+# 9.4.* a non-existant option.
+# 9.5.* 'hashsize' options.
#
do_execsql_test 9.0 {
CREATE VIRTUAL TABLE abc USING fts5(a, b);
INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
} {1 {SQL logic error or missing database}}
+do_catchsql_test 9.5.1 {
+ INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
+} {1 {SQL logic error or missing database}}
+do_catchsql_test 9.5.2 {
+ INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
+} {1 {SQL logic error or missing database}}
+do_catchsql_test 9.5.3 {
+ INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
+} {0 {}}
+
#-------------------------------------------------------------------------
# Too many prefix indexes. Maximum allowed is 31.
#
do_catchsql_test 10.$tn $sql {1 {too many prefix indexes (max 31)}}
}
+#-------------------------------------------------------------------------
+# errors in the detail= option.
+#
+foreach {tn opt} {
+ 1 {detail=x}
+ 2 {detail='x'}
+ 3 {detail='$'}
+ 4 {detail='1,2,'}
+ 5 {detail=',1'}
+ 6 {detail=''}
+} {
+ set res [list 1 {malformed detail=... directive}]
+ do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
+}
+
finish_test
faultsim_test_result [list 0 {}]
}
-#-------------------------------------------------------------------------
-# An OOM within an "ORDER BY rank" query.
-#
-db func rnddoc fts5_rnddoc
-do_execsql_test 2.0 {
- CREATE VIRTUAL TABLE xx USING fts5(x);
- INSERT INTO xx VALUES ('abc ' || rnddoc(10));
- INSERT INTO xx VALUES ('abc abc' || rnddoc(9));
- INSERT INTO xx VALUES ('abc abc abc' || rnddoc(8));
-} {}
-faultsim_save_and_close
-
-do_faultsim_test 2 -faults oom-* -prep {
- faultsim_restore_and_reopen
- execsql { SELECT * FROM xx }
-} -body {
- execsql { SELECT rowid FROM xx WHERE xx MATCH 'abc' ORDER BY rank }
-} -test {
- faultsim_test_result [list 0 {3 2 1}]
-}
-
#-------------------------------------------------------------------------
# An OOM while "reseeking" an FTS cursor.
#
--- /dev/null
+# 2015 September 3
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5fault9
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+ finish_test
+ return
+}
+
+foreach_detail_mode $testprefix {
+
+fts5_aux_test_functions db
+
+if 1 {
+
+do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
+ INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+ WITH seq(s) AS ( SELECT 1 UNION ALL SELECT s+1 FROM seq WHERE s<50)
+ INSERT INTO t1 SELECT 'x x x y y y', 'a b c d e f' FROM seq;
+}
+
+do_faultsim_test 1 -faults oom-* -body {
+ execsql { SELECT count(*) FROM t1('x AND y') }
+} -test {
+ faultsim_test_result {0 50}
+}
+
+do_execsql_test 2.0 {
+ CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%);
+ INSERT INTO t2(t2, rank) VALUES('pgsz', 32);
+ INSERT INTO t2 VALUES('abc cba', 'cba abc');
+ INSERT INTO t2 VALUES('abc cba', 'cba abc');
+ INSERT INTO t2 VALUES('abc cba', 'cba abc');
+
+ INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
+ INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
+ INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
+}
+
+do_faultsim_test 2 -faults oom-* -body {
+ execsql { SELECT count(*) FROM t2('a* AND c*') }
+} -test {
+ faultsim_test_result {0 6}
+}
+
+
+do_execsql_test 3.0 {
+ CREATE VIRTUAL TABLE t3 USING fts5(a, detail=%DETAIL%);
+ INSERT INTO t3 VALUES('a x x a x a a a');
+ INSERT INTO t3 VALUES('x a a x a x x x');
+}
+
+do_faultsim_test 3.1 -faults oom-* -body {
+ execsql { SELECT highlight(t3, 0, '[', ']') FROM t3('a') }
+} -test {
+ faultsim_test_result {0 {{[a] x x [a] x [a] [a] [a]} {x [a] [a] x [a] x x x}}}
+}
+
+do_faultsim_test 3.2 -faults oom-t* -body {
+ execsql { SELECT fts5_test_poslist2(t3) FROM t3('x') }
+} -test {
+ faultsim_test_result \
+ {0 {{0.0.1 0.0.2 0.0.4} {0.0.0 0.0.3 0.0.5 0.0.6 0.0.7}}} \
+ {1 SQLITE_NOMEM}
+}
+
+#-------------------------------------------------------------------------
+# Test OOM injection with the xPhraseFirstColumn() API and a tokenizer
+# uses query synonyms.
+#
+fts5_tclnum_register db
+do_execsql_test 4.0 {
+ CREATE VIRTUAL TABLE t4 USING fts5(x, y, z, detail=%DETAIL%, tokenize=tclnum);
+ INSERT INTO t4 VALUES('one two three', '1 2 3', 'i ii iii');
+ INSERT INTO t4 VALUES('1 2 3', 'i ii iii', 'one two three');
+ INSERT INTO t4 VALUES('i ii iii', 'one two three', 'i ii iii');
+
+ INSERT INTO t4 VALUES('a1 a2 a3', 'a4 a5 a6', 'a7 a8 a9');
+ INSERT INTO t4 VALUES('b1 b2 b3', 'b4 b5 b6', 'b7 b8 b9');
+ INSERT INTO t4 VALUES('c1 c2 c3', 'c4 c5 c6', 'c7 c8 c9');
+}
+
+do_faultsim_test 4.1 -faults oom-t* -body {
+ execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') }
+} -test {
+ faultsim_test_result \
+ {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM}
+}
+
+do_faultsim_test 4.2 -faults oom-t* -body {
+ execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') }
+} -test {
+ faultsim_test_result \
+ {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM}
+}
+
+}
+
+
+#-------------------------------------------------------------------------
+# An OOM within an "ORDER BY rank" query.
+#
+db func rnddoc fts5_rnddoc
+do_execsql_test 5.0 {
+ CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=%DETAIL%);
+ INSERT INTO xx VALUES ('def', 'abc ' || rnddoc(10));
+ INSERT INTO xx VALUES ('def', 'abc abc' || rnddoc(9));
+ INSERT INTO xx VALUES ('def', 'abc abc abc' || rnddoc(8));
+} {}
+faultsim_save_and_close
+
+do_faultsim_test 5 -faults oom-* -prep {
+ faultsim_restore_and_reopen
+ execsql { SELECT * FROM xx }
+} -body {
+ execsql { SELECT rowid FROM xx('abc AND def') ORDER BY rank }
+} -test {
+ faultsim_test_result [list 0 {3 2 1}]
+}
+
+} ;# foreach_detail_mode...
+
+finish_test
+
return $doc
}
+foreach_detail_mode $testprefix {
+
set vocab [build_vocab1]
db func r random_doc
do_execsql_test 1.0 {
- CREATE VIRTUAL TABLE eee USING fts5(e, ee);
+ CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%);
BEGIN;
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii;
}
} {}
-do_test 1.2 {
+do_test 1.3 {
db eval { SELECT term, doc FROM vocab } {
set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}]
if {$nRow != $doc} {
set {} {}
} {}
-do_execsql_test 1.3 {
+do_execsql_test 1.4 {
COMMIT;
INSERT INTO eee(eee) VALUES('integrity-check');
}
+} ;# foreach_detail_mode
+
finish_test
return
}
-foreach S {
- {zero 0}
- {one 1 i}
- {two 2 ii}
- {three 3 iii}
- {four 4 iv}
- {five 5 v}
- {six 6 vi}
- {seven 7 vii}
- {eight 8 viii}
- {nine 9 ix}
-} {
- foreach s $S {
- set o [list]
- foreach x $S {if {$x!=$s} {lappend o $x}}
- set ::syn($s) $o
- }
-}
-
-proc tcl_tokenize {tflags text} {
- foreach {w iStart iEnd} [fts5_tokenize_split $text] {
- sqlite3_fts5_token $w $iStart $iEnd
- }
-}
+proc tcl_create {args} { return "tcl_tokenize" }
-proc tcl_create {args} {
- return "tcl_tokenize"
-}
-
-sqlite3_fts5_create_tokenizer db tcl tcl_create
+foreach_detail_mode $testprefix {
#-------------------------------------------------------------------------
# Warm body test for the code in fts5_tcl.c.
#
+fts5_tclnum_register db
do_execsql_test 1.0 {
- CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
+ CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = "tclnum document", detail=%DETAIL%);
INSERT INTO ft VALUES('abc def ghi');
INSERT INTO ft VALUES('jkl mno pqr');
SELECT rowid, x FROM ft WHERE ft MATCH 'def';
# Test a tokenizer that supports synonyms by adding extra entries to the
# FTS index.
#
-
-proc tcl_tokenize {tflags text} {
- foreach {w iStart iEnd} [fts5_tokenize_split $text] {
- sqlite3_fts5_token $w $iStart $iEnd
- if {$tflags=="document" && [info exists ::syn($w)]} {
- foreach s $::syn($w) {
- sqlite3_fts5_token -colo $s $iStart $iEnd
- }
- }
- }
-}
reset_db
-sqlite3_fts5_create_tokenizer db tcl tcl_create
+fts5_tclnum_register db
do_execsql_test 2.0 {
- CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
+ CREATE VIRTUAL TABLE ft USING fts5(
+ x, tokenize = "tclnum document", detail=%DETAIL%
+ );
INSERT INTO ft VALUES('one two three');
INSERT INTO ft VALUES('four five six');
INSERT INTO ft VALUES('eight nine ten');
4 "1*" {1}
5 "1 + 2" {1}
} {
+ if {![fts5_expr_ok $expr ft]} continue
do_execsql_test 2.1.$tn {
SELECT rowid FROM ft WHERE ft MATCH $expr
} $res
# Check that expressions with synonyms can be parsed and executed.
#
reset_db
-sqlite3_fts5_create_tokenizer db tcl tcl_create
-proc tcl_tokenize {tflags text} {
- foreach {w iStart iEnd} [fts5_tokenize_split $text] {
- sqlite3_fts5_token $w $iStart $iEnd
- if {$tflags=="query" && [info exists ::syn($w)]} {
- foreach s $::syn($w) {
- sqlite3_fts5_token -colo $s $iStart $iEnd
- }
- }
- }
-}
+fts5_tclnum_register db
foreach {tn expr res} {
1 {abc} {"abc"}
3 {3} {"3"|"iii"|"three"}
4 {3*} {"3"|"iii"|"three" *}
} {
- do_execsql_test 4.1.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res]
+ do_execsql_test 4.1.$tn {
+ SELECT fts5_expr($expr, 'tokenize=tclnum')
+ } [list $res]
}
do_execsql_test 4.2.1 {
- CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tcl);
+ CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tclnum, detail=%DETAIL%);
INSERT INTO xx VALUES('one two');
INSERT INTO xx VALUES('three four');
}
do_test 5.0 {
execsql {
- CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tcl)
+ CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tclnum, detail=%DETAIL%)
}
foreach {rowid a b} {
1 {four v 4 i three} {1 3 five five 4 one}
5 {three i v i four 4 1} {ii [five five five] iii}
}
} {
+ if {![fts5_expr_ok $q t1]} continue
do_execsql_test 5.1.$tn {
SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']')
FROM t1 WHERE t1 MATCH $q
} $res
}
-
#-------------------------------------------------------------------------
# Test terms with more than 4 synonyms.
#
}
do_execsql_test 6.0.1 {
- CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl);
+ CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl, detail=%DETAIL%);
INSERT INTO t1 VALUES('yy xx qq');
INSERT INTO t1 VALUES('yy xx xx');
}
-do_execsql_test 6.0.2 {
- SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)';
-} {{yy xx qq}}
+if {[fts5_expr_ok "NEAR(y q)" t1]} {
+ do_execsql_test 6.0.2 {
+ SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)';
+ } {{yy xx qq}}
+}
do_test 6.0.3 {
execsql {
- CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl)
+ CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl, detail=%DETAIL%)
}
foreach {rowid a b} {
1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa}
2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]}
}
} {
+ if {![fts5_expr_ok $q t2]} continue
+
do_execsql_test 6.1.$tn.asc {
SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']')
FROM t2 WHERE t2 MATCH $q
}
do_execsql_test 7.0.1 {
- CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl);
+ CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl, detail=%DETAIL%);
INSERT INTO t1 VALUES('0 2 3', '4 5 6 7');
INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0';
}
do_execsql_test 7.1.1 {
- CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl);
+ CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl, detail=%DETAIL%);
INSERT INTO t2 VALUES('0 2 3', '4 5 6 7');
INSERT INTO t2 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
SELECT fts5_test_columnsize(t2) FROM t2 WHERE t2 MATCH '000 AND 00 AND 0';
INSERT INTO t2(t2) VALUES('integrity-check');
}
+} ;# foreach_detail_mode
+
finish_test
return
}
-#-------------------------------------------------------------------------
-# Code for a simple Tcl tokenizer that supports synonyms at query time.
-#
-foreach SYNDICT {
- {zero 0}
- {one 1 i}
- {two 2 ii}
- {three 3 iii}
- {four 4 iv}
- {five 5 v}
- {six 6 vi}
- {seven 7 vii}
- {eight 8 viii}
- {nine 9 ix}
-} {
- foreach s $SYNDICT {
- set o [list]
- foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
- set ::syn($s) $o
- }
-}
-
-proc tcl_tokenize {tflags text} {
- foreach {w iStart iEnd} [fts5_tokenize_split $text] {
- sqlite3_fts5_token $w $iStart $iEnd
- if {$tflags == "query"} {
- foreach s $::syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd }
- }
- }
-}
-
-proc tcl_create {args} {
- return "tcl_tokenize"
-}
-
-#
-# End of tokenizer code.
-#-------------------------------------------------------------------------
-
+foreach tok {query document} {
foreach_detail_mode $testprefix {
-sqlite3_fts5_create_tokenizer db tcl tcl_create
+fts5_tclnum_register db
fts5_aux_test_functions db
-do_execsql_test 1.0 {
- CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%);
+proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] }
+sqlite3_fts5_create_function db fts5_rowid fts5_rowid
+
+do_execsql_test 1.$tok.0.1 "
+ CREATE VIRTUAL TABLE ss USING fts5(a, b,
+ tokenize='tclnum $tok', detail=%DETAIL%);
+ INSERT INTO ss(ss, rank) VALUES('rank', 'fts5_rowid()');
+"
+
+do_execsql_test 1.$tok.0.2 {
INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i');
INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi');
INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3');
4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)"
} {
if {[fts5_expr_ok $expr ss]==0} {
- do_test 1.$tn.OMITTED { list } [list]
+ do_test 1.$tok.$tn.OMITTED { list } [list]
continue
}
- set res [fts5_query_data $expr ss ASC ::syn]
- breakpoint
- do_execsql_test 1.$tn.[llength $res].asc {
+ set res [fts5_query_data $expr ss ASC ::tclnum_syn]
+ do_execsql_test 1.$tok.$tn.[llength $res].asc.1 {
SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
} $res
+
+ do_execsql_test 1.$tok.$tn.[llength $res].asc.2 {
+ SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
+ ORDER BY rank ASC
+ } $res
}
+}
}
finish_test
-C Use\ssqlite3VdbeAddOp4()\srather\sthan\sa\sseparate\ssqlite3VdbeChangeP4()\scall,\sfor\na\sslightly\ssmaller\sand\sfaster\sbinary.
-D 2016-01-16T15:12:35.739
+C Add\sfurther\stests\sfor\sfts5.\sFix\ssome\sproblems\swith\sdetail=col\smode\sand\sauxiliary\sfunctions.
+D 2016-01-16T18:58:51.767
F Makefile.in a476545d0c8626224d0bacac85c6e2967474af81
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc 01e855f958932d0d3ed62ec675fc63e2cef61fcb
F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
F ext/fts5/fts5Int.h 313e3276ac9e0245ee722ef803253857a68722b9
F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e
-F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1
-F ext/fts5/fts5_config.c 9c243d04ac0ca997d2d2e2252891f2a10fbd7217
-F ext/fts5/fts5_expr.c 510db45967ca359f64f2ba2c707ab57d740cad56
+F ext/fts5/fts5_buffer.c ba59964c95f760bd5ff1dbe173b85f197a13864e
+F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238
+F ext/fts5/fts5_expr.c 3ba4c9588ebb2e4f852d807869af7130b6362e5a
F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
F ext/fts5/fts5_index.c cd036089b22b0340fccef88a3ad62ac1016c7cbb
-F ext/fts5/fts5_main.c 1d116f5c44c6f06ec282d33dd6d041c8131b3d6a
+F ext/fts5/fts5_main.c 833db0a3df10ab26e0221a9baa40cf871c450df3
F ext/fts5/fts5_storage.c fb2eaec3aa954b680d43096dc539f8270bd6390e
-F ext/fts5/fts5_tcl.c bcacc05dec0446e7b1a44d5d906057e677bd7ea4
+F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070
F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be
F ext/fts5/fts5_tokenize.c 504984ac6993323247221eebe3cd55bead01b5f8
F ext/fts5/fts5_vocab.c ee6df1a3be103414d7b7af833ae1885c7b83a9d0
F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl 393882afb225a21edf033043bbf936951e9198c1
+F ext/fts5/test/fts5_common.tcl 75b0af04898c18df289620dd4e7442d881e3ccc1
F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084
F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
F ext/fts5/test/fts5ac.test d5073ca7bd2d9fe8aab0c82c6c75a7e4b0d70ced
F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e
F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb
F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07
-F ext/fts5/test/fts5config.test 42c1336cc6ed33d7e9c4a05dbce81721b765e7d0
+F ext/fts5/test/fts5config.test 83941309b94d002ed6f55d9cd814e0353c9ae013
F ext/fts5/test/fts5conflict.test 26f4e46c4d31e16221794832a990dc4e30e18de5
F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
F ext/fts5/test/fts5fault1.test 4b39c47ca3544615daa8a2f733b911fa08022c77
F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341
F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
-F ext/fts5/test/fts5fault4.test 4864f2b5c2c083440dbe85aff60897bc1aa04603
+F ext/fts5/test/fts5fault4.test 532b6dacb963016cbf7003196bd87fb366540277
F ext/fts5/test/fts5fault5.test 10c13a783de3f42a21e3e53e123b62ed0c3a1618
F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda
F ext/fts5/test/fts5fault7.test 01be274bfc8d9bf22451a3bf5892e9399d044f1b
F ext/fts5/test/fts5fault8.test f2d8a2b673a5f72ca1fa0e85bdbfb2041ffd347d
+F ext/fts5/test/fts5fault9.test 964a3ee82b95053bd45d809fe2b7acf4a8c90ca9
F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
-F ext/fts5/test/fts5hash.test 7cf4607b8657c383f0b520668a99971e95d8b139
+F ext/fts5/test/fts5hash.test 00668f6fa9b9bffbd7c1be29f408aa2bdade0451
F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65
F ext/fts5/test/fts5matchinfo.test 86569026d20f1ed748236587ce798de8a96615f1
F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367
F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821
F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9
F ext/fts5/test/fts5simple2.test 7b51f8d411e9a77fa4519fb09ba5a3afda75c94d
-F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
-F ext/fts5/test/fts5synonym2.test d2d9099d9d105b55ea03fd52d61ae2847d534129
+F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48
+F ext/fts5/test/fts5synonym2.test eadb00c73ef0653258873e756b7e9102e0687539
F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7
F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 653ea15ad8e23b333e234eb5dde7b80134db2baf
-R 2155f55389b348e9a35fea6f93c01bbe
-U drh
-Z 32d5bf122f6c10496965338bc29a15a2
+P a4258cd4613c55acacb5c7b61faa3de7eb0759d2
+R 7d01724b879e1ef98656daadba5b9cfb
+U dan
+Z d745955e2f048d17831d9a126adffb23
-a4258cd4613c55acacb5c7b61faa3de7eb0759d2
\ No newline at end of file
+de77d6026e8035c505a704e7b8cfe5af6579d35f
\ No newline at end of file
projects / thirdparty / sqlite.git / commitdiff
