memset(pTerm, 0, sizeof(Fts5ExprTerm));
pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
pTerm->nFullTerm = pTerm->nQueryTerm = nToken;
- if( pCtx->pConfig->bTokendata ) pTerm->nQueryTerm = strlen(pTerm->pTerm);
+ if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){
+ pTerm->nQueryTerm = strlen(pTerm->pTerm);
+ }
}
}
int rc = sqlite3Fts5PoslistWriterAppend(
&pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
);
- if( rc==SQLITE_OK && pExpr->pConfig->bTokendata ){
+ if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){
int iCol = p->iOff>>32;
int iTokOff = p->iOff & 0x7FFFFFFF;
rc = sqlite3Fts5IndexIterWriteTokendata(
}
}
-/*
-** The iterator passed as the first argument must be a tokendata=1 iterator
-** (pIter->pTokenDataIter!=0). This function is used to access the token
-** instance located at offset iOff of column iCol of row iRowid. It is
-** returned via output pointers *ppOut and *pnOut. This is used by the
-** xInstToken() API.
-*/
-static int fts5TokendataIterToken(
- Fts5Iter *pIter,
- i64 iRowid,
- int iCol, int iOff,
- const char **ppOut, int *pnOut
-){
- Fts5TokenDataIter *pT = pIter->pTokenDataIter;
- Fts5TokenDataMap *aMap = pT->aMap;
- i64 iPos = (((i64)iCol)<<32) + iOff;
-
- int i1 = 0;
- int i2 = pT->nMap;
- int iTest = 0;
-
- while( i2>i1 ){
- iTest = (i1 + i2) / 2;
-
- if( aMap[iTest].iRowid<iRowid ){
- i1 = iTest+1;
- }else if( aMap[iTest].iRowid>iRowid ){
- i2 = iTest;
- }else{
- if( aMap[iTest].iPos<iPos ){
- if( aMap[iTest].iPos<0 ){
- break;
- }
- i1 = iTest+1;
- }else if( aMap[iTest].iPos>iPos ){
- i2 = iTest;
- }else{
- break;
- }
- }
- }
-
- if( i2>i1 ){
- Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter];
- *ppOut = (const char*)pMap->aSeg[0].term.p+1;
- *pnOut = pMap->aSeg[0].term.n-1;
- }
-
- return SQLITE_OK;
-}
-
/*
** Append a mapping to the token-map belonging to object pT.
*/
memcpy(pNewIter, pPrevIter, sizeof(Fts5SegIter));
memset(pPrevIter, 0, sizeof(Fts5SegIter));
bDone = 1;
- }else if( pPrevIter->pLeaf
- && pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf
- ){
+ }else if( pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf ){
fts5SegIterNextInit(p,(const char*)bSeek.p,bSeek.n-1,pSeg,pNewIter);
bDone = 1;
}
if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){
int iIdx = 0; /* Index to search */
int iPrefixIdx = 0; /* +1 prefix index */
- int bTokendata = (flags&FTS5INDEX_QUERY_NOTOKENDATA)?0:pConfig->bTokendata;
+ int bTokendata = pConfig->bTokendata;
if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken);
/* Figure out which index to search and set iIdx accordingly. If this
** for internal sanity checking by the integrity-check in debug
** mode only. */
#ifdef SQLITE_DEBUG
+ if( flags & FTS5INDEX_QUERY_NOTOKENDATA ) bTokendata = 0;
if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){
assert( flags & FTS5INDEX_QUERY_PREFIX );
iIdx = 1+pConfig->nPrefix;
/*
** This is used by xInstToken() to access the token at offset iOff, column
** iCol of row iRowid. The token is returned via output variables *ppOut
-** and *pnOut.
+** and *pnOut. The iterator passed as the first argument must be a tokendata=1
+** iterator (pIter->pTokenDataIter!=0).
*/
int sqlite3Fts5IterToken(
Fts5IndexIter *pIndexIter,
const char **ppOut, int *pnOut
){
Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
+ Fts5TokenDataIter *pT = pIter->pTokenDataIter;
+ Fts5TokenDataMap *aMap = pT->aMap;
+ i64 iPos = (((i64)iCol)<<32) + iOff;
- if( pIter->pTokenDataIter ){
- return fts5TokendataIterToken(pIter, iRowid, iCol, iOff, ppOut, pnOut);
+ int i1 = 0;
+ int i2 = pT->nMap;
+ int iTest = 0;
+
+ while( i2>i1 ){
+ iTest = (i1 + i2) / 2;
+
+ if( aMap[iTest].iRowid<iRowid ){
+ i1 = iTest+1;
+ }else if( aMap[iTest].iRowid>iRowid ){
+ i2 = iTest;
+ }else{
+ if( aMap[iTest].iPos<iPos ){
+ if( aMap[iTest].iPos<0 ){
+ break;
+ }
+ i1 = iTest+1;
+ }else if( aMap[iTest].iPos>iPos ){
+ i2 = iTest;
+ }else{
+ break;
+ }
+ }
+ }
+
+ if( i2>i1 ){
+ Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter];
+ *ppOut = (const char*)pMap->aSeg[0].term.p+1;
+ *pnOut = pMap->aSeg[0].term.n-1;
}
return SQLITE_OK;
Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
Fts5TokenDataIter *pT = pIter->pTokenDataIter;
Fts5Index *p = pIter->pIndex;
+ int ii;
assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL );
- if( pT ){
- int ii;
- for(ii=0; ii<pT->nIter; ii++){
- Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term;
- if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break;
- }
- if( ii<pT->nIter ){
- fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff);
- }
+ assert( pIter->pTokenDataIter );
+
+ for(ii=0; ii<pT->nIter; ii++){
+ Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term;
+ if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break;
+ }
+ if( ii<pT->nIter ){
+ fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff);
}
return fts5IndexReturn(p);
}
--- /dev/null
+# 2010 June 15
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultG
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+ finish_test
+ return
+}
+
+set ::testprefix fts5faultH
+
+sqlite3_fts5_register_origintext db
+
+do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(
+ x, tokenize="origintext unicode61", tokendata=1
+ );
+
+ BEGIN;
+ INSERT INTO t1 VALUES('oNe tWo thRee');
+ INSERT INTO t1 VALUES('One Two Three');
+ INSERT INTO t1 VALUES('onE twO threE');
+ COMMIT;
+ BEGIN;
+ INSERT INTO t1 VALUES('one two three');
+ INSERT INTO t1 VALUES('one two three');
+ INSERT INTO t1 VALUES('one two three');
+ COMMIT;
+}
+
+do_faultsim_test 1 -faults oom* -prep {
+} -body {
+ execsql {
+ SELECT rowid FROM t1('three');
+ }
+} -test {
+ faultsim_integrity_check
+ faultsim_test_result {0 {1 2 3 4 5 6}}
+}
+
+
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 2.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(
+ x, tokenize="origintext unicode61", tokendata=1
+ );
+ INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+
+ BEGIN;
+ INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
+ INSERT INTO t1(rowid, x) VALUES(12, 'bbb bbb bbb');
+ INSERT INTO t1(rowid, x) VALUES(13, 'bbb bbb bbb');
+ INSERT INTO t1(rowid, x) VALUES(14, 'bbb BBB bbb');
+ INSERT INTO t1(rowid, x) VALUES(15, 'bbb bbb bbb');
+ INSERT INTO t1(rowid, x) VALUES(16, 'bbb bbb bbb');
+ INSERT INTO t1(rowid, x) VALUES(17, 'bbb bbb bbb');
+ INSERT INTO t1(rowid, x) VALUES(18, 'bbb bbb bbb');
+ INSERT INTO t1(rowid, x) VALUES(19, 'bbb bbb bbb');
+ INSERT INTO t1(rowid, x) VALUES(20, 'bbb bbb bbb');
+ INSERT INTO t1(rowid, x) VALUES(21, 'bbb bbb bbb');
+ INSERT INTO t1(rowid, x) VALUES(22, 'bbb bbb bbb');
+ INSERT INTO t1(rowid, x) VALUES(23, 'bbb bbb bbb');
+ INSERT INTO t1(rowid, x) VALUES(24, 'aaa bbb BBB');
+ COMMIT;
+}
+
+do_faultsim_test 2 -faults oom* -prep {
+} -body {
+ execsql {
+ SELECT rowid FROM t1('BBB AND AAA');
+ }
+} -test {
+ faultsim_integrity_check
+ faultsim_test_result {0 {10 24}}
+}
+
+
+
+finish_test
do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9}
do_execsql_test 1.14 { SELECT rowid FROM ft('hello') ORDER BY rank; } {1 2 3}
+#------------------------------------------------------------------------
+reset_db
+sqlite3_fts5_register_origintext db
+proc tokens {cmd} {
+ set ret [list]
+ for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+ set txt [$cmd xInstToken $iTok 0]
+ set txt [string map [list "\0" "."] $txt]
+ lappend ret $txt
+ }
+ set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+do_execsql_test 2.0 {
+ CREATE VIRTUAL TABLE x1 USING fts5(
+ v, tokenize="origintext unicode61", tokendata=1, detail=none
+ );
+
+ INSERT INTO x1 VALUES('xxx Xxx XXX yyy YYY yyy');
+ INSERT INTO x1 VALUES('xxx yyy xxx yyy yyy yyy');
+}
+
+do_execsql_test 2.1 {
+ SELECT tokens(x1) FROM x1('xxx');
+} {
+ {xxx xxx.Xxx xxx.XXX} {xxx xxx}
+}
+
+do_execsql_test 2.2 {
+ UPDATE x1_content SET c0 = 'xxx xxX xxx yyy yyy yyy' WHERE id=1;
+}
+
+do_execsql_test 2.3 {
+ SELECT tokens(x1) FROM x1('xxx');
+} {
+ {xxx {} xxx} {xxx xxx}
+}
+
finish_test
do_execsql_test 1.6 {
SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank;
} {three.THREE 3 three 1 three 2}
+
+ do_execsql_test 1.7 {
+ INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB');
+ INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb');
+ INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb');
+ INSERT INTO ft2(rowid, x) VALUES(14, 'bbb BBB bbb');
+ INSERT INTO ft2(rowid, x) VALUES(15, 'bbb bbb bbb');
+ INSERT INTO ft2(rowid, x) VALUES(16, 'bbb bbb bbb');
+ INSERT INTO ft2(rowid, x) VALUES(17, 'bbb bbb bbb');
+ INSERT INTO ft2(rowid, x) VALUES(18, 'bbb bbb bbb');
+ INSERT INTO ft2(rowid, x) VALUES(19, 'bbb bbb bbb');
+ INSERT INTO ft2(rowid, x) VALUES(20, 'bbb bbb bbb');
+ INSERT INTO ft2(rowid, x) VALUES(21, 'bbb bbb bbb');
+ INSERT INTO ft2(rowid, x) VALUES(22, 'bbb bbb bbb');
+ INSERT INTO ft2(rowid, x) VALUES(23, 'bbb bbb bbb');
+ INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB');
+ }
+
+ do_execsql_test 1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24}
+ do_execsql_test 1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24}
+
}
finish_test
db func ctrl_tokens ctrl_tokens
proc do_all_vocab_test {tn} {
- foreach ::v [vocab] {
+ foreach ::v [concat [vocab] nnn] {
set answer [execsql {
SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%'
}]
}
}
-
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE ft USING fts5(
x, tokenize="origintext unicode61", content=, contentless_delete=1,
do_all_vocab_test 2.3.$ii
}
+#-------------------------------------------------------------------------
+
+unset -nocomplain ::expanded_vocab
+proc vocab {} {
+ list abcde fghij klmno
+}
+
+proc do_all_vocab_test3 {tn} {
+ foreach ::v [concat [vocab] nnn] {
+ set answer [execsql {
+ SELECT rowid, ctrl_tokens($::v, w) FROM ctrl3 WHERE w LIKE '%' || $::v || '%'
+ }]
+ do_execsql_test $tn.$::v.1 {
+ SELECT rowid, tokens(ft3) FROM ft3($::v)
+ } $answer
+ do_execsql_test $tn.$::v.2 {
+ SELECT rowid, tokens(ft3) FROM ft3($::v) ORDER BY rank
+ } $answer
+ }
+}
+
+do_execsql_test 3.0 {
+ CREATE VIRTUAL TABLE ft3 USING fts5(
+ w, tokenize="origintext unicode61", content=, contentless_delete=1,
+ tokendata=1
+ );
+ INSERT INTO ft3(ft3, rank) VALUES('rank', 'rankfunc()');
+ CREATE TABLE ctrl3(w);
+}
+
+do_execsql_test 3.1 {
+ WITH s(i) AS (
+ SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2
+ )
+ INSERT INTO ctrl3 SELECT document() FROM s;
+ INSERT INTO ft3(rowid, w) SELECT rowid, w FROM ctrl3;
+}
+
+do_all_vocab_test3 3.2
+
+
finish_test
-C Add\sfurther\stests\sfor\sxInstToken().
-D 2023-12-04T19:48:08.530
+C Further\stests\sfor\sthe\snew\scode\son\sthis\sbranch.
+D 2023-12-05T18:36:23.618
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
-F ext/fts5/fts5_expr.c 920516be4aac44eccdd9e747fe26f367442848b9a58971e4b36edb0c8284b6b8
+F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e
F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
-F ext/fts5/fts5_index.c 7111fed6c01048e227cc8c681306fa2db1e5ad29429b1373e665b8e95a7868ba
+F ext/fts5/fts5_index.c be39b44ff8773cff56bcbc01f74701a83e068c20d773cafd01e8bb2fa0fc1bc5
F ext/fts5/fts5_main.c fb7ec495d663f40d18e420e1986316591041a70e1e4b4696ab2a7384e4c7fd7a
F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075
F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1
F ext/fts5/test/fts5faultG.test d2e5a4d9a34e08dcaadcaeafef74d10cbc2abdd11aa2659a18af0294bf2812d3
+F ext/fts5/test/fts5faultH.test d845f45dac3e1a3f20c7e0a2be95280c95d3204c06802f86ab2c110e52ed3d14
F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079
F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717
F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785
F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca
F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e63fbc153b718dbe430a39
-F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969
-F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a
+F ext/fts5/test/fts5origintext2.test f3b9436de540828d01f0672df855b09ebc0863e126d5b56234701d71dfa73634
+F ext/fts5/test/fts5origintext3.test 0d25933506600452a5ab3873cbb418ed5f2de2446c3672b9997b1ea104b0e7f0
F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871
-F ext/fts5/test/fts5origintext5.test 067bfb3008323585df640ab29e8ef7c4ca6dec62c597be07a9f896d88f98cd10
+F ext/fts5/test/fts5origintext5.test a037bdf7235a22033c4663837bdb12d9738245464a3ac2f60c71fc40d07ede7d
F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 78fbb71598b1ca756acc078253880a1d0f7983a5a26b9efc683e6488122505a1
-R ced1cc2c59284a9ef3026043e080f745
+P 8582707f16133f003a6687f68cbea03d4eb6c2a0e2e07746b7cace0c44e84fa4
+R c18539c880ee946e73119264163283e7
U dan
-Z e2cce7acff968ef3c9472723192ae452
+Z 86bd4754d1465fbe5cee9e784de6e900
# Remove this line to create a well-formed Fossil manifest.
-8582707f16133f003a6687f68cbea03d4eb6c2a0e2e07746b7cace0c44e84fa4
\ No newline at end of file
+59d008b6c23ab900377bc696ee19381feb7614bac80546eae361e401c3620c4e
\ No newline at end of file
projects / thirdparty / sqlite.git / commitdiff
