int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
+/* Used to populate hash tables for xInstToken in detail=none/column mode. */
+void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
+int sqlite3Fts5IndexIterWriteTokendata(
+ Fts5IndexIter*, const char*, int, int iCol, int iOff
+);
+int sqlite3Fts5IndexIterHashifyTokendata(Fts5IndexIter*);
+
/*
** End of interface to code in fts5_index.c.
**************************************************************************/
return 0;
}
+static int fts5QueryTerm(const char *pToken, int nToken){
+ int ii;
+ for(ii=0; ii<nToken && pToken[ii]; ii++){}
+ return ii;
+}
+
static int fts5ExprPopulatePoslistsCb(
void *pCtx, /* Copy of 2nd argument to xTokenize() */
int tflags, /* Mask of FTS5_TOKEN_* flags */
Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
Fts5Expr *pExpr = p->pExpr;
int i;
+ int nQuery = nToken;
UNUSED_PARAM2(iUnused1, iUnused2);
- if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
+ if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE;
+ if( pExpr->pConfig->bTokendata ){
+ nQuery = fts5QueryTerm(pToken, nQuery);
+ }
if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
for(i=0; i<pExpr->nPhrase; i++){
Fts5ExprTerm *pT;
if( p->aPopulator[i].bOk==0 ) continue;
for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
- if( (pT->nFullTerm==nToken || (pT->nFullTerm<nToken && pT->bPrefix))
- && memcmp(pT->pTerm, pToken, pT->nFullTerm)==0
+ if( (pT->nQueryTerm==nQuery || (pT->nQueryTerm<nQuery && pT->bPrefix))
+ && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0
){
int rc = sqlite3Fts5PoslistWriterAppend(
&pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
);
+ if( rc==SQLITE_OK && pExpr->pConfig->bTokendata ){
+ int iCol = p->iOff>>32;
+ int iTokOff = p->iOff & 0x7FFFFFFF;
+ rc = sqlite3Fts5IndexIterWriteTokendata(
+ pT->pIter, pToken, nToken, iCol, iTokOff
+ );
+ }
if( rc ) return rc;
break;
}
const char *z, int n
){
int i;
+ int rc = SQLITE_OK;
Fts5ExprCtx sCtx;
sCtx.pExpr = pExpr;
sCtx.aPopulator = aPopulator;
sCtx.iOff = (((i64)iCol) << 32) - 1;
+ /* If this is a tokendata=1 table, clear out the hash tables of
+ ** full-terms. */
+ if( pConfig->bTokendata ){
+ for(i=0; i<pExpr->nPhrase; i++){
+ Fts5ExprTerm *pT;
+ for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
+ sqlite3Fts5IndexIterClearTokendata(pT->pIter);
+ }
+ }
+ }
+
for(i=0; i<pExpr->nPhrase; i++){
Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
Fts5Colset *pColset = pNode->pNear->pColset;
}
}
- return sqlite3Fts5Tokenize(pConfig,
+ rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
);
+
+ if( pConfig->bTokendata ){
+ for(i=0; i<pExpr->nPhrase; i++){
+ Fts5ExprTerm *pT;
+ for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
+ sqlite3Fts5IndexIterHashifyTokendata(pT->pIter);
+ }
+ }
+ }
+
+ return rc;
}
static void fts5ExprClearPoslists(Fts5ExprNode *pNode){
return SQLITE_OK;
}
+/*
+** Clear any existing entries from the token-map associated with the
+** iterator passed as the only argument.
+*/
+void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){
+ Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
+ assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL );
+ if( pIter->pTokenMap ){
+ pIter->pTokenMap->nEntry = 0;
+ }
+}
+
+int sqlite3Fts5IndexIterWriteTokendata(
+ Fts5IndexIter *pIndexIter,
+ const char *pToken, int nToken,
+ int iCol, int iOff
+){
+ Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
+ Fts5Index *p = pIter->pIndex;
+ assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL );
+ if( pIter->pTokenMap==0 ){
+ pIter->pTokenMap = (Fts5TokenMap*)fts5IdxMalloc(p, sizeof(Fts5TokenMap));
+ }
+ if( p->rc==SQLITE_OK ){
+ Fts5TokenMap *pMap = pIter->pTokenMap;
+ int ii;
+ for(ii=0; ii<pMap->nToken; ii++){
+ if( nToken==pMap->aToken[ii].nTerm
+ && 0==memcmp(pMap->aToken[ii].pTerm, pToken, nToken)
+ ){
+ break;
+ }
+ }
+ if( ii==pMap->nToken ){
+ fts5TokenMapTerm(p, pMap, (const u8*)pToken, nToken);
+ }
+ if( pMap->nEntry>=pMap->nEntryAlloc ){
+ int nNew = pMap->nEntryAlloc ? pMap->nEntryAlloc*2 : 32;
+ Fts5TokenMapEntry *aNew = (Fts5TokenMapEntry*)sqlite3_realloc(
+ pMap->aEntry, nNew * sizeof(Fts5TokenMapEntry)
+ );
+ if( aNew==0 ){
+ p->rc = SQLITE_NOMEM;
+ }else{
+ pMap->aEntry = aNew;
+ pMap->nEntryAlloc = nNew;
+ }
+ }
+ if( p->rc==SQLITE_OK ){
+ Fts5TokenMapEntry *pEntry = &pMap->aEntry[pMap->nEntry++];
+ pEntry->iRowid = pIndexIter->iRowid;
+ pEntry->iCol = iCol;
+ pEntry->iOff = iOff;
+ pEntry->iTok = ii+1;
+ }
+ }
+ return fts5IndexReturn(p);
+}
+
+int sqlite3Fts5IndexIterHashifyTokendata(Fts5IndexIter *pIndexIter){
+ Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
+ if( pIter->pTokenMap ){
+ fts5TokenMapHashify(pIter->pIndex, pIter->pTokenMap);
+ }
+ return fts5IndexReturn(pIter->pIndex);
+}
+
/*
** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
*/
return
}
+foreach_detail_mode $testprefix {
+
sqlite3_fts5_register_origintext db
do_execsql_test 1.0 {
- CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="origintext unicode61");
+ CREATE VIRTUAL TABLE ft USING fts5(
+ x, tokenize="origintext unicode61", detail=%DETAIL%
+ );
CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
}
sqlite3_fts5_register_origintext db
do_execsql_test 2.0 {
- CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="origintext unicode61");
+ CREATE VIRTUAL TABLE ft USING fts5(
+ x, tokenize="origintext unicode61", detail=%DETAIL%
+ );
INSERT INTO ft(ft, rank) VALUES('pgsz', 128);
CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
}
sqlite3_fts5_register_origintext db
do_execsql_test 3.0 {
- CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="origintext unicode61");
+ CREATE VIRTUAL TABLE ft USING fts5(
+ x, tokenize="origintext unicode61", detail=%DETAIL%
+ );
CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
INSERT INTO ft(rowid, x) VALUES(1, 'hello');
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE ft2 USING fts5(x,
tokenize="origintext unicode61",
- tokendata=1
+ tokendata=1,
+ detail=%DETAIL%
);
CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance);
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE ft USING fts5(
- x, tokenize='origintext unicode61', tokendata=1
+ x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
);
INSERT INTO ft VALUES('one two three four');
}
do_execsql_test 4.3 {
SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE')
} {1 two.TWO}
-do_execsql_test 4.4 {
- SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
-} {1 three.ThreE}
-
-do_catchsql_test 4.5 {
- SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
-} {1 SQLITE_RANGE}
-do_catchsql_test 4.6 {
- SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
-} {1 SQLITE_RANGE}
-do_catchsql_test 4.7 {
- SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
-} {1 SQLITE_RANGE}
+
+if {"%DETAIL%"=="full"} {
+ # Phrase queries are only supported for detail=full.
+ #
+ do_execsql_test 4.4 {
+ SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
+ } {1 three.ThreE}
+ do_catchsql_test 4.5 {
+ SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
+ } {1 SQLITE_RANGE}
+ do_catchsql_test 4.6 {
+ SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
+ } {1 SQLITE_RANGE}
+ do_catchsql_test 4.7 {
+ SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
+ } {1 SQLITE_RANGE}
+}
#-------------------------------------------------------------------------
#
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE ft USING fts5(
- x, tokenize='origintext unicode61', tokendata=1
+ x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
);
INSERT INTO ft VALUES('one ONE One oNe oNE one');
}
{0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5}
}
+}
+
finish_test
#
source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5origintext
+set testprefix fts5origintext2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
--- /dev/null
+# 2023 November 22
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext3
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+ finish_test
+ return
+}
+
+foreach_detail_mode $testprefix {
+ reset_db
+
+ sqlite3_fts5_register_origintext db
+ fts5_aux_test_functions db
+ proc insttoken {cmd iIdx iToken} {
+ set txt [$cmd xInstToken $iIdx $iToken]
+ string map [list "\0" "."] $txt
+ }
+ sqlite3_fts5_create_function db insttoken insttoken
+
+ do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE ft USING fts5(
+ x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
+ );
+ }
+
+ do_execsql_test 1.1 {
+ INSERT INTO ft VALUES('Hello world HELLO WORLD hello');
+ }
+
+ do_execsql_test 1.2 {
+ SELECT fts5_test_poslist(ft) FROM ft('hello');
+ } {{0.0.0 0.0.2 0.0.4}}
+
+ do_execsql_test 1.3 {
+ SELECT
+ insttoken(ft, 0, 0),
+ insttoken(ft, 1, 0),
+ insttoken(ft, 2, 0)
+ FROM ft('hello');
+ } {hello.Hello hello.HELLO hello}
+
+}
+
+finish_test
+
-C When\squerying\sa\stokendata=1\sfts5\stable,\sdo\snot\suse\sa\sprefix\scursor\sfor\sthe\scase\swhere\sthe\sterm\shas\sonly\sone\svariant.
-D 2023-11-16T21:11:56.608
+C Fix\stokendata=1\sand\sxInstToken()\sAPIs\sfor\sdetail=none\sand\sdetail=column\stables.
+D 2023-11-22T19:02:54.078
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
F ext/fts5/fts5.h e27cdb10e38d87cb041dcb56cef97addf7d902aeab07e84e7102f5fc65d3357c
-F ext/fts5/fts5Int.h 88ab1ee1eefa6f98e4c7fd3c96c99ef76ed2819cc3058736c87bb01e4a301628
+F ext/fts5/fts5Int.h d330c2e20051c300b26325b8ba29aa89e99d301c80e2f51092e5bb70346a17cd
F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
-F ext/fts5/fts5_expr.c 4b50ed0c724cb160f086e20e964ed2d57b99d0d3c1cb1b029901c0300b11bd9f
+F ext/fts5/fts5_expr.c 0d846134eafeeb1f0724b9c8cc02a2ef9c4082519aa3923173deadd5155910b1
F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
-F ext/fts5/fts5_index.c 70fa4a6d8a062ca4b63a62d0721d72ce2f6336413c6e8b0703881c708797d24d
+F ext/fts5/fts5_index.c 7b87808d788238eff4a0a68728e6ed49817e71bbfb328a18050d7d8e92a5d66a
F ext/fts5/fts5_main.c f151eb2c6d27418d907c88cd623ad4508bdcf518a79d504e850270754c228b74
F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1
F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785
F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca
-F ext/fts5/test/fts5origintext.test 908a1fb6b1106e4b6ed0f9cf683c2ad7f986cce1aea1e0a13b3309c6f568932b
-F ext/fts5/test/fts5origintext2.test a654c77f1548ccd8eab7f6d07230655c0070cdf32dcd4740ccdf496f77d5282c
+F ext/fts5/test/fts5origintext.test 7caef7634889bab8b44d145141c0d9325299398fb89b116bccd6262fde5659db
+F ext/fts5/test/fts5origintext2.test 26482f4af1f2785cb01d06af9aae202289b6e8cf7b708d18aea305b459c2f302
+F ext/fts5/test/fts5origintext3.test 87a212b8235794348c56cb70f21e122d182a5af688c56057b90b7c151d0aa347
F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a34b26fe7f60b74e7ae5cf64900920a3d352a20da2496401bcbc27041689cd07
-R d7c277a055a404d272fdcb5090bf371a
+P d711c96ba855686d6881a50498418de3492144f005684b5ae55bca24413dce47
+R 161c23f360cac1706a2c5f6b11155312
U dan
-Z 0e1bf556ad9eba9db356685a09c7ab31
+Z 281a4d74e3cce55af028079774718a8b
# Remove this line to create a well-formed Fossil manifest.
-d711c96ba855686d6881a50498418de3492144f005684b5ae55bca24413dce47
\ No newline at end of file
+37b271c19d772bd06524db816ded03377b426efed7a7783c8a96f6fb156ecd86
\ No newline at end of file
projects / thirdparty / sqlite.git / commitdiff
