From: dan Date: Wed, 2 Sep 2015 08:22:41 +0000 (+0000) Subject: Fix a problem with fts5 synonyms and the xQueryPhrase() auxiliary function API. X-Git-Tag: version-3.9.0~153^2~13 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=df998c3d4f05e59411db9abca726ae2cc81d2d5c;p=thirdparty%2Fsqlite.git Fix a problem with fts5 synonyms and the xQueryPhrase() auxiliary function API. FossilOrigin-Name: cf3e45e76d23e10ee06296c3561a341591597a04 --- diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 1ad61617d6..1da017fb01 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -599,7 +599,7 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); -int sqlite3Fts5ExprPhraseExpr(Fts5Config*, Fts5Expr*, int, Fts5Expr**); +int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a44c4e4427..d4fd6c3634 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -251,79 +251,6 @@ int sqlite3Fts5ExprNew( return sParse.rc; } -/* -** Create a new FTS5 expression by cloning phrase iPhrase of the -** expression passed as the second argument. -*/ -int sqlite3Fts5ExprPhraseExpr( - Fts5Config *pConfig, - Fts5Expr *pExpr, - int iPhrase, - Fts5Expr **ppNew -){ - int rc = SQLITE_OK; /* Return code */ - Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ - Fts5ExprPhrase *pCopy; /* Copy of pOrig */ - Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ - - pOrig = pExpr->apExprPhrase[iPhrase]; - pCopy = (Fts5ExprPhrase*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm - ); - if( pCopy ){ - int i; /* Used to iterate through phrase terms */ - Fts5ExprPhrase **apPhrase; - Fts5ExprNode *pNode; - Fts5ExprNearset *pNear; - - pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); - apPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprPhrase*) - ); - pNode = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprNode)); - pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*) - ); - - for(i=0; inTerm; i++){ - pCopy->aTerm[i].zTerm = sqlite3Fts5Strndup(&rc, pOrig->aTerm[i].zTerm,-1); - pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; - } - - if( rc==SQLITE_OK ){ - /* All the allocations succeeded. Put the expression object together. */ - pNew->pIndex = pExpr->pIndex; - pNew->pRoot = pNode; - pNew->nPhrase = 1; - pNew->apExprPhrase = apPhrase; - pNew->apExprPhrase[0] = pCopy; - - pNode->eType = (pOrig->nTerm==1 ? FTS5_TERM : FTS5_STRING); - pNode->pNear = pNear; - - pNear->nPhrase = 1; - pNear->apPhrase[0] = pCopy; - - pCopy->nTerm = pOrig->nTerm; - pCopy->pNode = pNode; - }else{ - /* At least one allocation failed. Free them all. */ - for(i=0; inTerm; i++){ - sqlite3_free(pCopy->aTerm[i].zTerm); - } - sqlite3_free(pCopy); - sqlite3_free(pNear); - sqlite3_free(pNode); - sqlite3_free(apPhrase); - sqlite3_free(pNew); - pNew = 0; - } - } - - *ppNew = pNew; - return rc; -} - /* ** Free the expression node object passed as the only argument. */ @@ -1550,8 +1477,8 @@ static int fts5ParseTokenize( int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ - int iStart, /* Start offset of token */ - int iEnd /* End offset of token */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ ){ int rc = SQLITE_OK; const int SZALLOC = 8; @@ -1577,7 +1504,7 @@ static int fts5ParseTokenize( pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew - ); + ); if( pNew==0 ) return SQLITE_NOMEM; if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); pCtx->pPhrase = pPhrase = pNew; @@ -1675,6 +1602,83 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( return sCtx.pPhrase; } +/* +** Create a new FTS5 expression by cloning phrase iPhrase of the +** expression passed as the second argument. +*/ +int sqlite3Fts5ExprClonePhrase( + Fts5Config *pConfig, + Fts5Expr *pExpr, + int iPhrase, + Fts5Expr **ppNew +){ + int rc = SQLITE_OK; /* Return code */ + Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ + Fts5ExprPhrase *pCopy; /* Copy of pOrig */ + int i; /* Used to iterate through phrase terms */ + + Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ + Fts5ExprPhrase **apPhrase; /* pNew->apPhrase */ + Fts5ExprNode *pNode; /* pNew->pRoot */ + Fts5ExprNearset *pNear; /* pNew->pRoot->pNear */ + + TokenCtx sCtx = {0}; /* Context object for fts5ParseTokenize */ + + + pOrig = pExpr->apExprPhrase[iPhrase]; + + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + if( rc==SQLITE_OK ){ + pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprPhrase*)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNode)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); + } + + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + const char *zTerm = p->zTerm; + rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, strlen(zTerm), 0, 0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + } + } + + if( rc==SQLITE_OK ){ + /* All the allocations succeeded. Put the expression object together. */ + pNew->pIndex = pExpr->pIndex; + pNew->nPhrase = 1; + pNew->apExprPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->nPhrase = 1; + sCtx.pPhrase->pNode = pNew->pRoot; + + if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ + pNew->pRoot->eType = FTS5_TERM; + }else{ + pNew->pRoot->eType = FTS5_STRING; + } + }else{ + sqlite3Fts5ExprFree(pNew); + fts5ExprPhraseFree(sCtx.pPhrase); + pNew = 0; + } + + *ppNew = pNew; + return rc; +} + + /* ** Token pTok has appeared in a MATCH expression where the NEAR operator ** is expected. If token pTok does not contain "NEAR", store an error diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 4704fb8906..f337005d00 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1860,7 +1860,7 @@ static int fts5ApiQueryPhrase( pNew->iFirstRowid = SMALLEST_INT64; pNew->iLastRowid = LARGEST_INT64; pNew->base.pVtab = (sqlite3_vtab*)pTab; - rc = sqlite3Fts5ExprPhraseExpr(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr); + rc = sqlite3Fts5ExprClonePhrase(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr); } if( rc==SQLITE_OK ){ diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index ddd685c155..7bc3ffaf90 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -298,6 +298,32 @@ foreach {tn q res} { } $res } +# Test that the xQueryPhrase() API works with synonyms. +# +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit +sqlite3_fts5_register_matchinfo db + +foreach {tn q res} { + 1 {one} { + 1 {1 11 7 2 12 6} 2 {2 11 7 0 12 6} + 3 {2 11 7 1 12 6} 4 {1 11 7 2 12 6} + 5 {3 11 7 0 12 6} 6 {0 11 7 2 12 6} + 7 {0 11 7 3 12 6} 8 {1 11 7 0 12 6} + 9 {1 11 7 2 12 6} + } +} { + do_execsql_test 5.2.$tn { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH $q + } $res +} + + #------------------------------------------------------------------------- # Test terms with more than 4 synonyms. # diff --git a/manifest b/manifest index 55a78b93ad..c5af116d50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sthat\soccurs\swhen\smore\sthan\s4\ssynonyms\sfor\sa\sterm\sappear\swithin\sa\ssingle\srow. -D 2015-09-01T18:44:33.194 +C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sthe\sxQueryPhrase()\sauxiliary\sfunction\sAPI. +D 2015-09-02T08:22:41.779 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,14 +106,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 -F ext/fts5/fts5Int.h d46f89aeb357fbcf5b268d71b0d2c5000cd27bd9 +F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 44caa0ccd7e9a392864fda0c14e9b9829a395a84 +F ext/fts5/fts5_expr.c d92beea335f96f9256cc19203b21fd45826655c9 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 -F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 +F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e F ext/fts5/fts5_storage.c c888defbb961d64c12299b3d1725a24a770b047e F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 4321e59c29186d9187cf8ab576e70530176eae49 +F ext/fts5/test/fts5synonym.test fd66afccec36bf719ba66a5b6579efe007607b05 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dbcb73802b88f76be17f09f3eb83ffac542de633 -R 07c5ba5d7a2806d5caa7dc123b3f32ad +P cd359550bdc2bf7be4c294b60130c9fc3f583902 +R f0fefb84cb3010de72f87418a092a81a U dan -Z fc32f7df10846658b7b8e62d59624854 +Z b6230653499a35c239ca936f1dd1428a diff --git a/manifest.uuid b/manifest.uuid index 6c61644fa9..7c523ff1d0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cd359550bdc2bf7be4c294b60130c9fc3f583902 \ No newline at end of file +cf3e45e76d23e10ee06296c3561a341591597a04 \ No newline at end of file