Fix bugs in fts5 synonym processing for detail=col and other modes.

FossilOrigin-Name: 0e3c545423246231ddac8fed2f103d71d556e17f

diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 3804a2abfe5067e2f4e85734d599c847d95b339d..16209bf4ba2eb272ed60833d0f47d08e1b7dcf29 100644 (file)
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -40,6 +40,7 @@ void sqlite3Fts5ParserTrace(FILE*, char*);
 
 struct Fts5Expr {
   Fts5Index *pIndex;
+  Fts5Config *pConfig;
   Fts5ExprNode *pRoot;
   int bDesc;                      /* Iterate in descending rowid order */
   int nPhrase;                    /* Number of phrases in expression */
@@ -235,6 +236,7 @@ int sqlite3Fts5ExprNew(
     }else{
       pNew->pRoot = sParse.pExpr;
       pNew->pIndex = 0;
+      pNew->pConfig = pConfig;
       pNew->apExprPhrase = sParse.apPhrase;
       pNew->nPhrase = sParse.nPhrase;
       sParse.apPhrase = 0;
@@ -299,8 +301,9 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
 /*
 ** Argument pTerm must be a synonym iterator.
 */
-static int fts5ExprSynonymPoslist(
+static int fts5ExprSynonymList(
   Fts5ExprTerm *pTerm, 
+  int bCollist, 
   Fts5Colset *pColset,
   i64 iRowid,
   int *pbDel,                     /* OUT: Caller should sqlite3_free(*pa) */
@@ -319,9 +322,16 @@ static int fts5ExprSynonymPoslist(
     if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){
       const u8 *a;
       int n;
-      i64 dummy;
-      rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
+
+      if( bCollist ){
+        rc = sqlite3Fts5IterCollist(pIter, &a, &n);
+      }else{
+        i64 dummy;
+        rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
+      }
+
       if( rc!=SQLITE_OK ) goto synonym_poslist_out;
+      if( n==0 ) continue;
       if( nIter==nAlloc ){
         int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
         Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte);
@@ -422,8 +432,8 @@ static int fts5ExprPhraseIsMatch(
     int bFlag = 0;
     const u8 *a = 0;
     if( pTerm->pSynonym ){
-      rc = fts5ExprSynonymPoslist(
-          pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
+      rc = fts5ExprSynonymList(
+          pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
       );
     }else{
       rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy);
@@ -757,30 +767,51 @@ static int fts5ExprNearTest(
 ){
   Fts5ExprNearset *pNear = pNode->pNear;
   int rc = *pRc;
-  int i;
 
-  /* Check that each phrase in the nearset matches the current row.
-  ** Populate the pPhrase->poslist buffers at the same time. If any
-  ** phrase is not a match, break out of the loop early.  */
-  for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
-    Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
-    if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
-      int bMatch = 0;
-      rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
-      if( bMatch==0 ) break;
-    }else{
-      rc = sqlite3Fts5IterPoslistBuffer(
-          pPhrase->aTerm[0].pIter, &pPhrase->poslist
-      );
+  if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){
+    Fts5ExprTerm *pTerm;
+    Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
+    pPhrase->poslist.n = 0;
+    for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
+      Fts5IndexIter *pIter = pTerm->pIter;
+      if( sqlite3Fts5IterEof(pIter)==0 ){
+        int n;
+        i64 iRowid;
+        rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid);
+        if( rc!=SQLITE_OK ){
+          *pRc = rc;
+          return 0;
+        }else if( iRowid==pNode->iRowid && n>0 ){
+          pPhrase->poslist.n = 1;
+        }
+      }
     }
-  }
+    return pPhrase->poslist.n;
+  }else{
+    int i;
 
-  *pRc = rc;
-  if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
-    return 1;
-  }
+    /* Check that each phrase in the nearset matches the current row.
+    ** Populate the pPhrase->poslist buffers at the same time. If any
+    ** phrase is not a match, break out of the loop early.  */
+    for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
+      Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
+      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
+        int bMatch = 0;
+        rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
+        if( bMatch==0 ) break;
+      }else{
+        rc = sqlite3Fts5IterPoslistBuffer(
+            pPhrase->aTerm[0].pIter, &pPhrase->poslist
+        );
+      }
+    }
 
-  return 0;
+    *pRc = rc;
+    if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
+      return 1;
+    }
+    return 0;
+  }
 }
 
 static int fts5ExprTokenTest(
@@ -1606,6 +1637,7 @@ int sqlite3Fts5ExprClonePhrase(
   if( rc==SQLITE_OK ){
     /* All the allocations succeeded. Put the expression object together. */
     pNew->pIndex = pExpr->pIndex;
+    pNew->pConfig = pExpr->pConfig;
     pNew->nPhrase = 1;
     pNew->apExprPhrase[0] = sCtx.pPhrase;
     pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
@@ -2453,17 +2485,35 @@ int sqlite3Fts5ExprPhraseCollist(
 ){
   Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
   Fts5ExprNode *pNode = pPhrase->pNode;
-  assert( iPhrase>=0 && iPhrase<pExpr->nPhrase );
+  int rc = SQLITE_OK;
 
+  assert( iPhrase>=0 && iPhrase<pExpr->nPhrase );
   if( pNode->bEof==0 
    && pNode->iRowid==pExpr->pRoot->iRowid 
    && pPhrase->poslist.n>0
   ){
-    sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist);
+    Fts5ExprTerm *pTerm = &pPhrase->aTerm[0];
+    if( pTerm->pSynonym ){
+      int bDel = 0;
+      u8 *a;
+      rc = fts5ExprSynonymList(
+          pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist
+      );
+      if( bDel ){
+        sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a);
+        *ppCollist = pPhrase->poslist.p;
+        sqlite3_free(a);
+      }else{
+        *ppCollist = a;
+      }
+    }else{
+      sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist);
+    }
   }else{
     *ppCollist = 0;
     *pnCollist = 0;
   }
-  return SQLITE_OK;
+
+  return rc;
 }
 

diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test
index e3a800375853b4a28db76358dd0dde3c1eb7e8f3..96cacb293f2960b71efce47f1416bdbbbe980377 100644 (file)
--- a/ext/fts5/test/fts5synonym2.test
+++ b/ext/fts5/test/fts5synonym2.test
@@ -13,7 +13,7 @@
 #
 
 source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5synonym
+set testprefix fts5synonym2
 
 # If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
@@ -47,7 +47,7 @@ proc tcl_tokenize {tflags text} {
   foreach {w iStart iEnd} [fts5_tokenize_split $text] {
     sqlite3_fts5_token $w $iStart $iEnd
     if {$tflags == "query"} {
-      foreach s $::SYNDICT($w)  { sqlite3_fts5_token -colo $s $iStart $iEnd }
+      foreach s $::syn($w)  { sqlite3_fts5_token -colo $s $iStart $iEnd }
     }
   }
 }
@@ -67,7 +67,6 @@ fts5_aux_test_functions db
 
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%);
-
   INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i');
   INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi');
   INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3');
@@ -120,14 +119,30 @@ do_execsql_test 1.0 {
 }
 
 foreach {tn expr} {
-  1 "eight"
+  1.1 "one"   1.2 "two"   1.3 "three"   1.4 "four"
+  1.5 "v"     1.6 "vi"    1.7 "vii"     1.8 "viii"
+  1.9 "9"    1.10 "0"    1.11 "1"      1.12 "2"
+
+  2.1 "one OR two OR three OR four"
+  2.2 "(one AND two) OR (three AND four)"
+  2.3 "(one AND two) OR (three AND four) NOT five"
+  2.4 "(one AND two) NOT 6"
+
+  3.1 "b:one AND a:two"
+  3.2 "b:one OR a:two"
+  3.3 "a:one OR b:1 OR {a b} : i"
+
+  4.1 "NEAR(one two, 2)"
+  4.2 "NEAR(one two three, 2)"
+  4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)"
 } {
   if {[fts5_expr_ok $expr ss]==0} {
     do_test 1.$tn.OMITTED { list } [list]
     continue
   }
 
-  set res [fts5_query_data $expr ss ASC ::SYNDICT]
+  set res [fts5_query_data $expr ss ASC ::syn]
+  breakpoint
   do_execsql_test 1.$tn.[llength $res].asc {
     SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
   } $res

diff --git a/manifest b/manifest
index 48daf058c3463855150d2d1488abe01f69d9b192..e4ef860b2cbede1f142882ccaf5d8cef4d02a644 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Begin\sadding\sfts5\stests\sinvolving\ssynonyms\sand\sdetail=none/col\stables.
-D 2016-01-08T17:21:18.901
+C Fix\sbugs\sin\sfts5\ssynonym\sprocessing\sfor\sdetail=col\sand\sother\smodes.
+D 2016-01-11T17:30:28.056
 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042
@@ -101,7 +101,7 @@ F ext/fts5/fts5Int.h 3918045ebceb1d600c9c1c1b460489ff0c788e96
 F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e
 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1
 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6
-F ext/fts5/fts5_expr.c 913ab9f9a7651d76f055f9847ca0ae0e2f5da78d
+F ext/fts5/fts5_expr.c 6eba2220747ea1b20a358fb3b34b2ab78323e285
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
 F ext/fts5/fts5_index.c 04fe098ffab66d9424ba4e7c0b88ce7c98748cb1
 F ext/fts5/fts5_main.c 03bd44e4bd0ba16213ca9259ad5df1d4d743fd7e
@@ -173,7 +173,7 @@ F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821
 F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9
 F ext/fts5/test/fts5simple2.test 843f1f7fe439ff32bf74f4fd6430632f9636ef3a
 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
-F ext/fts5/test/fts5synonym2.test 6aa842d0e5bd019db0c3597e0860eb68eb2867e4
+F ext/fts5/test/fts5synonym2.test d2d9099d9d105b55ea03fd52d61ae2847d534129
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
@@ -1410,7 +1410,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P d9135cc723fc4227aace6dcf4ffa4630c9d23aa0
-R c443735bd0d019a3a4798f3f3350cb78
+P b3e6f15ec2d9a834e2c80b91ffd7097553816228
+R f239c4ba67c1753046748953bb2cde92
 U dan
-Z 48c62f9fa751846d5897bddd7cc5158f
+Z 915aa1f2ec9df261dd260c37cdbace05

diff --git a/manifest.uuid b/manifest.uuid
index dc0f491a48db07fafbff9f0b2e6df55b0ed96084..3b1a59f4b64564bec889cc5137dbc7c987319015 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b3e6f15ec2d9a834e2c80b91ffd7097553816228
\ No newline at end of file
+0e3c545423246231ddac8fed2f103d71d556e17f
\ No newline at end of file