Begin changes to allow synonym support by adding multiple terms to a query (an altern...

FossilOrigin-Name: ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index c61f8c405236522a10c3d7a12a683046baa48e31..e8d997b060fd007dad9cc93dff839c0bfd4fc580 100644 (file)
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -236,7 +236,7 @@ struct Fts5PoslistReader {
   int i;                          /* Current offset in a[] */
 
   /* Output variables */
-  int bEof;                       /* Set to true at EOF */
+  u8 bEof;                        /* Set to true at EOF */
   i64 iPos;                       /* (iCol<<32) + iPos */
 };
 int sqlite3Fts5PoslistReaderInit(

diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 9b52bb08c6c41b0539dff35f9a80192bc27a652f..bba9932dafb07497cc86a909ddf3fa91a81a3b5b 100644 (file)
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -73,6 +73,7 @@ struct Fts5ExprTerm {
   int bPrefix;                    /* True for a prefix term */
   char *zTerm;                    /* nul-terminated term */
   Fts5IndexIter *pIter;           /* Iterator for this term */
+  Fts5ExprTerm *pSynonym;         /* Pointer to first in list of synonyms */
 };
 
 /*
@@ -181,6 +182,10 @@ static int fts5ExprGetToken(
 
     default: {
       const char *z2;
+      if( sqlite3Fts5IsBareword(z[0])==0 ){
+        sqlite3Fts5ParseError(pParse, "fts5: syntax error near \"%.1s\"", z);
+        return FTS5_EOF;
+      }
       tok = FTS5_STRING;
       for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++);
       pToken->n = (z2 - z);
@@ -350,6 +355,60 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){
   return 0;
 }
 
+/*
+** Argument pTerm must be a synonym iterator. Return the current rowid
+** that it points to.
+*/
+static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){
+  i64 iRet;
+  int bRetValid = 0;
+  Fts5ExprTerm *p;
+
+  assert( pTerm->pSynonym );
+  assert( bDesc==0 || bDesc==1 );
+  for(p=pTerm; p; p=p->pSynonym){
+    if( 0==sqlite3Fts5IterEof(p->pIter) ){
+      i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
+      if( bRetValid==0 || (bDesc!=(iRowid<iRet)) ){
+        iRet = iRowid;
+        bRetValid = 1;
+      }
+    }
+  }
+
+  assert( bRetValid );
+  return iRet;
+}
+
+/*
+** Argument pTerm must be a synonym iterator.
+*/
+static int fts5ExprSynonymPoslist(
+  Fts5ExprTerm *pTerm, 
+  i64 iRowid,
+  u8 **pa, int *pn
+){
+  Fts5PoslistWriter writer = {0};
+  Fts5PoslistReader aStatic[4];
+  Fts5PoslistReader *aIter = aStatic;
+  int nIter = 0;
+  Fts5ExprTerm *p;
+
+  assert( pTerm->pSynonym );
+  for(p=pTerm; p; p=p->pSynonym){
+    Fts5IndexIter *pIter = p->pIter;
+    if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){
+      i64 dummy;
+      int rc = sqlite3Fts5IterPoslist(pIter, (const u8**)pa, pn, &dummy);
+      return rc;
+    }
+  }
+
+  assert( 0 );
+  return SQLITE_ERROR;
+}
+
+
 /*
 ** All individual term iterators in pPhrase are guaranteed to be valid and
 ** pointing to the same rowid when this function is called. This function 
@@ -362,7 +421,7 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){
 ** not a match.
 */
 static int fts5ExprPhraseIsMatch(
-  Fts5Expr *pExpr,                /* Expression pPhrase belongs to */
+  Fts5ExprNode *pNode,            /* Node pPhrase belongs to */
   Fts5ExprColset *pColset,        /* Restrict matches to these columns */
   Fts5ExprPhrase *pPhrase,        /* Phrase object to initialize */
   int *pbMatch                    /* OUT: Set to true if really a match */
@@ -391,10 +450,15 @@ static int fts5ExprPhraseIsMatch(
 
   /* Initialize a term iterator for each term in the phrase */
   for(i=0; i<pPhrase->nTerm; i++){
+    Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
     i64 dummy;
     int n;
     const u8 *a;
-    rc = sqlite3Fts5IterPoslist(pPhrase->aTerm[i].pIter, &a, &n, &dummy);
+    if( pTerm->pSynonym ){
+      rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, (u8**)&a, &n);
+    }else{
+      rc = sqlite3Fts5IterPoslist(pTerm->pIter, &a, &n, &dummy);
+    }
     if( rc || sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ){
       goto ismatch_out;
     }
@@ -598,17 +662,43 @@ static int fts5ExprNearAdvanceFirst(
   int bFromValid,
   i64 iFrom 
 ){
-  Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter;
+  Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0];
   int rc;
 
-  assert( Fts5NodeIsString(pNode) );
-  if( bFromValid ){
-    rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
+  if( pTerm->pSynonym ){
+    int bEof = 1;
+    Fts5ExprTerm *p;
+
+    /* Find the firstest rowid any synonym points to. */
+    i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc);
+
+    /* Advance each iterator that currently points to iRowid */
+    for(p=pTerm; p; p=p->pSynonym){
+      if( sqlite3Fts5IterEof(p->pIter)==0 ){
+        bEof = 0;
+        if( sqlite3Fts5IterRowid(p->pIter)==iRowid ){
+          rc = sqlite3Fts5IterNext(p->pIter);
+          if( rc!=SQLITE_OK ) break;
+        }
+      }
+    }
+
+    /* Set the EOF flag if either all synonym iterators are at EOF or an
+    ** error has occurred.  */
+    pNode->bEof = (rc || bEof);
   }else{
-    rc = sqlite3Fts5IterNext(pIter);
+    Fts5IndexIter *pIter = pTerm->pIter;
+
+    assert( Fts5NodeIsString(pNode) );
+    if( bFromValid ){
+      rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
+    }else{
+      rc = sqlite3Fts5IterNext(pIter);
+    }
+
+    pNode->bEof = (rc || sqlite3Fts5IterEof(pIter));
   }
 
-  pNode->bEof = (rc || sqlite3Fts5IterEof(pIter));
   return rc;
 }
 
@@ -719,7 +809,7 @@ static int fts5ExprNearTest(
     Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
     if( pPhrase->nTerm>1 || pNear->pColset ){
       int bMatch = 0;
-      rc = fts5ExprPhraseIsMatch(pExpr, pNear->pColset, pPhrase, &bMatch);
+      rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
       if( bMatch==0 ) break;
     }else{
       rc = sqlite3Fts5IterPoslistBuffer(
@@ -755,6 +845,7 @@ static int fts5ExprTokenTest(
 
   assert( pNode->eType==FTS5_TERM );
   assert( pNear->nPhrase==1 && pPhrase->nTerm==1 );
+  assert( pPhrase->aTerm[0].pSynonym==0 );
 
   rc = sqlite3Fts5IterPoslist(pIter, &pPos, &nPos, &pNode->iRowid);
 
@@ -801,69 +892,107 @@ static int fts5ExprNearNextMatch(
   i64 iLast;                      /* Lastest rowid any iterator points to */
   int i, j;                       /* Phrase and token index, respectively */
   int bMatch;                     /* True if all terms are at the same rowid */
+  const int bDesc = pExpr->bDesc;
 
-  assert( pNear->nPhrase>1 || pNear->apPhrase[0]->nTerm>1 );
+  /* Check that this node should not be FTS5_TERM */
+  assert( pNear->nPhrase>1 
+       || pNear->apPhrase[0]->nTerm>1 
+       || pNear->apPhrase[0]->aTerm[0].pSynonym
+  );
 
   /* Initialize iLast, the "lastest" rowid any iterator points to. If the
   ** iterator skips through rowids in the default ascending order, this means
   ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
   ** means the minimum rowid.  */
-  iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter);
+  if( pLeft->aTerm[0].pSynonym ){
+    iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc);
+  }else{
+    iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter);
+  }
 
   do {
     bMatch = 1;
     for(i=0; i<pNear->nPhrase; i++){
       Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
       for(j=0; j<pPhrase->nTerm; j++){
-        Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
-        i64 iRowid = sqlite3Fts5IterRowid(pIter);
-        if( iRowid!=iLast ) bMatch = 0;
-        if( fts5ExprAdvanceto(pIter, pExpr->bDesc, &iLast,&rc,&pNode->bEof) ){
-          return rc;
+        Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
+        if( pTerm->pSynonym ){
+          Fts5ExprTerm *p;
+          int bEof = 1;
+          i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc);
+          if( iRowid==iLast ) continue;
+          for(p=pTerm; p; p=p->pSynonym){
+            Fts5IndexIter *pIter = p->pIter;
+            int dummy;
+            if( 0==sqlite3Fts5IterEof(pIter)
+             && 0==fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &dummy)==0 
+            ){
+              bEof = 0;
+            }
+          }
+          if( bEof || rc ){
+            pNode->bEof = 1;
+            return rc;
+          }
+        }else{
+          Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
+          i64 iRowid = sqlite3Fts5IterRowid(pIter);
+          if( iRowid==iLast ) continue;
+          bMatch = 0;
+          if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
+            return rc;
+          }
         }
       }
     }
   }while( bMatch==0 );
 
-  pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode));
   pNode->iRowid = iLast;
+  pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode));
 
   return rc;
 }
 
 /*
 ** Initialize all term iterators in the pNear object. If any term is found
-** to match no documents at all, set *pbEof to true and return immediately,
-** without initializing any further iterators.
+** to match no documents at all, return immediately without initializing any
+** further iterators.
 */
 static int fts5ExprNearInitAll(
   Fts5Expr *pExpr,
   Fts5ExprNode *pNode
 ){
   Fts5ExprNearset *pNear = pNode->pNear;
-  Fts5ExprTerm *pTerm;
-  Fts5ExprPhrase *pPhrase;
   int i, j;
   int rc = SQLITE_OK;
 
   for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
-    pPhrase = pNear->apPhrase[i];
+    Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
     for(j=0; j<pPhrase->nTerm; j++){
-      pTerm = &pPhrase->aTerm[j];
-      if( pTerm->pIter ){
-        sqlite3Fts5IterClose(pTerm->pIter);
-        pTerm->pIter = 0;
+      Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
+      Fts5ExprTerm *p;
+      int bEof = 1;
+
+      for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){
+        if( p->pIter ){
+          sqlite3Fts5IterClose(p->pIter);
+          p->pIter = 0;
+        }
+        rc = sqlite3Fts5IndexQuery(
+            pExpr->pIndex, p->zTerm, strlen(p->zTerm),
+            (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
+            (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
+            &p->pIter
+        );
+        assert( rc==SQLITE_OK || p->pIter==0 );
+        if( p->pIter && 0==sqlite3Fts5IterEof(pTerm->pIter) ){
+          bEof = 0;
+        }
       }
-      rc = sqlite3Fts5IndexQuery(
-          pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm),
-          (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
-          (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
-          &pTerm->pIter
-      );
-      assert( rc==SQLITE_OK || pTerm->pIter==0 );
-      if( pTerm->pIter==0 || sqlite3Fts5IterEof(pTerm->pIter) ){
+
+      if( bEof ){
         pNode->bEof = 1;
-        break;
+        return rc;
       }
     }
   }
@@ -1266,10 +1395,16 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
   if( pPhrase ){
     int i;
     for(i=0; i<pPhrase->nTerm; i++){
+      Fts5ExprTerm *pSyn;
+      Fts5ExprTerm *pNext;
       Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
       sqlite3_free(pTerm->zTerm);
-      if( pTerm->pIter ){
-        sqlite3Fts5IterClose(pTerm->pIter);
+      sqlite3Fts5IterClose(pTerm->pIter);
+
+      for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
+        pNext = pSyn->pSynonym;
+        sqlite3Fts5IterClose(pSyn->pIter);
+        sqlite3_free(pSyn);
       }
     }
     if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist);
@@ -1348,27 +1483,37 @@ static int fts5ParseTokenize(
   const int SZALLOC = 8;
   TokenCtx *pCtx = (TokenCtx*)pContext;
   Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
-  Fts5ExprTerm *pTerm;
 
-  if( tflags & FTS5_TOKEN_COLOCATED ) return rc;
-
-  if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
-    Fts5ExprPhrase *pNew;
-    int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0);
+  assert( pPhrase==0 || pPhrase->nTerm>0 );
+  if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){
+    Fts5ExprTerm *pSyn;
+    int nByte = sizeof(Fts5ExprTerm) + nToken+1;
+    pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte);
+    if( pSyn==0 ) return SQLITE_NOMEM;
+    memset(pSyn, 0, nByte);
+    pSyn->zTerm = (char*)&pSyn[1];
+    memcpy(pSyn->zTerm, pToken, nToken);
+    pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
+    pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
+  }else{
+    Fts5ExprTerm *pTerm;
+    if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
+      Fts5ExprPhrase *pNew;
+      int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0);
+
+      pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, 
+          sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew
+          );
+      if( pNew==0 ) return SQLITE_NOMEM;
+      if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase));
+      pCtx->pPhrase = pPhrase = pNew;
+      pNew->nTerm = nNew - SZALLOC;
+    }
 
-    pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, 
-        sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew
-    );
-    if( pNew==0 ) return SQLITE_NOMEM;
-    if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase));
-    pCtx->pPhrase = pPhrase = pNew;
-    pNew->nTerm = nNew - SZALLOC;
+    pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
+    memset(pTerm, 0, sizeof(Fts5ExprTerm));
+    pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
   }
-
-  pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
-  memset(pTerm, 0, sizeof(Fts5ExprTerm));
-  pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
-
   return rc;
 }
 
@@ -1636,7 +1781,10 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
         for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
           pNear->apPhrase[iPhrase]->pNode = pRet;
         }
-        if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){
+        if( pNear->nPhrase==1 
+         && pNear->apPhrase[0]->nTerm==1 
+         && pNear->apPhrase[0]->aTerm[0].pSynonym==0
+        ){
           pRet->eType = FTS5_TERM;
         }
       }else{
@@ -1656,16 +1804,28 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
 }
 
 static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
-  char *zQuoted = sqlite3_malloc(strlen(pTerm->zTerm) * 2 + 3 + 2);
+  int nByte = 0;
+  Fts5ExprTerm *p;
+  char *zQuoted;
+
+  /* Determine the maximum amount of space required. */
+  for(p=pTerm; p; p=p->pSynonym){
+    nByte += strlen(pTerm->zTerm) * 2 + 3 + 2;
+  }
+  zQuoted = sqlite3_malloc(nByte);
+
   if( zQuoted ){
     int i = 0;
-    char *zIn = pTerm->zTerm;
-    zQuoted[i++] = '"';
-    while( *zIn ){
-      if( *zIn=='"' ) zQuoted[i++] = '"';
-      zQuoted[i++] = *zIn++;
+    for(p=pTerm; p; p=p->pSynonym){
+      char *zIn = p->zTerm;
+      zQuoted[i++] = '"';
+      while( *zIn ){
+        if( *zIn=='"' ) zQuoted[i++] = '"';
+        zQuoted[i++] = *zIn++;
+      }
+      zQuoted[i++] = '"';
+      if( p->pSynonym ) zQuoted[i++] = '|';
     }
-    zQuoted[i++] = '"';
     if( pTerm->bPrefix ){
       zQuoted[i++] = ' ';
       zQuoted[i++] = '*';

diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test
index daa535cd9b9e346e6b476d4d0519189583cbee56..22aa1b0f3513b0a08c1813d91305e0f0a18e8d6c 100644 (file)
--- a/ext/fts5/test/fts5aa.test
+++ b/ext/fts5/test/fts5aa.test
@@ -343,7 +343,7 @@ do_execsql_test 13.5 {
 } {1}
 
 do_execsql_test 13.6 {
-  SELECT rowid FROM t1 WHERE t1 MATCH '.';
+  SELECT rowid FROM t1 WHERE t1 MATCH '""';
 } {}
 
 #-------------------------------------------------------------------------

diff --git a/ext/fts5/test/fts5ea.test b/ext/fts5/test/fts5ea.test
index ad05412ba966b6afa8ad3ae2887a0adda077646c..3ccbd7d7a2d8dbe222cdd021bd5a9b0fe7c4c609 100644 (file)
--- a/ext/fts5/test/fts5ea.test
+++ b/ext/fts5/test/fts5ea.test
@@ -87,6 +87,12 @@ do_execsql_test 4.0 {
   SELECT fts5_expr('a AND """"', 'x', 'tokenize="unicode61 tokenchars ''""''"');
 } {{"a" AND """"}}
 
+#-------------------------------------------------------------------------
+# Experiment with a tokenizer that considers " to be a token character.
+#
+do_catchsql_test 5.0 {
+  SELECT fts5_expr('abc | def');
+} {1 {fts5: syntax error near "|"}}
 
 
 

diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test
index 352e1b4a17c8de062c2f2b8fda6c78f48746bed4..55c4b15cf3b66b64e9b20a18e7e4fc14798be7fe 100644 (file)
--- a/ext/fts5/test/fts5eb.test
+++ b/ext/fts5/test/fts5eb.test
@@ -30,18 +30,18 @@ proc do_syntax_test {tn expr res} {
 }
 
 foreach {tn expr res} {
-  1  {abc}                           {"abc"}
-  2  {abc .}                         {"abc"}
-  3  {.}                             {}
-  4  {abc OR .}                      {"abc"}
-  5  {abc NOT .}                     {"abc"}
-  6  {abc AND .}                     {"abc"}
-  7  {. OR abc}                      {"abc"}
-  8  {. NOT abc}                     {"abc"}
-  9  {. AND abc}                     {"abc"}
-  10 {abc + . + def}                 {"abc" + "def"}
-  11 {abc . def}                     {"abc" AND "def"}
-  12 {r+e OR w}                      {"r" + "e" OR "w"}
+  1  {abc}                            {"abc"}
+  2  {abc ""}                         {"abc"}
+  3  {""}                             {}
+  4  {abc OR ""}                      {"abc"}
+  5  {abc NOT ""}                     {"abc"}
+  6  {abc AND ""}                     {"abc"}
+  7  {"" OR abc}                      {"abc"}
+  8  {"" NOT abc}                     {"abc"}
+  9  {"" AND abc}                     {"abc"}
+  10 {abc + "" + def}                 {"abc" + "def"}
+  11 {abc "" def}                     {"abc" AND "def"}
+  12 {r+e OR w}                       {"r" + "e" OR "w"}
 } {
   do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
 }

diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test
index 812f7c58824e2f339d138b9001b46ffa07d9489b..a607b7e29c09dd240af5c8194ffc84aa18ba7844 100644 (file)
--- a/ext/fts5/test/fts5synonym.test
+++ b/ext/fts5/test/fts5synonym.test
@@ -21,6 +21,24 @@ ifcapable !fts5 {
   return
 }
 
+foreach S {
+  {zero 0}
+  {one 1}
+  {two 2}
+  {three 3 iii}
+  {four 4}
+  {five 5}
+  {six 6}
+  {seven 7}
+  {eight 8}
+  {nine 9}
+} {
+  foreach s $S {
+    set o [list]
+    foreach x $S {if {$x!=$s} {lappend o $x}}
+    set ::syn($s) $o
+  }
+}
 
 proc gobble_whitespace {textvar} {
   upvar $textvar t
@@ -75,24 +93,6 @@ do_execsql_test 1.0 {
 # Test a tokenizer that supports synonyms by adding extra entries to the
 # FTS index.
 #
-foreach S {
-  {zero 0}
-  {one 1}
-  {two 2}
-  {three 3 iii}
-  {four 4}
-  {five 5}
-  {six 6}
-  {seven 7}
-  {eight 8}
-  {nine 9}
-} {
-  foreach s $S {
-    set o [list]
-    foreach x $S {if {$x!=$s} {lappend o $x}}
-    set ::syn($s) $o
-  }
-}
 
 proc tcl_tokenize {tflags text} {
   foreach {w iStart iEnd} [do_tokenize_split $text] {
@@ -194,5 +194,30 @@ do_execsql_test 3.2.2 {
   SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three';
 } {1}
 
+#-------------------------------------------------------------------------
+# Check that expressions with synonyms can be parsed.
+#
+reset_db
+sqlite3_fts5_create_tokenizer db tcl tcl_create
+proc tcl_tokenize {tflags text} {
+  foreach {w iStart iEnd} [do_tokenize_split $text] {
+    sqlite3_fts5_token $w $iStart $iEnd
+    if {$tflags=="query" && [info exists ::syn($w)]} {
+      foreach s $::syn($w) {
+        sqlite3_fts5_token -colo $s $iStart $iEnd
+      }
+    }
+  }
+}
+
+foreach {tn expr res} {
+  1  {abc}                           {"abc"}
+  2  {one}                           {"one"|"1"}
+  3  {3}                             {"3"|"iii"|"three"}
+  4  {3*}                            {"3"|"iii"|"three" *}
+} {
+  do_execsql_test 4.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res]
+}
+
 finish_test
 

diff --git a/manifest b/manifest
index fd2b4870e3f039baef3d40bc159c33fe77df2259..4d5891206e302e5a390194c0872018c2043823bd 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\stest\sfor\san\sfts5\stokenizer\sthat\ssupports\ssynonyms\sby\sadding\smultiple\sentries\sto\sthe\sfts\sindex.
-D 2015-08-29T18:46:12.456
+C Begin\schanges\sto\sallow\ssynonym\ssupport\sby\sadding\smultiple\sterms\sto\sa\squery\s(an\salternative\sto\sadding\smultiple\sterms\sto\sthe\sFTS\sindex).
+D 2015-08-31T20:06:06.235
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in e2218eb228374422969de7b1680eda6864affcef
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -106,11 +106,11 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03
 F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42
-F ext/fts5/fts5Int.h 9fd31e682acae32806f77e7c3b543c4294274c92
+F ext/fts5/fts5Int.h 9c538f0fcc1c3bc2fa12f7199d1326bd2362ce9c
 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e
 F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384
-F ext/fts5/fts5_expr.c 7ea46f676491989069d31ae1f75c9439b0858711
+F ext/fts5/fts5_expr.c 1458c3e33c1ec3ad99284f9692edfc49c44afd7c
 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374
 F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42
@@ -124,7 +124,7 @@ F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc
 F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl 3338968de1880ca12b0451ae8f9b8b12d14e0ba7
-F ext/fts5/test/fts5aa.test c6e680a0d1b6c2616a382f1006d5d91eca697bd0
+F ext/fts5/test/fts5aa.test caa44c528f7270aa4e325c4f2c28d355c1e8c307
 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
 F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c
 F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d
@@ -149,8 +149,8 @@ F ext/fts5/test/fts5corrupt2.test 1a830ccd6dbe1b601c7e3f5bbc1cf77bd8c8803b
 F ext/fts5/test/fts5corrupt3.test 1ccf575f5126e79f9fec7979fd02a1f40a076be3
 F ext/fts5/test/fts5dlidx.test 59b80bbe34169a082c575d9c26f0a7019a7b79c1
 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
-F ext/fts5/test/fts5ea.test 451bb37310ee6df8ef72e4354fda5621b3b51448
-F ext/fts5/test/fts5eb.test 46f49497edc25ef3b2bff9fb6d75b6d201e2b39e
+F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
+F ext/fts5/test/fts5eb.test 3e5869af2008cbc4ad03a175a0b6f6e58134cd43
 F ext/fts5/test/fts5fault1.test 7a562367cb4a735b57b410dbdb62dcc8d971faec
 F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341
 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
@@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460
-F ext/fts5/test/fts5synonym.test 6f1cfa5022bdae999f018075254e9fc51da2e618
+F ext/fts5/test/fts5synonym.test 3343a05f7b1d3fc6fa4559bd3bd0ee979928c30d
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
@@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P fc71868496f45f9c7a79ed2bf2d164a7c4718ce1
-R 8d19f3ef91012c48b3ad32684321db98
+P 98d07d16cab92f1e7001afbe370df3ec6343fc1f
+R e2a3883c4d0440bf5b53c5a71ba46260
 U dan
-Z 14116ac3039ffa0cba068c002654bc88
+Z 7c01c61f2a729597f3365ce7db7a6ce6

diff --git a/manifest.uuid b/manifest.uuid
index 3c4a7727be87240eef5f38de8bb423e9df8318d4..b0ebea4bef4d608dbae8f21d64b41bb8ec5440c1 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-98d07d16cab92f1e7001afbe370df3ec6343fc1f
\ No newline at end of file
+ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3
\ No newline at end of file