Update fts5 to support "<colset> : ( <expr> )" for column filtering, as well

as "<colset> : NEAR(...)" and "<colset> : <phrase>".

FossilOrigin-Name: c847543f8bb1376fef52bca72b4191162a32eb7e6c5f0cd1aa0ab116b3183396

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 83e5238b488f15d768bb515ea7c266333fe41585..1cd6877c9fe9150add52e107b1b9aaad65143220 100644 (file)
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -738,7 +738,7 @@ void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
 
 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
-void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
+void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
 Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);

diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index e18691ad0be2c4251f2b7c2c2532f3cb220c429a..f10d3d1043955859fae14abf09e5854235dc6559 100644 (file)
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -1886,25 +1886,110 @@ Fts5Colset *sqlite3Fts5ParseColset(
   return pRet;
 }
 
+/*
+** If argument pOrig is NULL, or if (*pRc) is set to anything other than
+** SQLITE_OK when this function is called, NULL is returned. 
+**
+** Otherwise, a copy of (*pOrig) is made into memory obtained from
+** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
+** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
+*/
+static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
+  Fts5Colset *pRet;
+  if( pOrig ){
+    int nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int);
+    pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
+    if( pRet ){ 
+      memcpy(pRet, pOrig, nByte);
+    }
+  }else{
+    pRet = 0;
+  }
+  return pRet;
+}
+
+/*
+** Remove from colset pColset any columns that are not also in colset pMerge.
+*/
+static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
+  int iIn = 0;          /* Next input in pColset */
+  int iMerge = 0;       /* Next input in pMerge */
+  int iOut = 0;         /* Next output slot in pColset */
+
+  while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
+    int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
+    if( iDiff==0 ){
+      pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
+      iMerge++;
+      iIn++;
+    }else if( iDiff>0 ){
+      iMerge++;
+    }else{
+      iIn++;
+    }
+  }
+  pColset->nCol = iOut;
+}
+
+/*
+** Recursively apply colset pColset to expression node pNode and all of
+** its decendents. If (*ppFree) is not NULL, it contains a spare copy
+** of pColset. This function may use the spare copy and set (*ppFree) to
+** zero, or it may create copies of pColset using fts5CloneColset().
+*/
+static void fts5ParseSetColset(
+  Fts5Parse *pParse, 
+  Fts5ExprNode *pNode, 
+  Fts5Colset *pColset,
+  Fts5Colset **ppFree
+){
+  if( pParse->rc==SQLITE_OK ){
+    assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING 
+         || pNode->eType==FTS5_AND  || pNode->eType==FTS5_OR
+         || pNode->eType==FTS5_NOT  || pNode->eType==FTS5_EOF
+    );
+    if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
+      Fts5ExprNearset *pNear = pNode->pNear;
+      if( pNear->pColset ){
+        fts5MergeColset(pNear->pColset, pColset);
+        if( pNear->pColset->nCol==0 ){
+          pNode->eType = FTS5_EOF;
+          pNode->xNext = 0;
+        }
+      }else if( *ppFree ){
+        pNear->pColset = pColset;
+        *ppFree = 0;
+      }else{
+        pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
+      }
+    }else{
+      int i;
+      assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 );
+      for(i=0; i<pNode->nChild; i++){
+        fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
+      }
+    }
+  }
+}
+
+/*
+** Apply colset pColset to expression node pExpr and all of its descendents.
+*/
 void sqlite3Fts5ParseSetColset(
   Fts5Parse *pParse, 
-  Fts5ExprNearset *pNear, 
+  Fts5ExprNode *pExpr, 
   Fts5Colset *pColset 
 ){
+  Fts5Colset *pFree = pColset;
   if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
     pParse->rc = SQLITE_ERROR;
     pParse->zErr = sqlite3_mprintf(
       "fts5: column queries are not supported (detail=none)"
     );
-    sqlite3_free(pColset);
-    return;
-  }
-
-  if( pNear ){
-    pNear->pColset = pColset;
   }else{
-    sqlite3_free(pColset);
+    fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
   }
+  sqlite3_free(pFree);
 }
 
 static void fts5ExprAssignXNext(Fts5ExprNode *pNode){

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 27e28a84ab2817f88abbbf29c4c77d3b69beb363..955489a35a61020c778252c6a4cd88c54470e582 100644 (file)
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -3158,23 +3158,23 @@ static int fts5IndexExtractCol(
   return p - (*pa);
 }
 
-static int fts5IndexExtractColset (
+static void fts5IndexExtractColset(
+  int *pRc,
   Fts5Colset *pColset,            /* Colset to filter on */
   const u8 *pPos, int nPos,       /* Position list */
   Fts5Buffer *pBuf                /* Output buffer */
 ){
-  int rc = SQLITE_OK;
-  int i;
-
-  fts5BufferZero(pBuf);
-  for(i=0; i<pColset->nCol; i++){
-    const u8 *pSub = pPos;
-    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
-    if( nSub ){
-      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
+  if( *pRc==SQLITE_OK ){
+    int i;
+    fts5BufferZero(pBuf);
+    for(i=0; i<pColset->nCol; i++){
+      const u8 *pSub = pPos;
+      int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
+      if( nSub ){
+        fts5BufferAppendBlob(pRc, pBuf, nSub, pSub);
+      }
     }
   }
-  return rc;
 }
 
 /*
@@ -3298,8 +3298,9 @@ static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
       pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
       pIter->base.pData = a;
     }else{
+      int *pRc = &pIter->pIndex->rc;
       fts5BufferZero(&pIter->poslist);
-      fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
+      fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, &pIter->poslist);
       pIter->base.pData = pIter->poslist.p;
       pIter->base.nData = pIter->poslist.n;
     }

diff --git a/ext/fts5/fts5parse.y b/ext/fts5/fts5parse.y
index 1cc4b88e18de6b8e564a097d789d8ca52bf63129..1582909aa863b813af48ea1d5788a9ff75c5b400 100644 (file)
--- a/ext/fts5/fts5parse.y
+++ b/ext/fts5/fts5parse.y
@@ -89,6 +89,29 @@ input ::= expr(X). { sqlite3Fts5ParseFinished(pParse, X); }
 %destructor expr     { sqlite3Fts5ParseNodeFree($$); }
 %destructor exprlist { sqlite3Fts5ParseNodeFree($$); }
 
+%type colset {Fts5Colset*}
+%destructor colset { sqlite3_free($$); }
+%type colsetlist {Fts5Colset*}
+%destructor colsetlist { sqlite3_free($$); }
+
+colset(A) ::= MINUS LCP colsetlist(X) RCP. { 
+    A = sqlite3Fts5ParseColsetInvert(pParse, X);
+}
+colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
+colset(A) ::= STRING(X). {
+  A = sqlite3Fts5ParseColset(pParse, 0, &X);
+}
+colset(A) ::= MINUS STRING(X). {
+  A = sqlite3Fts5ParseColset(pParse, 0, &X);
+  A = sqlite3Fts5ParseColsetInvert(pParse, A);
+}
+
+colsetlist(A) ::= colsetlist(Y) STRING(X). { 
+  A = sqlite3Fts5ParseColset(pParse, Y, &X); }
+colsetlist(A) ::= STRING(X). { 
+  A = sqlite3Fts5ParseColset(pParse, 0, &X); 
+}
+
 expr(A) ::= expr(X) AND expr(Y). {
   A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
 }
@@ -99,6 +122,10 @@ expr(A) ::= expr(X) NOT expr(Y). {
   A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
 }
 
+expr(A) ::= colset(X) COLON LP expr(Y) RP. {
+  sqlite3Fts5ParseSetColset(pParse, Y, X);
+  A = Y;
+}
 expr(A) ::= LP expr(X) RP. {A = X;}
 expr(A) ::= exprlist(X).   {A = X;}
 
@@ -111,32 +138,10 @@ cnearset(A) ::= nearset(X). {
   A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
 }
 cnearset(A) ::= colset(X) COLON nearset(Y). { 
-  sqlite3Fts5ParseSetColset(pParse, Y, X);
   A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
+  sqlite3Fts5ParseSetColset(pParse, A, X);
 }
 
-%type colset {Fts5Colset*}
-%destructor colset { sqlite3_free($$); }
-%type colsetlist {Fts5Colset*}
-%destructor colsetlist { sqlite3_free($$); }
-
-colset(A) ::= MINUS LCP colsetlist(X) RCP. { 
-    A = sqlite3Fts5ParseColsetInvert(pParse, X);
-}
-colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
-colset(A) ::= STRING(X). {
-  A = sqlite3Fts5ParseColset(pParse, 0, &X);
-}
-colset(A) ::= MINUS STRING(X). {
-  A = sqlite3Fts5ParseColset(pParse, 0, &X);
-  A = sqlite3Fts5ParseColsetInvert(pParse, A);
-}
-
-colsetlist(A) ::= colsetlist(Y) STRING(X). { 
-  A = sqlite3Fts5ParseColset(pParse, Y, &X); }
-colsetlist(A) ::= STRING(X). { 
-  A = sqlite3Fts5ParseColset(pParse, 0, &X); 
-}
 
 %type nearset     {Fts5ExprNearset*}
 %type nearphrases {Fts5ExprNearset*}

diff --git a/ext/fts5/test/fts5colset.test b/ext/fts5/test/fts5colset.test
index e3333241ee05aff70cdd855d1105e0100169ca0e..21918a609d77ff68374c600a56b6222037935635 100644 (file)
--- a/ext/fts5/test/fts5colset.test
+++ b/ext/fts5/test/fts5colset.test
@@ -44,13 +44,25 @@ foreach_detail_mode $::testprefix {
     9 "-c : a"  {1 2 4}
     10 "-\"c\" : a"  {1 2 4}
   } {
-  breakpoint
     do_execsql_test 1.$tn {
       SELECT rowid FROM t1($q)
     } $res
   }
 
-
+  foreach {tn q res} {
+    0 {{a} : (a AND ":")}     {}
+    1 "{a b c} : (a AND d)"   {2 3}
+    2 "{a b c} : (a AND b:d)" {3}
+    3 "{a b c} : (a AND d:d)" {}
+    4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4}
+    5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3}
+    6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {}
+    7 "{a b c} : (b:a AND c:b)" {2}
+  } {
+    do_execsql_test 2.$tn {
+      SELECT rowid FROM t1($q)
+    } $res
+  }
 }
 
 

diff --git a/ext/fts5/test/fts5faultB.test b/ext/fts5/test/fts5faultB.test
index 2ce33f4bc4207585b3d5da72b78fe92669f50c76..e30b03a13e9222b9470f952ae9643adb74dfaf64 100644 (file)
--- a/ext/fts5/test/fts5faultB.test
+++ b/ext/fts5/test/fts5faultB.test
@@ -107,5 +107,29 @@ do_faultsim_test 3.3 -faults oom* -body {
   faultsim_test_result {0 {2 3}}
 }
 
+#-------------------------------------------------------------------------
+# Test OOM injection with nested colsets.
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d);
+  INSERT INTO t1 VALUES('a', 'b', 'c', 'd');  -- 1
+  INSERT INTO t1 VALUES('d', 'a', 'b', 'c');  -- 2
+  INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
+  INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
+}
+do_faultsim_test 4.1 -faults oom* -body {
+  execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); }
+} -test {
+  faultsim_test_result {0 2}
+}
+
+do_faultsim_test 4.2 -faults oom* -body {
+  execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
+} -test {
+  faultsim_test_result {0 {2 3}}
+}
+
+
 finish_test
 

diff --git a/manifest b/manifest
index ce75a7afa6ddcb928282564d1337bb3b6519f2a7..59900f1e3b465c41a3c3c4a15ddfa5c30b3370e5 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improved\s\\n\sand\s\\r\sescapes\sin\sthe\sext/misc/dbdump.c\sutility\sfunction.\s\sThe\nimplementation\sof\sdbdump.c\snow\smatches\sthe\simplementation\sin\sthe\sCLI.
-D 2017-04-12T17:38:24.004
+C Update\sfts5\sto\ssupport\s"<colset>\s:\s(\s<expr>\s)"\sfor\scolumn\sfiltering,\sas\swell\nas\s"<colset>\s:\sNEAR(...)"\sand\s"<colset>\s:\s<phrase>".
+D 2017-04-12T17:50:12.869
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
@@ -99,13 +99,13 @@ F ext/fts3/unicode/mkunicode.tcl ab0543a3b2399092ea2dd75df1bef333405b0d7f6b8c495
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
-F ext/fts5/fts5Int.h 88c1a9a01cd5cd7b9a3c95699d1f61cdb277b2fa087a07d3fc989b49970437d9
+F ext/fts5/fts5Int.h d514c81997be588c2b23b82c77f6449e20a43903c503066f13d3d0ed8746ccde
 F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
-F ext/fts5/fts5_expr.c c6ecc2280162a3714d15dce2a8f2299f748b627c
+F ext/fts5/fts5_expr.c e5522f89e599e7e53c0014003b66480f676d2eff8b9cdb2c03c32d859b4b7cb3
 F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2
-F ext/fts5/fts5_index.c a3a9ae1c6f8a3652eb45dd732a71cdc92b48df047af134da38cd0c425649e042
+F ext/fts5/fts5_index.c 551add2b7762a2857336747def3b9d4fa304df476e188fb323492e2fd851772b
 F ext/fts5/fts5_main.c 24cafdc44c06b9665d73c34b75966f6223e9299cc9fd10184c6bf888e3ff563f
 F ext/fts5/fts5_storage.c 7750986004f3f0c94619a85ecb5dd6cbef53e5e3853488e8a906c269d4d11db6
 F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a
@@ -115,7 +115,7 @@ F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8
 F ext/fts5/fts5_unicode2.c b450b209b157d598f7b9df9f837afb75a14c24bf
 F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738
 F ext/fts5/fts5_vocab.c e44fefa7f0c1db252998af071daf06a7147e17e7
-F ext/fts5/fts5parse.y e51b375403421b8b37428a89b095d00597129aae
+F ext/fts5/fts5parse.y a070b538e08ae9e2177d15c337ed2a3464408f0f886e746307098f746efd94ca
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841
 F ext/fts5/test/fts5aa.test b3cb080db4851580705c5e261c9d4308edf030d5cbbc24b5d6668403b73f023b
@@ -136,7 +136,7 @@ F ext/fts5/test/fts5aux.test 5dd158a1e7869e27e9762a2a452b189c728d1be3
 F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e
 F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb
 F ext/fts5/test/fts5bigtok.test 017a9397b14e7598883a6328ead4a6539b42d59a
-F ext/fts5/test/fts5colset.test 1cdf56e079316005aabda790059aee86f2222ee4
+F ext/fts5/test/fts5colset.test 5d6cc35a7cdf39b2709a36620c3f398f6f8c88de39e1485f367e5294ffc7c153
 F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07
 F ext/fts5/test/fts5config.test 7788b9c058074d640dfcdd81d97b6a9480000368
 F ext/fts5/test/fts5conflict.test 26f4e46c4d31e16221794832a990dc4e30e18de5
@@ -160,7 +160,7 @@ F ext/fts5/test/fts5fault7.test cb14ea3c1f42394f06f2284abc58eecee6ff8080
 F ext/fts5/test/fts5fault8.test 6785af34bd1760de74e2824ea9c161965af78f85
 F ext/fts5/test/fts5fault9.test e10e395428a9ea0596ebe752ff7123d16ab78e08
 F ext/fts5/test/fts5faultA.test fa5d59c0ff62b7125cd14eee38ded1c46e15a7ea
-F ext/fts5/test/fts5faultB.test 7f3bba790fa172073ac314f9b8ed197390b61eca
+F ext/fts5/test/fts5faultB.test 8a7e4c89c663e599e88fa1b9f2b8dcf4b85a338bd250e1a94341773b1e9ebfcc
 F ext/fts5/test/fts5faultD.test cc5d1225556e356615e719c612e845d41bff7d5a
 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
 F ext/fts5/test/fts5fuzz1.test bece4695fc169b61ab236ada7931c6e4942cbef9
@@ -1571,7 +1571,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7aae5c0f99aa2fda85654242cfc9e23a0f981d9ce4ab17610d619cd208540b3d
-R 8af8a0ed024db0518fa75f74af560a36
-U drh
-Z 8e506040e94c8b0ee35c49d49bd8bac0
+P f2643315bb41a71eebd79f5d671f9163187e299a52ff8a481186f1e8fa7e5262
+R 0002da416f4fbc93cd5fcd1248781e80
+U dan
+Z 28e57cabc052d8f141d26e29a9907759

diff --git a/manifest.uuid b/manifest.uuid
index ceed2cbabfff3021cbeee534557ae7b74bcadf81..17ea2da68c0bb7439ecd0dc0d1e33f4403fdfca8 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f2643315bb41a71eebd79f5d671f9163187e299a52ff8a481186f1e8fa7e5262
\ No newline at end of file
+c847543f8bb1376fef52bca72b4191162a32eb7e6c5f0cd1aa0ab116b3183396
\ No newline at end of file