From: dan <dan@noemail.net>
Date: Tue, 5 Jan 2016 21:04:35 +0000 (+0000)
Subject: Rationalize and extend some fts5 test code. Fix bugs revealed by the same.
X-Git-Tag: version-3.11.0~157^2~11
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=10e1535dee1085a602c09e7847532e61057c7e4d;p=thirdparty%2Fsqlite.git

Rationalize and extend some fts5 test code. Fix bugs revealed by the same.

FossilOrigin-Name: fd217fdb396cf22278a496de0e9dec7902ba8b1f
---

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index b853bb3a43..1a3d57ca26 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -643,11 +643,12 @@ int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
 
 typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
-Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*);
+Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
 int sqlite3Fts5ExprPopulatePoslists(
     Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
 );
 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
+void sqlite3Fts5ExprClearEof(Fts5Expr*);
 
 int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
 
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index ae00a12c82..9988c557b4 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -1218,6 +1218,9 @@ static int fts5ExprNodeNextMatch(
         }
         pNode->bEof = p1->bEof;
         pNode->iRowid = p1->iRowid;
+        if( p1->bEof ){
+          fts5ExprNodeZeroPoslist(p2);
+        }
         break;
       }
     }
@@ -2260,9 +2263,10 @@ int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){
 struct Fts5PoslistPopulator {
   Fts5PoslistWriter writer;
   int bOk;                        /* True if ok to populate */
+  int bMiss;
 };
 
-Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){
+Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){
   Fts5PoslistPopulator *pRet;
   pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
   if( pRet ){
@@ -2270,8 +2274,15 @@ Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){
     memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
     for(i=0; i<pExpr->nPhrase; i++){
       Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
+      Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
       assert( pExpr->apExprPhrase[i]->nTerm==1 );
-      pBuf->n = 0;
+      if( bLive && 
+          (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof)
+      ){
+        pRet[i].bMiss = 1;
+      }else{
+        pBuf->n = 0;
+      }
     }
   }
   return pRet;
@@ -2341,8 +2352,11 @@ int sqlite3Fts5ExprPopulatePoslists(
   sCtx.iOff = (((i64)iCol) << 32) - 1;
 
   for(i=0; i<pExpr->nPhrase; i++){
-    Fts5Colset *pColset = pExpr->apExprPhrase[i]->pNode->pNear->pColset;
-    if( pColset && 0==fts5ExprColsetTest(pColset, iCol) ){
+    Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
+    Fts5Colset *pColset = pNode->pNear->pColset;
+    if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) 
+     || aPopulator[i].bMiss
+    ){
       aPopulator[i].bOk = 0;
     }else{
       aPopulator[i].bOk = 1;
@@ -2412,10 +2426,22 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
     }
   }
 }
+
 void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
   fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
 }
 
+static void fts5ExprClearEof(Fts5ExprNode *pNode){
+  int i;
+  for(i=0; i<pNode->nChild; i++){
+    fts5ExprClearEof(pNode->apChild[i]);
+  }
+  pNode->bEof = 0;
+}
+void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
+  fts5ExprClearEof(pExpr->pRoot);
+}
+
 /*
 ** This function is only called for detail=columns tables. 
 */
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 99c9eb05db..7f6f6dacef 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1179,6 +1179,7 @@ static int fts5FilterMethod(
     pCsr->ePlan = FTS5_PLAN_SOURCE;
     pCsr->pExpr = pTab->pSortCsr->pExpr;
     rc = fts5CursorFirst(pTab, pCsr, bDesc);
+    sqlite3Fts5ExprClearEof(pCsr->pExpr);
   }else if( pMatch ){
     const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
     if( zExpr==0 ) zExpr = "";
@@ -1688,13 +1689,14 @@ static int fts5CsrPoslist(
 ){
   Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
   int rc = SQLITE_OK;
+  int bLive = (pCsr->pSorter==0);
 
   if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
 
     if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
       Fts5PoslistPopulator *aPopulator;
       int i;
-      aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr);
+      aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive);
       if( aPopulator==0 ) rc = SQLITE_NOMEM;
       for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
         int n; const char *z;
@@ -1706,6 +1708,7 @@ static int fts5CsrPoslist(
         }
       }
       sqlite3_free(aPopulator);
+
       if( pCsr->pSorter ){
         sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid);
       }
diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl
index 3bcd33dcfc..aadf148716 100644
--- a/ext/fts5/test/fts5_common.tcl
+++ b/ext/fts5/test/fts5_common.tcl
@@ -390,5 +390,118 @@ proc detail_is_col {}  { detail_check ; expr {$::detail == "col" } }
 proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }
 
 
+#-------------------------------------------------------------------------
+# Convert a poslist of the type returned by fts5_test_poslist() to a 
+# collist as returned by fts5_test_collist().
+#
+proc fts5_poslist2collist {poslist} {
+  set res [list]
+  foreach h $poslist {
+    regexp {(.*)\.[1234567890]+} $h -> cand
+    lappend res $cand
+  }
+  set res [lsort -command fts5_collist_elem_compare -unique $res]
+  return $res
+}
+
+# Comparison function used by fts5_poslist2collist to sort collist entries.
+#
+proc fts5_collist_elem_compare {a b} {
+  foreach {a1 a2} [split $a .] {}
+  foreach {b1 b2} [split $b .] {}
+
+  if {$a1==$b1} { return [expr $a2 - $b2] }
+  return [expr $a1 - $b1]
+}
+
+
+#--------------------------------------------------------------------------
+# Construct and return a tcl list equivalent to that returned by the SQL
+# query executed against database handle [db]:
+#
+#   SELECT 
+#     rowid, 
+#     fts5_test_poslist($tbl),
+#     fts5_test_collist($tbl) 
+#   FROM $tbl('$expr')
+#   ORDER BY rowid $order;
+#
+proc fts5_query_data {expr tbl {order ASC}} {
+
+  # Figure out the set of columns in the FTS5 table. This routine does
+  # not handle tables with UNINDEXED columns, but if it did, it would
+  # have to be here.
+  db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
+
+  set cols ""
+  foreach e $lCols { append cols ", '$e'" }
+  set tclexpr [db one [subst -novar {
+    SELECT fts5_expr_tcl( $expr, 'nearset $cols -pc ::pc' [set cols] )
+  }]]
+
+  set res [list]
+  db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
+    set cols [list]
+    foreach col $lCols { lappend cols $x($col) }
+    
+    set ::pc 0
+    set rowdata [eval $tclexpr]
+    if {$rowdata != ""} { 
+      lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata]
+    }
+  }
+
+  set res
+}
+
+#-------------------------------------------------------------------------
+# Similar to [fts5_query_data], but omit the collist field.
+#
+proc fts5_poslist_data {expr tbl {order ASC}} {
+  set res [list]
+  foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order] {
+    lappend res $rowid $poslist
+  }
+  set res
+}
+
+#-------------------------------------------------------------------------
+#
+proc nearset_rf {aCol args} {
+  set idx [lsearch -exact $args --]
+  if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} {
+    set ::expr_not_ok 1
+  }
+  list
+}
+
+proc nearset_rc {aCol args} {
+  nearset_rf $aCol {*}$args
+  if {[lsearch $args -col]>=0} { 
+    set ::expr_not_ok 1
+  }
+  list
+}
+
+proc fts5_expr_ok {expr tbl} {
 
+  if {![detail_is_full]} {
+    set nearset "nearset_rc"
+    if {[detail_is_col]} { set nearset "nearset_rf" }
+
+    set ::expr_not_ok 0
+    db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
+
+    set cols ""
+    foreach e $lCols { append cols ", '$e'" }
+    set ::pc 0
+    set tclexpr [db one [subst -novar {
+      SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] )
+    }]]
+    eval $tclexpr
+    if {$::expr_not_ok} { return 0 }
+  }
+
+  return 1
+}
 
diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test
index 692929167b..f53f5a4622 100644
--- a/ext/fts5/test/fts5ac.test
+++ b/ext/fts5/test/fts5ac.test
@@ -211,6 +211,7 @@ foreach {tn2 sql} {
   1  {}
   2  {BEGIN}
 } {
+if {[detail_is_full]} continue
   reset_db
   fts5_aux_test_functions db
 
@@ -228,192 +229,115 @@ foreach {tn2 sql} {
     execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
   } {}
 
-  #-------------------------------------------------------------------------
-  # The following work with all detail= modes.
-  #
-  foreach {tn expr} {
-    1 "a   AND b"
-    2 "a   OR b"
-    3 "o"
-    4 "b q"
-    5 "e a e"
-    6 "m d g q q b k b w f q q p p"
-    7 "l o o l v v k"
-    8 "a"
-    9 "b"
-    10 "c"
-    11 "no"
-    12 "L O O L V V K"
-    13 "a AND b AND c"
-  } {
-    set res [poslist_data 1 $expr]
-    do_execsql_test 1.$tn2.1.$tn.p.[llength $res] { 
-      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-    } $res
-
-    set res [collist_data $expr]
-    do_execsql_test 1.$tn2.1.$tn.c.[llength $res] { 
-      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
-    } $res
-  }
-
-  if {[detail_is_none]} continue
 
   #-------------------------------------------------------------------------
-  # Queries on a specific column.
   #
-  foreach {tn expr} {
-    1.1 "x:a"
-    1.2 "y:a"
-    1.3 "x:b"
-    1.4 "y:b"
-    2.1 "{x}:a"
-    2.2 "{y}:a"
-    2.3 "{x}:b"
-    2.4 "{y}:b"
-
-    3.1 "{x y}:a"
-    3.2 "{y x}:a"
-    3.3 "{x x}:b"
-    3.4 "{y y}:b"
-
-    4.1 {{"x" "y"}:a}
-    4.2 {{"y" x}:a}
-    4.3 {{x "x"}:b}
-    4.4 {{"y" y}:b}
-  } {
-    set res [poslist_data 1 $expr]
-    do_execsql_test 1.$tn2.4.$tn.p.[llength $res] { 
-      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-    } $res
-
-    set res [collist_data $expr]
-    do_execsql_test 1.$tn2.4.$tn.c.[llength $res] { 
-      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
-    } $res
+  do_execsql_test 1.$tn2.integrity {
+    INSERT INTO xx(xx) VALUES('integrity-check');
   }
 
-  if {[detail_is_col]} continue
-
   #-------------------------------------------------------------------------
-  # Test phrase queries.
   #
   foreach {tn expr} {
-    1 "b + q"
-    2 "e + a + e"
-    3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p"
-    4 "l + o + o + l + v + v + k"
-    5 "L + O + O + L + V + V + K"
+    1.2 "a   OR b"
+    1.1 "a   AND b"
+    1.3 "o"
+    1.4 "b q"
+    1.5 "e a e"
+    1.6 "m d g q q b k b w f q q p p"
+    1.7 "l o o l v v k"
+    1.8 "a"
+    1.9 "b"
+    1.10 "c"
+    1.11 "no"
+    1.12 "L O O L V V K"
+    1.13 "a AND b AND c"
+    1.14 "x:a"
+
+    2.1 "x:a"
+    2.2 "y:a"
+    2.3 "x:b"
+    2.4 "y:b"
+
+    3.1 "{x}:a"
+    3.2 "{y}:a"
+    3.3 "{x}:b"
+    3.4 "{y}:b"
+
+    4.1 "{x y}:a"
+    4.2 "{y x}:a"
+    4.3 "{x x}:b"
+    4.4 "{y y}:b"
+
+    5.1 {{"x" "y"}:a}
+    5.2 {{"y" x}:a}
+    5.3 {{x "x"}:b}
+    5.4 {{"y" y}:b}
+
+    6.1 "b + q"
+    6.2 "e + a + e"
+    6.3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p"
+    6.4 "l + o + o + l + v + v + k"
+    6.5 "L + O + O + L + V + V + K"
+
+    7.1 "a+b AND c"
+    7.2 "d+c AND u"
+    7.3 "d+c AND u+d"
+    7.4 "a+b OR c"
+    7.5 "d+c OR u"
+    7.6 "d+c OR u+d"
+
+    8.1 "NEAR(a b)"
+    8.2 "NEAR(r c)"
+    8.2 { NEAR(r c, 5) }
+    8.3 { NEAR(r c, 3) }
+    8.4 { NEAR(r c, 2) }
+    8.5 { NEAR(r c, 0) }
+    8.6 { NEAR(a b c) }
+    8.7 { NEAR(a b c, 8) }
+    8.8  { x : NEAR(r c) }
+    8.9  { y : NEAR(r c) }
+
+    9.1 { NEAR(r c) }
+    9.2 { NEAR(r c, 5) }
+    9.3 { NEAR(r c, 3) }
+    9.4 { NEAR(r c, 2) }
+    9.5 { NEAR(r c, 0) }
+    9.6 { NEAR(a b c) }
+    9.7 { NEAR(a b c, 8) }
+    9.8  { x : NEAR(r c) }
+    9.9  { y : NEAR(r c) }
+    9.10 { x : "r c" }
+    9.11 { y : "r c" }
+    9.12 { a AND b }
+    9.13 { a AND b AND c }
+    9.14a { a }
+    9.14b { a OR b }
+    9.15 { a OR b AND c }
+    9.16 { c AND b OR a }
+    9.17 { c AND (b OR a) }
+    9.18 { c NOT (b OR a) }
+    9.19 { (c NOT b) OR (a AND d) }
   } {
-    set res [poslist_data 1 $expr]
-
-    do_execsql_test 1.$tn2.2.$tn.p.[llength $res] { 
-      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-    } $res
-
-    set res [collist_data $expr]
-    do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { 
-      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
-    } $res
-  }
-
-  #-------------------------------------------------------------------------
-  # Test some AND and OR queries.
-  #
-  foreach {tn expr} {
-    1.1 "a+b AND c"
-    1.2 "d+c AND u"
-    1.3 "d+c AND u+d"
 
-    2.1 "a+b OR c"
-    2.2 "d+c OR u"
-    2.3 "d+c OR u+d"
-  } {
-    set res [poslist_data 1 $expr]
-    do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { 
-      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-    } $res
+    if {[fts5_expr_ok $expr xx]==0} {
+      do_test 1.$tn2.$tn.OMITTED { list } [list]
+      continue
+    }
 
-    set res [collist_data $expr]
-    do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { 
-      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
+    set res [fts5_query_data $expr xx]
+    do_execsql_test 1.$tn2.$tn.[llength $res].asc {
+      SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
+      FROM xx WHERE xx match $expr
     } $res
-  }
 
-  #-------------------------------------------------------------------------
-  # Some NEAR queries.
-  #
-  foreach {tn expr} {
-    1 "NEAR(a b)"
-    2 "NEAR(r c)"
-    2 { NEAR(r c, 5) }
-    3 { NEAR(r c, 3) }
-    4 { NEAR(r c, 2) }
-    5 { NEAR(r c, 0) }
-    6 { NEAR(a b c) }
-    7 { NEAR(a b c, 8) }
-    8  { x : NEAR(r c) }
-    9  { y : NEAR(r c) }
-  } {
-    set res [poslist_data 1 $expr]
-    do_execsql_test 1.$tn2.5.1.$tn.p.[llength $res] { 
-      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-    } $res
 
-    set res [collist_data $expr]
-    do_execsql_test 1.$tn2.5.1.$tn.c.[llength $res] { 
-      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
+    set res [fts5_query_data $expr xx DESC]
+    do_execsql_test 1.$tn2.$tn.[llength $res].desc {
+      SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
+      FROM xx WHERE xx match $expr ORDER BY 1 DESC
     } $res
   }
-
-  do_test $tn2.4.1  { nearset {{a b c}} -- a } {0.0.0}
-  do_test $tn2.4.2  { nearset {{a b c}} -- c } {0.0.2}
-
-  foreach {tn expr tclexpr} {
-    1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
-  } {
-    do_execsql_test 1.$tn2.6.$tn {
-      SELECT fts5_expr_tcl($expr, 'N $x')
-    } [list $tclexpr]
-  }
-
-  #-------------------------------------------------------------------------
-  #
-  do_execsql_test 1.$tn2.7.integrity {
-    INSERT INTO xx(xx) VALUES('integrity-check');
-  }
-  #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r}
-  foreach {bAsc sql} {
-    1 {SELECT rowid FROM xx WHERE xx MATCH $expr}
-    0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC}
-  } {
-    foreach {tn expr} {
-      0.1 x
-      1 { NEAR(r c) }
-      2 { NEAR(r c, 5) }
-      3 { NEAR(r c, 3) }
-      4 { NEAR(r c, 2) }
-      5 { NEAR(r c, 0) }
-      6 { NEAR(a b c) }
-      7 { NEAR(a b c, 8) }
-      8  { x : NEAR(r c) }
-      9  { y : NEAR(r c) }
-      10 { x : "r c" }
-      11 { y : "r c" }
-      12 { a AND b }
-      13 { a AND b AND c }
-      14a { a }
-      14b { a OR b }
-      15 { a OR b AND c }
-      16 { c AND b OR a }
-      17 { c AND (b OR a) }
-      18 { c NOT (b OR a) }
-      19 { c NOT b OR a AND d }
-    } {
-      set res [poslist_data 0 $expr $bAsc]
-      do_execsql_test 1.$tn2.7.$bAsc.$tn.[llength $res] $sql $res
-    }
-  }
 }
 
 }
@@ -422,7 +346,16 @@ do_execsql_test 2.1 {
   SELECT fts5_expr_tcl('a AND b');
 } {{AND [nearset -- {a}] [nearset -- {b}]}}
 
-finish_test
+do_test 2.2.1  { nearset {{a b c}} -- a } {0.0.0}
+do_test 2.2.2  { nearset {{a b c}} -- c } {0.0.2}
+
+foreach {tn expr tclexpr} {
+  1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
+} {
+  do_execsql_test 2.3.$tn {
+    SELECT fts5_expr_tcl($expr, 'N $x')
+  } [list $tclexpr]
+}
 
 finish_test
 
diff --git a/ext/fts5/test/fts5auto.test b/ext/fts5/test/fts5auto.test
index 6efaa9cb7c..218b3f4862 100644
--- a/ext/fts5/test/fts5auto.test
+++ b/ext/fts5/test/fts5auto.test
@@ -22,7 +22,6 @@ ifcapable !fts5 {
   return
 }
 
-
 set data {
     -4026076
     {n x w k b p x b n t t d s}     {f j j s p j o}               
@@ -232,37 +231,9 @@ do_execsql_test 1.0 {
 
 fts5_aux_test_functions db
 
-proc matchdata {expr tbl collist {order ASC}} {
-
-  set cols ""
-  foreach e $collist {
-    append cols ", '$e'"
-  }
-
-  set tclexpr [db one [subst -novar {
-    SELECT fts5_expr_tcl(
-      $expr, 'nearset $cols -pc ::pc' [set cols]
-    )
-  }]]
-  set res [list]
-
-  db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
-    set cols [list]
-    foreach col $x(*) {
-      if {$col != "rowid"} { lappend cols $x($col) }
-    }
-    # set cols [list $a $b $c $d $e $f]
-    set ::pc 0
-    set rowdata [eval $tclexpr]
-    if {$rowdata != ""} { lappend res $x(rowid) $rowdata }
-  }
-
-  set res
-}
-
-proc do_auto_test {tn tbl cols expr} { 
+proc do_auto_test {tn tbl expr} {
   foreach order {asc desc} {
-    set res [matchdata $expr $tbl $cols $order]
+    set res [fts5_poslist_data $expr $tbl $order]
     set testname "$tn.[string range $order 0 0].rows=[expr [llength $res]/2]"
 
     set ::autotest_expr $expr
@@ -271,8 +242,6 @@ proc do_auto_test {tn tbl cols expr} {
       WHERE [set tbl] MATCH $::autotest_expr ORDER BY rowid [set order]
     }] $res
   }
-
-
 }
 
 #-------------------------------------------------------------------------
@@ -332,7 +301,7 @@ for {set fold 0} {$fold < 3} {incr fold} {
     C.1 { a OR (b AND "b c") }
     C.2 { a OR (b AND "z c") }
   } {
-    do_auto_test 3.$fold.$tn tt {a b c d e f} $expr
+    do_auto_test 3.$fold.$tn tt $expr
   }
 }
 
@@ -366,10 +335,8 @@ foreach {tn expr} {
   4 {c1 : x} 5 {c2 : x} 6 {c3 : x}
   7 {c1 : y} 8 {c2 : y} 9 {c3 : y}
   10 {c1 : z} 11 {c2 : z} 12 {c3 : z}
-
-
 } {
-  do_auto_test 4.$tn yy {c1 c2 c3} $expr
+  do_auto_test 4.$tn yy $expr
 }
 
 
diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test
index 870a912bb1..e29c13117c 100644
--- a/ext/fts5/test/fts5simple.test
+++ b/ext/fts5/test/fts5simple.test
@@ -19,6 +19,8 @@ ifcapable !fts5 {
   return
 }
 
+ if 1 {
+
 #-------------------------------------------------------------------------
 #
 set doc "x x [string repeat {y } 50]z z"
@@ -361,5 +363,24 @@ do_execsql_test 15.1 {
   INSERT INTO x2(x2) VALUES('integrity-check');
 }
 
+}
+
+#-------------------------------------------------------------------------
+foreach_detail_mode $testprefix {
+  reset_db
+  fts5_aux_test_functions db
+  do_execsql_test 16.0 {
+    CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%);
+    INSERT INTO x3 VALUES('a b c d e f');
+  }
+  do_execsql_test 16.1 {
+    SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c');
+  } {2.0.2}
+
+  do_execsql_test 16.1 {
+    SELECT fts5_test_poslist(x3) FROM x3('a OR c');
+  } {{0.0.0 1.0.2}}
+}
+
 finish_test
 
diff --git a/manifest b/manifest
index 25592e2931..0ccb6b20cc 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sOOM\shandling\sproblem\sin\sfts5\sdetail=none\sand\sdetail=col\smodes.\sAlso\sa\sbug\sin\sthe\sxInst()\sAPI\swhen\sused\swith\sdetail=col\sand\scolumn\sfiltering\sexpressions.\sUpdate\sthe\smatchinfo()\stest\sfunction\sso\sthat\s'b'\sis\sfast\swith\sdetail=col\stables.
-D 2016-01-04T19:12:00.230
+C Rationalize\sand\sextend\ssome\sfts5\stest\scode.\sFix\sbugs\srevealed\sby\sthe\ssame.
+D 2016-01-05T21:04:35.294
 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751
@@ -97,14 +97,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 88fec577a2148f34df75930dc2b0c45b0bad72c3
-F ext/fts5/fts5Int.h 10608c346cccf7dd1da4d6b46f7921949072ed60
+F ext/fts5/fts5Int.h 3918045ebceb1d600c9c1c1b460489ff0c788e96
 F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35
 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1
 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6
-F ext/fts5/fts5_expr.c 3fe372518c0191230971321048e10930952dad5b
+F ext/fts5/fts5_expr.c 1aa0f5d597a22789172efe138659271fccb3cf19
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
 F ext/fts5/fts5_index.c 8665393f8be3ae9ed4f1033d7ce0b805108fb95c
-F ext/fts5/fts5_main.c 15c8b702e28d032224a6fc2b6a9c03ba2e4deeb1
+F ext/fts5/fts5_main.c cd5f6f0bf868a19e8f8ef01c7b4b86dd152f0bc7
 F ext/fts5/fts5_storage.c 771dd0fda3ee513e32937a386dc2a4aa2d9ea64e
 F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb
 F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070
@@ -114,10 +114,10 @@ F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
 F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2
 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl fefb4ceb27d02b431f69045a7ac0061a80d97824
+F ext/fts5/test/fts5_common.tcl c9169fe40bf751e1b311271df31aec15732b26c0
 F ext/fts5/test/fts5aa.test 4abbc14eb2d9d6d46a53ab008151512871d54ebc
 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
-F ext/fts5/test/fts5ac.test 30707d2972e5f485f7bdef019f3041782edabfb3
+F ext/fts5/test/fts5ac.test 63f669ad7381b0c29556da6de41eefdbb007e810
 F ext/fts5/test/fts5ad.test 049f7511a79c155d2d8dfd2ddcfeb640c50ad0dc
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c
@@ -128,7 +128,7 @@ F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8
 F ext/fts5/test/fts5ak.test fb26389985407826f6076bb9f382c67d3db6b5d9
 F ext/fts5/test/fts5al.test 18c277f5986df0a3d9071dfd7128afeb16fe9d5d
 F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04
-F ext/fts5/test/fts5auto.test 2a6241673657b340427f521528f7809ddaa02a9e
+F ext/fts5/test/fts5auto.test 401c20e89f1114d733b94809be1e6f893e16c09e
 F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f
 F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e
 F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb
@@ -170,7 +170,7 @@ F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821
-F ext/fts5/test/fts5simple.test a599b7577bc3827a9a678add3b43d8b818b93456
+F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9
 F ext/fts5/test/fts5simple2.test d453e3ca100e08078a46b12547417891b326240a
 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
@@ -1408,7 +1408,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 3fcf3b1e24483b2cd1c1710f053ed8403e09106d
-R 133d47de041747f0d7e918f07a474874
+P ee38f9dff5e9239c541515cd8a1aa3d81fdc0ae9
+R 6eca7a3ddda49a17d0ee2479d97ee402
 U dan
-Z 4f94b859477e06c41fefb0b71cc6e55e
+Z 551e607b90efc1e18e4dc205bbcdae9f
diff --git a/manifest.uuid b/manifest.uuid
index 1f4ad7217a..49ed937a10 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ee38f9dff5e9239c541515cd8a1aa3d81fdc0ae9
\ No newline at end of file
+fd217fdb396cf22278a496de0e9dec7902ba8b1f
\ No newline at end of file