From: dan <dan@noemail.net>
Date: Mon, 4 Jan 2016 19:12:00 +0000 (+0000)
Subject: Fix an OOM handling problem in fts5 detail=none and detail=col modes. Also a bug... 
X-Git-Tag: version-3.11.0~157^2~12
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=211bd3a20a50305837d48374d1ced88710511d52;p=thirdparty%2Fsqlite.git

Fix an OOM handling problem in fts5 detail=none and detail=col modes. Also a bug in the xInst() API when used with detail=col and column filtering expressions. Update the matchinfo() test function so that 'b' is fast with detail=col tables.

FossilOrigin-Name: ee38f9dff5e9239c541515cd8a1aa3d81fdc0ae9
---

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index c369687042..b853bb3a43 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -642,9 +642,10 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
 
-Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr*);
+typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
+Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*);
 int sqlite3Fts5ExprPopulatePoslists(
-    Fts5Config*, Fts5Expr*, Fts5PoslistWriter*, int, const char*, int
+    Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
 );
 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
 
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 0a74c7884a..ae00a12c82 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -2257,28 +2257,44 @@ int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){
   return nRet;
 }
 
-Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){
-  int i;
-  Fts5PoslistWriter *pRet;
-  for(i=0; i<pExpr->nPhrase; i++){
-    Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
-    assert( pExpr->apExprPhrase[i]->nTerm==1 );
-    pBuf->n = 0;
-  }
-  pRet = sqlite3_malloc(sizeof(Fts5PoslistWriter)*pExpr->nPhrase);
+struct Fts5PoslistPopulator {
+  Fts5PoslistWriter writer;
+  int bOk;                        /* True if ok to populate */
+};
+
+Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){
+  Fts5PoslistPopulator *pRet;
+  pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
   if( pRet ){
-    memset(pRet, 0, sizeof(Fts5PoslistWriter)*pExpr->nPhrase);
+    int i;
+    memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
+    for(i=0; i<pExpr->nPhrase; i++){
+      Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
+      assert( pExpr->apExprPhrase[i]->nTerm==1 );
+      pBuf->n = 0;
+    }
   }
   return pRet;
 }
 
 struct Fts5ExprCtx {
   Fts5Expr *pExpr;
-  Fts5PoslistWriter *aWriter;
+  Fts5PoslistPopulator *aPopulator;
   i64 iOff;
 };
 typedef struct Fts5ExprCtx Fts5ExprCtx;
 
+/*
+** TODO: Make this more efficient!
+*/
+static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
+  int i;
+  for(i=0; i<pColset->nCol; i++){
+    if( pColset->aiCol[i]==iCol ) return 1;
+  }
+  return 0;
+}
+
 static int fts5ExprPopulatePoslistsCb(
   void *pCtx,                /* Copy of 2nd argument to xTokenize() */
   int tflags,                /* Mask of FTS5_TOKEN_* flags */
@@ -2294,13 +2310,14 @@ static int fts5ExprPopulatePoslistsCb(
   if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
   for(i=0; i<pExpr->nPhrase; i++){
     Fts5ExprTerm *pTerm;
+    if( p->aPopulator[i].bOk==0 ) continue;
     for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
       int nTerm = strlen(pTerm->zTerm);
       if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
        && memcmp(pTerm->zTerm, pToken, nTerm)==0
       ){
         int rc = sqlite3Fts5PoslistWriterAppend(
-            &pExpr->apExprPhrase[i]->poslist, &p->aWriter[i], p->iOff
+            &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
         );
         if( rc ) return rc;
         break;
@@ -2313,15 +2330,25 @@ static int fts5ExprPopulatePoslistsCb(
 int sqlite3Fts5ExprPopulatePoslists(
   Fts5Config *pConfig,
   Fts5Expr *pExpr, 
-  Fts5PoslistWriter *aWriter,
+  Fts5PoslistPopulator *aPopulator,
   int iCol, 
   const char *z, int n
 ){
+  int i;
   Fts5ExprCtx sCtx;
   sCtx.pExpr = pExpr;
-  sCtx.aWriter = aWriter;
+  sCtx.aPopulator = aPopulator;
   sCtx.iOff = (((i64)iCol) << 32) - 1;
 
+  for(i=0; i<pExpr->nPhrase; i++){
+    Fts5Colset *pColset = pExpr->apExprPhrase[i]->pNode->pNear->pColset;
+    if( pColset && 0==fts5ExprColsetTest(pColset, iCol) ){
+      aPopulator[i].bOk = 0;
+    }else{
+      aPopulator[i].bOk = 1;
+    }
+  }
+
   return sqlite3Fts5Tokenize(pConfig, 
       FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
   );
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 95d72ebbc1..99c9eb05db 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1608,7 +1608,7 @@ static int fts5RollbackMethod(sqlite3_vtab *pVtab){
   return rc;
 }
 
-static int fts5CsrPoslist(Fts5Cursor*, int, const u8**);
+static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*);
 
 static void *fts5ApiUserData(Fts5Context *pCtx){
   Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
@@ -1680,28 +1680,32 @@ static int fts5ApiColumnText(
   return rc;
 }
 
-static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){
+static int fts5CsrPoslist(
+  Fts5Cursor *pCsr, 
+  int iPhrase, 
+  const u8 **pa,
+  int *pn
+){
   Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
-  int n;
   int rc = SQLITE_OK;
 
   if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
 
     if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
-      Fts5PoslistWriter *aWriter;
+      Fts5PoslistPopulator *aPopulator;
       int i;
-      aWriter = sqlite3Fts5ExprClearPoslists(pCsr->pExpr);
-      if( aWriter==0 ) rc = SQLITE_NOMEM;
+      aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr);
+      if( aPopulator==0 ) rc = SQLITE_NOMEM;
       for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
         int n; const char *z;
         rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n);
         if( rc==SQLITE_OK ){
           rc = sqlite3Fts5ExprPopulatePoslists(
-              pConfig, pCsr->pExpr, aWriter, i, z, n
+              pConfig, pCsr->pExpr, aPopulator, i, z, n
           );
         }
       }
-      sqlite3_free(aWriter);
+      sqlite3_free(aPopulator);
       if( pCsr->pSorter ){
         sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid);
       }
@@ -1712,12 +1716,13 @@ static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){
   if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){
     Fts5Sorter *pSorter = pCsr->pSorter;
     int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
-    n = pSorter->aIdx[iPhrase] - i1;
+    *pn = pSorter->aIdx[iPhrase] - i1;
     *pa = &pSorter->aPoslist[i1];
   }else{
-    n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
+    *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
   }
-  return n;
+
+  return rc;
 }
 
 /*
@@ -1742,43 +1747,46 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){
     int i;
 
     /* Initialize all iterators */
-    for(i=0; i<nIter; i++){
+    for(i=0; i<nIter && rc==SQLITE_OK; i++){
       const u8 *a;
-      int n = fts5CsrPoslist(pCsr, i, &a);
+      int n; 
+      rc = fts5CsrPoslist(pCsr, i, &a, &n);
       sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
     }
 
-    while( 1 ){
-      int *aInst;
-      int iBest = -1;
-      for(i=0; i<nIter; i++){
-        if( (aIter[i].bEof==0) 
-         && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) 
-        ){
-          iBest = i;
+    if( rc==SQLITE_OK ){
+      while( 1 ){
+        int *aInst;
+        int iBest = -1;
+        for(i=0; i<nIter; i++){
+          if( (aIter[i].bEof==0) 
+              && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) 
+            ){
+            iBest = i;
+          }
         }
-      }
-      if( iBest<0 ) break;
-
-      nInst++;
-      if( nInst>=pCsr->nInstAlloc ){
-        pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32;
-        aInst = (int*)sqlite3_realloc(
-            pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3
-        );
-        if( aInst ){
-          pCsr->aInst = aInst;
-        }else{
-          rc = SQLITE_NOMEM;
-          break;
+        if( iBest<0 ) break;
+
+        nInst++;
+        if( nInst>=pCsr->nInstAlloc ){
+          pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32;
+          aInst = (int*)sqlite3_realloc(
+              pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3
+              );
+          if( aInst ){
+            pCsr->aInst = aInst;
+          }else{
+            rc = SQLITE_NOMEM;
+            break;
+          }
         }
-      }
 
-      aInst = &pCsr->aInst[3 * (nInst-1)];
-      aInst[0] = iBest;
-      aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos);
-      aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos);
-      sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
+        aInst = &pCsr->aInst[3 * (nInst-1)];
+        aInst[0] = iBest;
+        aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos);
+        aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos);
+        sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
+      }
     }
 
     pCsr->nInstCount = nInst;
@@ -1981,11 +1989,15 @@ static int fts5ApiPhraseFirst(
   int *piCol, int *piOff
 ){
   Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
-  int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a);
-  pIter->b = &pIter->a[n];
-  *piCol = 0;
-  *piOff = 0;
-  fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
+  int n;
+  int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
+  if( rc==SQLITE_OK ){
+    pIter->b = &pIter->a[n];
+    *piCol = 0;
+    *piOff = 0;
+    fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
+  }
+  return rc;
 }
 
 static void fts5ApiPhraseNextColumn(
@@ -2037,14 +2049,17 @@ static int fts5ApiPhraseFirstColumn(
       fts5ApiPhraseNextColumn(pCtx, pIter, piCol);
     }
   }else{
-    int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a);
-    pIter->b = &pIter->a[n];
-    if( n<=0 ){
-      *piCol = -1;
-    }else if( pIter->a[0]==0x01 ){
-      pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol);
-    }else{
-      *piCol = 0;
+    int n;
+    rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
+    if( rc==SQLITE_OK ){
+      pIter->b = &pIter->a[n];
+      if( n<=0 ){
+        *piCol = -1;
+      }else if( pIter->a[0]==0x01 ){
+        pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol);
+      }else{
+        *piCol = 0;
+      }
     }
   }
 
diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c
index fb4a2102b4..28331773c0 100644
--- a/ext/fts5/fts5_test_mi.c
+++ b/ext/fts5/fts5_test_mi.c
@@ -211,18 +211,31 @@ static int fts5MatchinfoLocalCb(
   int rc = SQLITE_OK;
 
   switch( f ){
-    case 'b': 
+    case 'b': {
+      int iPhrase;
+      int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
+      for(i=0; i<nInt; i++) aOut[i] = 0;
+
+      for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
+        Fts5PhraseIter iter;
+        int iCol;
+        for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
+            iCol>=0; 
+            pApi->xPhraseNextColumn(pFts, &iter, &iCol)
+        ){
+          aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
+        }
+      }
+
+      break;
+    }
+
     case 'x':
     case 'y': {
       int nMul = (f=='x' ? 3 : 1);
       int iPhrase;
 
-      if( f=='b' ){
-        int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
-        for(i=0; i<nInt; i++) aOut[i] = 0;
-      }else{
-        for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
-      }
+      for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
 
       for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
         Fts5PhraseIter iter;
diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test
index c3163147cb..692929167b 100644
--- a/ext/fts5/test/fts5ac.test
+++ b/ext/fts5/test/fts5ac.test
@@ -257,7 +257,43 @@ foreach {tn2 sql} {
     } $res
   }
 
-  if {[detail_is_none] || [detail_is_col]} continue
+  if {[detail_is_none]} continue
+
+  #-------------------------------------------------------------------------
+  # Queries on a specific column.
+  #
+  foreach {tn expr} {
+    1.1 "x:a"
+    1.2 "y:a"
+    1.3 "x:b"
+    1.4 "y:b"
+    2.1 "{x}:a"
+    2.2 "{y}:a"
+    2.3 "{x}:b"
+    2.4 "{y}:b"
+
+    3.1 "{x y}:a"
+    3.2 "{y x}:a"
+    3.3 "{x x}:b"
+    3.4 "{y y}:b"
+
+    4.1 {{"x" "y"}:a}
+    4.2 {{"y" x}:a}
+    4.3 {{x "x"}:b}
+    4.4 {{"y" y}:b}
+  } {
+    set res [poslist_data 1 $expr]
+    do_execsql_test 1.$tn2.4.$tn.p.[llength $res] { 
+      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
+    } $res
+
+    set res [collist_data $expr]
+    do_execsql_test 1.$tn2.4.$tn.c.[llength $res] { 
+      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
+    } $res
+  }
+
+  if {[detail_is_col]} continue
 
   #-------------------------------------------------------------------------
   # Test phrase queries.
@@ -304,40 +340,6 @@ foreach {tn2 sql} {
     } $res
   }
 
-  #-------------------------------------------------------------------------
-  # Queries on a specific column.
-  #
-  foreach {tn expr} {
-    1.1 "x:a"
-    1.2 "y:a"
-    1.3 "x:b"
-    1.4 "y:b"
-    2.1 "{x}:a"
-    2.2 "{y}:a"
-    2.3 "{x}:b"
-    2.4 "{y}:b"
-
-    3.1 "{x y}:a"
-    3.2 "{y x}:a"
-    3.3 "{x x}:b"
-    3.4 "{y y}:b"
-
-    4.1 {{"x" "y"}:a}
-    4.2 {{"y" x}:a}
-    4.3 {{x "x"}:b}
-    4.4 {{"y" y}:b}
-  } {
-    set res [poslist_data 1 $expr]
-    do_execsql_test 1.$tn2.4.$tn.p.[llength $res] { 
-      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-    } $res
-
-    set res [collist_data $expr]
-    do_execsql_test 1.$tn2.4.$tn.c.[llength $res] { 
-      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
-    } $res
-  }
-
   #-------------------------------------------------------------------------
   # Some NEAR queries.
   #
diff --git a/ext/fts5/test/fts5fault8.test b/ext/fts5/test/fts5fault8.test
new file mode 100644
index 0000000000..01a1876955
--- /dev/null
+++ b/ext/fts5/test/fts5fault8.test
@@ -0,0 +1,49 @@
+# 2015 September 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5fault8
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+
+if {[detail_is_none]==0} continue
+
+fts5_aux_test_functions db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
+  INSERT INTO t1 VALUES('a b c d', '1 2 3 4');
+  INSERT INTO t1 VALUES('a b a b', NULL);
+  INSERT INTO t1 VALUES(NULL, '1 2 1 2');
+}
+
+do_faultsim_test 1 -faults oom-t* -body {
+  execsql { 
+    SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' 
+  }
+} -test {
+  faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \
+                       {1 SQLITE_NOMEM}
+}
+
+}
+
+finish_test
+
diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test
index 21f9b003e7..06f4550b47 100644
--- a/ext/fts5/test/fts5matchinfo.test
+++ b/ext/fts5/test/fts5matchinfo.test
@@ -16,6 +16,8 @@ set testprefix fts5matchinfo
 # If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 { finish_test ; return }
 
+foreach_detail_mode $testprefix {
+
 proc mit {blob} {
   set scan(littleEndian) i*
   set scan(bigEndian) I*
@@ -27,7 +29,7 @@ db func mit mit
 sqlite3_fts5_register_matchinfo db
 
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(content);
+  CREATE VIRTUAL TABLE t1 USING fts5(content, detail=%DETAIL%);
 } 
 
 do_execsql_test 1.1 {
@@ -41,7 +43,7 @@ do_execsql_test 1.1 {
 # Now create an FTS4 table that does not specify matchinfo=fts3.
 #
 do_execsql_test 1.2 {
-  CREATE VIRTUAL TABLE t2 USING fts5(content);
+  CREATE VIRTUAL TABLE t2 USING fts5(content, detail=%DETAIL%);
   INSERT INTO t2 SELECT * FROM t1;
   SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I';
 } {{1 1 1 2 2} {1 1 1 2 2}}
@@ -149,9 +151,17 @@ proc normalize2 {list_of_lists} {
   return $res
 }
 
+# Similar to [do_matchinfo_test], except that this is a no-op if the FTS5
+# mode is not detail=full.
+#
+proc do_matchinfo_p_test {tn tbl expr results} {
+  if {[detail_is_full]} {
+    uplevel [list do_matchinfo_test $tn $tbl $expr $results]
+  }
+}
 
 do_execsql_test 4.1.0 {
-  CREATE VIRTUAL TABLE t4 USING fts5(x, y);
+  CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%);
   INSERT INTO t4 VALUES('a b c d e', 'f g h i j');
   INSERT INTO t4 VALUES('f g h i j', 'a b c d e');
 }
@@ -185,7 +195,7 @@ do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
   xpxsscplax -
 }
 
-do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
+do_matchinfo_p_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
   p {1 1}
   c {2 2}
   x {
@@ -203,8 +213,8 @@ do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
 }
 
 do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'}     { s {{2 0} {0 2}} }
-do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
-do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
+do_matchinfo_p_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
+do_matchinfo_p_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
 do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'}     { s {{1 0} {0 1}} }
 do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} {
   x { 
@@ -220,7 +230,7 @@ do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f NOT abcd'} {
 }
 
 do_execsql_test 4.2.0 {
-  CREATE VIRTUAL TABLE t5 USING fts5(content);
+  CREATE VIRTUAL TABLE t5 USING fts5(content, detail=%DETAIL%);
   INSERT INTO t5 VALUES('a a a a a');
   INSERT INTO t5 VALUES('a b a b a');
   INSERT INTO t5 VALUES('c b c b c');
@@ -233,7 +243,7 @@ do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'}         {
 do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'}         { s {2} }
 do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'}       { s {3} }
 do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
-do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
+do_matchinfo_p_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
 do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1} }
 
 do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')";
@@ -250,7 +260,7 @@ if 0 {
 do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'}         { s {2} }
 do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'}       { s {3} }
 do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
-do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
+do_matchinfo_p_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
 do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1 1} }
 
 do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') }
@@ -260,10 +270,10 @@ do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'}         { s {2 1} }
 do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
 do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'}       { s {3} }
 do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
-do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
+do_matchinfo_p_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
 
 do_execsql_test 4.5.0 {
-  CREATE VIRTUAL TABLE t6 USING fts5(a, b, c);
+  CREATE VIRTUAL TABLE t6 USING fts5(a, b, c, detail=%DETAIL%);
   INSERT INTO t6 VALUES('a', 'b', 'c');
 }
 do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'}       { s {{1 1 1}} }
@@ -274,7 +284,7 @@ do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'}       { s {{1 1 1}} }
 # use the full-text index (i.e. lookup by rowid or full-table scan).
 #
 do_execsql_test 7.1 {
-  CREATE VIRTUAL TABLE t10 USING fts5(content);
+  CREATE VIRTUAL TABLE t10 USING fts5(content, detail=%DETAIL%);
   INSERT INTO t10 VALUES('first record');
   INSERT INTO t10 VALUES('second record');
 }
@@ -299,7 +309,7 @@ do_execsql_test 7.4 {
 #   SELECT sum(length(content)) < count(*) FROM fts4table;
 #
 do_execsql_test 8.1 {
-  CREATE VIRTUAL TABLE t11 USING fts5(content);
+  CREATE VIRTUAL TABLE t11 USING fts5(content, detail=%DETAIL%);
   INSERT INTO t11(t11, rank) VALUES('pgsz', 32);
   INSERT INTO t11 VALUES('quitealongstringoftext');
   INSERT INTO t11 VALUES('anotherquitealongstringoftext');
@@ -318,22 +328,24 @@ do_execsql_test 8.3 {
 
 #-------------------------------------------------------------------------
 
-do_execsql_test 9.1 {
-  CREATE VIRTUAL TABLE t12 USING fts5(content);
-  INSERT INTO t12 VALUES('a b c d');
-  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
-} {{0 1 1 0 1 1 1 1 1}}
-do_execsql_test 9.2 {
-  INSERT INTO t12 VALUES('a d c d');
-  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
-} {
-  {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2}
-}
-do_execsql_test 9.3 {
-  INSERT INTO t12 VALUES('a d d a');
-  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
-} {
-  {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3}
+if {[detail_is_full]} {
+  do_execsql_test 9.1 {
+    CREATE VIRTUAL TABLE t12 USING fts5(content, detail=%DETAIL%);
+    INSERT INTO t12 VALUES('a b c d');
+    SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
+  } {{0 1 1 0 1 1 1 1 1}}
+  do_execsql_test 9.2 {
+    INSERT INTO t12 VALUES('a d c d');
+    SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
+  } {
+    {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2}
+  }
+  do_execsql_test 9.3 {
+    INSERT INTO t12 VALUES('a d d a');
+    SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
+  } {
+    {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3}
+  }
 }
 
 #---------------------------------------------------------------------------
@@ -341,7 +353,7 @@ do_execsql_test 9.3 {
 #
 do_execsql_test 10.1 {
   DROP TABLE t10;
-  CREATE VIRTUAL TABLE t10 USING fts5(idx, value);
+  CREATE VIRTUAL TABLE t10 USING fts5(idx, value, detail=%DETAIL%);
   INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three');
   SELECT t10.rowid, t10.*
     FROM t10
@@ -358,7 +370,7 @@ do_execsql_test 10.1 {
 reset_db
 sqlite3_fts5_register_matchinfo db
 do_execsql_test 11.0 {
-  CREATE VIRTUAL TABLE tt USING fts5(x, y);
+  CREATE VIRTUAL TABLE tt USING fts5(x, y, detail=%DETAIL%);
   INSERT INTO tt VALUES('c d a c d d', 'e a g b d a');   -- 1
   INSERT INTO tt VALUES('c c g a e b', 'c g d g e c');   -- 2
   INSERT INTO tt VALUES('b e f d e g', 'b a c b c g');   -- 3
@@ -410,6 +422,8 @@ foreach {tn expr res} {
   }
 
 } {
+
+  if {[string match *:* $expr] && [detail_is_none]} continue
   do_execsql_test 11.1.$tn.1  {
     SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr
   } $res
@@ -443,7 +457,7 @@ db func mit mit
 do_test 12.0 {
   set cols [list]
   for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" }
-  execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,])"
+  execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,], detail=%DETAIL%)"
 } {}
 
 do_execsql_test 12.1 {
@@ -451,5 +465,7 @@ do_execsql_test 12.1 {
   SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc';
 } [list [list [expr 1<<4] [expr 1<<(45-32)]]]
 
+} ;# foreach_detail_mode
+
 finish_test
 
diff --git a/manifest b/manifest
index ff5d797e2d..25592e2931 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\smore\sfts5\stests\sto\srun\sin\sdetail=none\sand\sdetail=column\smodes\sas\swell\sas\sthe\sdefault\sdetail=full.
-D 2016-01-04T16:19:51.421
+C Fix\san\sOOM\shandling\sproblem\sin\sfts5\sdetail=none\sand\sdetail=col\smodes.\sAlso\sa\sbug\sin\sthe\sxInst()\sAPI\swhen\sused\swith\sdetail=col\sand\scolumn\sfiltering\sexpressions.\sUpdate\sthe\smatchinfo()\stest\sfunction\sso\sthat\s'b'\sis\sfast\swith\sdetail=col\stables.
+D 2016-01-04T19:12:00.230
 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751
@@ -97,17 +97,17 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 88fec577a2148f34df75930dc2b0c45b0bad72c3
-F ext/fts5/fts5Int.h cff7dd3131ba0db6f699df97237bf993f22c1a1f
+F ext/fts5/fts5Int.h 10608c346cccf7dd1da4d6b46f7921949072ed60
 F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35
 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1
 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6
-F ext/fts5/fts5_expr.c 898e0bb53280b2c4c2702a2dc08dec731c7b32fe
+F ext/fts5/fts5_expr.c 3fe372518c0191230971321048e10930952dad5b
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
 F ext/fts5/fts5_index.c 8665393f8be3ae9ed4f1033d7ce0b805108fb95c
-F ext/fts5/fts5_main.c f33439bde2e1023888e9b7f16e463e42fe4c00c5
+F ext/fts5/fts5_main.c 15c8b702e28d032224a6fc2b6a9c03ba2e4deeb1
 F ext/fts5/fts5_storage.c 771dd0fda3ee513e32937a386dc2a4aa2d9ea64e
 F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb
-F ext/fts5/fts5_test_mi.c c9c8cf455c49ded156a234e0de1c8ba9be85e5c3
+F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070
 F ext/fts5/fts5_tokenize.c 618efe033bceb80c521b1e9ddfd9fee85fb5946e
 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c
 F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
@@ -117,7 +117,7 @@ F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl fefb4ceb27d02b431f69045a7ac0061a80d97824
 F ext/fts5/test/fts5aa.test 4abbc14eb2d9d6d46a53ab008151512871d54ebc
 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
-F ext/fts5/test/fts5ac.test 9a3c5eb052a5e8c871d56d3d0f144e5fdcf075c2
+F ext/fts5/test/fts5ac.test 30707d2972e5f485f7bdef019f3041782edabfb3
 F ext/fts5/test/fts5ad.test 049f7511a79c155d2d8dfd2ddcfeb640c50ad0dc
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c
@@ -151,10 +151,11 @@ F ext/fts5/test/fts5fault4.test 4864f2b5c2c083440dbe85aff60897bc1aa04603
 F ext/fts5/test/fts5fault5.test f2b8645053d48982e8979749e93994c43011c118
 F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda
 F ext/fts5/test/fts5fault7.test 01be274bfc8d9bf22451a3bf5892e9399d044f1b
+F ext/fts5/test/fts5fault8.test aeb4717b7b293678bc4d2f3c0159206a525375d9
 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
 F ext/fts5/test/fts5hash.test 7cf4607b8657c383f0b520668a99971e95d8b139
 F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65
-F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2
+F ext/fts5/test/fts5matchinfo.test 86569026d20f1ed748236587ce798de8a96615f1
 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367
 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc
 F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c
@@ -1407,7 +1408,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 6322a1d984e7946735bace8a069ef24b31754b3b
-R 32e8694f89e6ec744ed1fb193a76a532
+P 3fcf3b1e24483b2cd1c1710f053ed8403e09106d
+R 133d47de041747f0d7e918f07a474874
 U dan
-Z 780f62d5b61ae786fb811bd6c42602f7
+Z 4f94b859477e06c41fefb0b71cc6e55e
diff --git a/manifest.uuid b/manifest.uuid
index 3f2a04cfe4..1f4ad7217a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3fcf3b1e24483b2cd1c1710f053ed8403e09106d
\ No newline at end of file
+ee38f9dff5e9239c541515cd8a1aa3d81fdc0ae9
\ No newline at end of file