From: dan <dan@noemail.net>
Date: Sat, 16 Jan 2016 18:58:51 +0000 (+0000)
Subject: Add further tests for fts5. Fix some problems with detail=col mode and auxiliary... 
X-Git-Tag: version-3.11.0~134
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=8631402e6a53599344406d814962a1fab72fe474;p=thirdparty%2Fsqlite.git

Add further tests for fts5. Fix some problems with detail=col mode and auxiliary functions.

FossilOrigin-Name: de77d6026e8035c505a704e7b8cfe5af6579d35f
---

diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c
index 251a543c5a..8558687d8a 100644
--- a/ext/fts5/fts5_buffer.c
+++ b/ext/fts5/fts5_buffer.c
@@ -322,14 +322,17 @@ int sqlite3Fts5TermsetAdd(
   *pbPresent = 0;
   if( p ){
     int i;
-    int hash;
+    int hash = 13;
     Fts5TermsetEntry *pEntry;
 
-    /* Calculate a hash value for this term */
-    hash = 104 + iIdx;
-    for(i=0; i<nTerm; i++){
-      hash += (hash << 3) + (int)pTerm[i];
+    /* Calculate a hash value for this term. This is the same hash checksum
+    ** used by the fts5_hash.c module. This is not important for correct
+    ** operation of the module, but is necessary to ensure that some tests
+    ** designed to produce hash table collisions really do work.  */
+    for(i=nTerm-1; i>=0; i--){
+      hash = (hash << 3) ^ hash ^ pTerm[i];
     }
+    hash = (hash << 3) ^ hash ^ iIdx;
     hash = hash % ArraySize(p->apHash);
 
     for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index d9778bca20..70b1229edb 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -278,7 +278,7 @@ static int fts5ConfigParseSpecial(
         p++;
       }
 
-      if( rc==SQLITE_OK && (nPre<=0 || nPre>=1000) ){
+      if( nPre<=0 || nPre>=1000 ){
         *pzErr = sqlite3_mprintf("prefix length out of range (max 999)");
         rc = SQLITE_ERROR;
         break;
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 409fbd1d05..7a433b69a0 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -2396,7 +2396,7 @@ int sqlite3Fts5ExprPopulatePoslists(
   }
 
   return sqlite3Fts5Tokenize(pConfig, 
-      FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
+      FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
   );
 }
 
@@ -2412,49 +2412,44 @@ static void fts5ExprClearPoslists(Fts5ExprNode *pNode){
 }
 
 static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
-  if( pNode ){
-    pNode->iRowid = iRowid;
-    pNode->bEof = 0;
-    switch( pNode->eType ){
-      case FTS5_TERM:
-      case FTS5_STRING:
-        return (pNode->pNear->apPhrase[0]->poslist.n>0);
+  pNode->iRowid = iRowid;
+  pNode->bEof = 0;
+  switch( pNode->eType ){
+    case FTS5_TERM:
+    case FTS5_STRING:
+      return (pNode->pNear->apPhrase[0]->poslist.n>0);
 
-      case FTS5_AND: {
-        int i;
-        for(i=0; i<pNode->nChild; i++){
-          if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
-            fts5ExprClearPoslists(pNode);
-            return 0;
-          }
+    case FTS5_AND: {
+      int i;
+      for(i=0; i<pNode->nChild; i++){
+        if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
+          fts5ExprClearPoslists(pNode);
+          return 0;
         }
-        break;
       }
+      break;
+    }
 
-      case FTS5_OR: {
-        int i;
-        int bRet = 0;
-        for(i=0; i<pNode->nChild; i++){
-          if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
-            bRet = 1;
-          }
-        }
-        if( bRet==0 ){
-          fts5ExprClearPoslists(pNode);
+    case FTS5_OR: {
+      int i;
+      int bRet = 0;
+      for(i=0; i<pNode->nChild; i++){
+        if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
+          bRet = 1;
         }
-        return bRet;
       }
+      return bRet;
+    }
 
-      default: {
-        assert( pNode->eType==FTS5_NOT );
-        if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
-         || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
+    default: {
+      assert( pNode->eType==FTS5_NOT );
+      if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
+          || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
         ){
-          fts5ExprClearPoslists(pNode);
-          return 0;
-        }
-        break;
+        fts5ExprClearPoslists(pNode);
+        return 0;
       }
+      break;
     }
   }
   return 1;
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 95fb10baeb..7447f9b5e1 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -844,33 +844,32 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
 }
 
 
-static sqlite3_stmt *fts5PrepareStatement(
-  int *pRc,
+static int fts5PrepareStatement(
+  sqlite3_stmt **ppStmt,
   Fts5Config *pConfig, 
   const char *zFmt,
   ...
 ){
   sqlite3_stmt *pRet = 0;
+  int rc;
+  char *zSql;
   va_list ap;
-  va_start(ap, zFmt);
 
-  if( *pRc==SQLITE_OK ){
-    int rc;
-    char *zSql = sqlite3_vmprintf(zFmt, ap);
-    if( zSql==0 ){
-      rc = SQLITE_NOMEM; 
-    }else{
-      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
-      if( rc!=SQLITE_OK ){
-        *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
-      }
-      sqlite3_free(zSql);
+  va_start(ap, zFmt);
+  zSql = sqlite3_vmprintf(zFmt, ap);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM; 
+  }else{
+    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
+    if( rc!=SQLITE_OK ){
+      *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
     }
-    *pRc = rc;
+    sqlite3_free(zSql);
   }
 
   va_end(ap);
-  return pRet;
+  *ppStmt = pRet;
+  return rc;
 } 
 
 static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
@@ -878,7 +877,7 @@ static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
   Fts5Sorter *pSorter;
   int nPhrase;
   int nByte;
-  int rc = SQLITE_OK;
+  int rc;
   const char *zRank = pCsr->zRank;
   const char *zRankArgs = pCsr->zRankArgs;
   
@@ -896,7 +895,7 @@ static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
   ** table, saving it creates a circular reference.
   **
   ** If SQLite a built-in statement cache, this wouldn't be a problem. */
-  pSorter->pStmt = fts5PrepareStatement(&rc, pConfig,
+  rc = fts5PrepareStatement(&pSorter->pStmt, pConfig,
       "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s",
       pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
       (zRankArgs ? ", " : ""),
@@ -2039,8 +2038,15 @@ static int fts5ApiPhraseFirstColumn(
   Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
 
   if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
+    Fts5Sorter *pSorter = pCsr->pSorter;
     int n;
-    rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
+    if( pSorter ){
+      int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
+      n = pSorter->aIdx[iPhrase] - i1;
+      pIter->a = &pSorter->aPoslist[i1];
+    }else{
+      rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
+    }
     if( rc==SQLITE_OK ){
       pIter->b = &pIter->a[n];
       *piCol = 0;
diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c
index e633f0ac58..72db65777f 100644
--- a/ext/fts5/fts5_tcl.c
+++ b/ext/fts5/fts5_tcl.c
@@ -447,10 +447,12 @@ static int xF5tApi(
       zColvar = Tcl_GetString(objv[3]);
       zOffvar = Tcl_GetString(objv[4]);
 
-      for(p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
-          iCol>=0;
-          p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff)
-      ){
+      rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
+      if( rc!=SQLITE_OK ){
+        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+        return TCL_ERROR;
+      }
+      for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){
         Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
         Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0);
         rc = Tcl_EvalObjEx(interp, pScript, 0);
@@ -474,10 +476,12 @@ static int xF5tApi(
       if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
       zColvar = Tcl_GetString(objv[3]);
 
-      for(p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
-          iCol>=0;
-          p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)
-      ){
+      rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
+      if( rc!=SQLITE_OK ){
+        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+        return TCL_ERROR;
+      }
+      for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
         Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
         rc = Tcl_EvalObjEx(interp, pScript, 0);
         if( rc==TCL_CONTINUE ) rc = TCL_OK;
diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl
index 2c7fedcefe..e45355a6dd 100644
--- a/ext/fts5/test/fts5_common.tcl
+++ b/ext/fts5/test/fts5_common.tcl
@@ -510,6 +510,22 @@ proc fts5_poslist_data {expr tbl {order ASC} {aDictVar ""}} {
   set res
 }
 
+proc fts5_collist_data {expr tbl {order ASC} {aDictVar ""}} {
+  set res [list]
+
+  if {$aDictVar!=""} {
+    upvar $aDictVar aDict
+    set dict aDict
+  } else {
+    set dict ""
+  }
+
+  foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
+    lappend res $rowid $collist
+  }
+  set res
+}
+
 #-------------------------------------------------------------------------
 #
 
@@ -561,3 +577,54 @@ proc nearset_rc {aCol args} {
   list
 }
 
+
+#-------------------------------------------------------------------------
+# Code for a simple Tcl tokenizer that supports synonyms at query time.
+#
+proc tclnum_tokenize {mode tflags text} {
+  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+    sqlite3_fts5_token $w $iStart $iEnd
+    if {$tflags == $mode && [info exists ::tclnum_syn($w)]} {
+      foreach s $::tclnum_syn($w)  { sqlite3_fts5_token -colo $s $iStart $iEnd }
+    }
+  }
+}
+
+proc tclnum_create {args} {
+  set mode query
+  if {[llength $args]} {
+    set mode [lindex $args 0]
+  }
+  if {$mode != "query" && $mode != "document"} { error "bad mode: $mode" }
+  return [list tclnum_tokenize $mode]
+}
+
+proc fts5_tclnum_register {db} {
+  foreach SYNDICT {
+    {zero  0}
+    {one   1 i}
+    {two   2 ii}
+    {three 3 iii}
+    {four  4 iv}
+    {five  5 v}
+    {six   6 vi}
+    {seven 7 vii}
+    {eight 8 viii}
+    {nine  9 ix}
+
+    {a1 a2 a3 a4 a5 a6 a7 a8 a9}
+    {b1 b2 b3 b4 b5 b6 b7 b8 b9}
+    {c1 c2 c3 c4 c5 c6 c7 c8 c9}
+  } {
+    foreach s $SYNDICT {
+      set o [list]
+      foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
+      set ::tclnum_syn($s) $o
+    }
+  }
+  sqlite3_fts5_create_tokenizer db tclnum tclnum_create
+}
+#
+# End of tokenizer code.
+#-------------------------------------------------------------------------
+
diff --git a/ext/fts5/test/fts5config.test b/ext/fts5/test/fts5config.test
index dcda2d42a6..223e504a65 100644
--- a/ext/fts5/test/fts5config.test
+++ b/ext/fts5/test/fts5config.test
@@ -44,6 +44,8 @@ foreach {tn opt} {
   3 {prefix='$'}
   4 {prefix='1,2,'}
   5 {prefix=',1'}
+  6 {prefix='1,2,3...'}
+  7 {prefix='1,2,3xyz'}
 } {
   set res [list 1 {malformed prefix=... directive}]
   do_catchsql_test 2.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
@@ -159,6 +161,8 @@ do_catchsql_test 8.1 {
 #   9.1.* 'pgsz' options.
 #   9.2.* 'automerge' options.
 #   9.3.* 'crisismerge' options.
+#   9.4.* a non-existant option.
+#   9.5.* 'hashsize' options.
 #
 do_execsql_test 9.0 {
   CREATE VIRTUAL TABLE abc USING fts5(a, b);
@@ -203,6 +207,16 @@ do_catchsql_test 9.4.1 {
   INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
 } {1 {SQL logic error or missing database}}
 
+do_catchsql_test 9.5.1 {
+  INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
+} {1 {SQL logic error or missing database}}
+do_catchsql_test 9.5.2 {
+  INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
+} {1 {SQL logic error or missing database}}
+do_catchsql_test 9.5.3 {
+  INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
+} {0 {}}
+
 #-------------------------------------------------------------------------
 # Too many prefix indexes. Maximum allowed is 31.
 #
@@ -214,5 +228,20 @@ foreach {tn spec} {
   do_catchsql_test 10.$tn $sql {1 {too many prefix indexes (max 31)}}
 }
 
+#-------------------------------------------------------------------------
+# errors in the detail= option.
+#
+foreach {tn opt} {
+  1 {detail=x}  
+  2 {detail='x'}
+  3 {detail='$'}
+  4 {detail='1,2,'}
+  5 {detail=',1'}
+  6 {detail=''}
+} {
+  set res [list 1 {malformed detail=... directive}]
+  do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
+}
+
 finish_test
 
diff --git a/ext/fts5/test/fts5fault4.test b/ext/fts5/test/fts5fault4.test
index 989a372d49..acc43ebfc6 100644
--- a/ext/fts5/test/fts5fault4.test
+++ b/ext/fts5/test/fts5fault4.test
@@ -40,27 +40,6 @@ do_faultsim_test 1 -faults oom-* -prep {
   faultsim_test_result [list 0 {}]
 }
 
-#-------------------------------------------------------------------------
-# An OOM within an "ORDER BY rank" query.
-#
-db func rnddoc fts5_rnddoc 
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE xx USING fts5(x);
-  INSERT INTO xx VALUES ('abc ' || rnddoc(10));
-  INSERT INTO xx VALUES ('abc abc' || rnddoc(9));
-  INSERT INTO xx VALUES ('abc abc abc' || rnddoc(8));
-} {}
-faultsim_save_and_close
-
-do_faultsim_test 2 -faults oom-* -prep {
-  faultsim_restore_and_reopen
-  execsql { SELECT * FROM xx }
-} -body {
-  execsql { SELECT rowid FROM xx WHERE xx MATCH 'abc' ORDER BY rank }
-} -test {
-  faultsim_test_result [list 0 {3 2 1}]
-}
-
 #-------------------------------------------------------------------------
 # An OOM while "reseeking" an FTS cursor.
 #
diff --git a/ext/fts5/test/fts5fault9.test b/ext/fts5/test/fts5fault9.test
new file mode 100644
index 0000000000..b827c32df4
--- /dev/null
+++ b/ext/fts5/test/fts5fault9.test
@@ -0,0 +1,140 @@
+# 2015 September 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5fault9
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+
+fts5_aux_test_functions db
+
+if 1 {
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  WITH seq(s) AS ( SELECT 1 UNION ALL SELECT s+1 FROM seq WHERE s<50)
+  INSERT INTO t1 SELECT 'x x x y y y', 'a b c d e f' FROM seq;
+}
+
+do_faultsim_test 1 -faults oom-* -body {
+  execsql { SELECT count(*) FROM t1('x AND y') }
+} -test {
+  faultsim_test_result {0 50}
+}
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%);
+  INSERT INTO t2(t2, rank) VALUES('pgsz', 32);
+  INSERT INTO t2 VALUES('abc cba', 'cba abc');
+  INSERT INTO t2 VALUES('abc cba', 'cba abc');
+  INSERT INTO t2 VALUES('abc cba', 'cba abc');
+
+  INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
+  INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
+  INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
+}
+
+do_faultsim_test 2 -faults oom-* -body {
+  execsql { SELECT count(*) FROM t2('a* AND c*') }
+} -test {
+  faultsim_test_result {0 6}
+}
+
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t3 USING fts5(a, detail=%DETAIL%);
+  INSERT INTO t3 VALUES('a x x a x a a a');
+  INSERT INTO t3 VALUES('x a a x a x x x');
+}
+
+do_faultsim_test 3.1 -faults oom-* -body {
+  execsql { SELECT highlight(t3, 0, '[', ']') FROM t3('a') }
+} -test {
+  faultsim_test_result {0 {{[a] x x [a] x [a] [a] [a]} {x [a] [a] x [a] x x x}}}
+}
+
+do_faultsim_test 3.2 -faults oom-t* -body {
+  execsql { SELECT fts5_test_poslist2(t3) FROM t3('x') }
+} -test {
+  faultsim_test_result \
+      {0 {{0.0.1 0.0.2 0.0.4} {0.0.0 0.0.3 0.0.5 0.0.6 0.0.7}}} \
+      {1 SQLITE_NOMEM}
+}
+
+#-------------------------------------------------------------------------
+# Test OOM injection with the xPhraseFirstColumn() API and a tokenizer
+# uses query synonyms.
+#
+fts5_tclnum_register db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t4 USING fts5(x, y, z, detail=%DETAIL%, tokenize=tclnum);
+  INSERT INTO t4 VALUES('one two three', '1 2 3', 'i ii iii');
+  INSERT INTO t4 VALUES('1 2 3', 'i ii iii', 'one two three');
+  INSERT INTO t4 VALUES('i ii iii', 'one two three', 'i ii iii');
+
+  INSERT INTO t4 VALUES('a1 a2 a3', 'a4 a5 a6', 'a7 a8 a9');
+  INSERT INTO t4 VALUES('b1 b2 b3', 'b4 b5 b6', 'b7 b8 b9');
+  INSERT INTO t4 VALUES('c1 c2 c3', 'c4 c5 c6', 'c7 c8 c9');
+}
+
+do_faultsim_test 4.1 -faults oom-t* -body {
+  execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') }
+} -test {
+  faultsim_test_result \
+      {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM}
+}
+
+do_faultsim_test 4.2 -faults oom-t* -body {
+  execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') }
+} -test {
+  faultsim_test_result \
+      {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM}
+}
+
+}
+
+
+#-------------------------------------------------------------------------
+# An OOM within an "ORDER BY rank" query.
+#
+db func rnddoc fts5_rnddoc 
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=%DETAIL%);
+  INSERT INTO xx VALUES ('def', 'abc ' || rnddoc(10));
+  INSERT INTO xx VALUES ('def', 'abc abc' || rnddoc(9));
+  INSERT INTO xx VALUES ('def', 'abc abc abc' || rnddoc(8));
+} {}
+faultsim_save_and_close
+
+do_faultsim_test 5 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT * FROM xx }
+} -body {
+  execsql { SELECT rowid FROM xx('abc AND def') ORDER BY rank }
+} -test {
+  faultsim_test_result [list 0 {3 2 1}]
+}
+
+} ;# foreach_detail_mode...
+
+finish_test
+
diff --git a/ext/fts5/test/fts5hash.test b/ext/fts5/test/fts5hash.test
index a4a4f28343..ac8486f070 100644
--- a/ext/fts5/test/fts5hash.test
+++ b/ext/fts5/test/fts5hash.test
@@ -64,11 +64,13 @@ proc random_doc {vocab nWord} {
   return $doc
 }
 
+foreach_detail_mode $testprefix {
+
 set vocab [build_vocab1]
 db func r random_doc 
 
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE eee USING fts5(e, ee);
+  CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%);
   BEGIN;
     WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
     INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii;
@@ -91,7 +93,7 @@ do_test 1.2 {
   }
 } {}
   
-do_test 1.2 {
+do_test 1.3 {
   db eval { SELECT term, doc FROM vocab } {
     set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}]
     if {$nRow != $doc} {
@@ -101,10 +103,12 @@ do_test 1.2 {
   set {} {}
 } {}
 
-do_execsql_test 1.3 {
+do_execsql_test 1.4 {
   COMMIT;
   INSERT INTO eee(eee) VALUES('integrity-check');
 }
 
+} ;# foreach_detail_mode
+
 finish_test
 
diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test
index 008d2fc543..185dda3ff4 100644
--- a/ext/fts5/test/fts5synonym.test
+++ b/ext/fts5/test/fts5synonym.test
@@ -21,42 +21,16 @@ ifcapable !fts5 {
   return
 }
 
-foreach S {
-  {zero 0}
-  {one 1 i}
-  {two 2 ii}
-  {three 3 iii}
-  {four 4 iv}
-  {five 5 v}
-  {six 6 vi}
-  {seven 7 vii}
-  {eight 8 viii}
-  {nine 9 ix}
-} {
-  foreach s $S {
-    set o [list]
-    foreach x $S {if {$x!=$s} {lappend o $x}}
-    set ::syn($s) $o
-  }
-}
-
-proc tcl_tokenize {tflags text} {
-  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
-    sqlite3_fts5_token $w $iStart $iEnd
-  }
-}
+proc tcl_create {args} { return "tcl_tokenize" }
 
-proc tcl_create {args} {
-  return "tcl_tokenize"
-}
-
-sqlite3_fts5_create_tokenizer db tcl tcl_create
+foreach_detail_mode $testprefix {
 
 #-------------------------------------------------------------------------
 # Warm body test for the code in fts5_tcl.c.
 #
+fts5_tclnum_register db
 do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
+  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = "tclnum document", detail=%DETAIL%);
   INSERT INTO ft VALUES('abc def ghi');
   INSERT INTO ft VALUES('jkl mno pqr');
   SELECT rowid, x FROM ft WHERE ft MATCH 'def';
@@ -67,22 +41,13 @@ do_execsql_test 1.0 {
 # Test a tokenizer that supports synonyms by adding extra entries to the
 # FTS index.
 #
-
-proc tcl_tokenize {tflags text} {
-  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
-    sqlite3_fts5_token $w $iStart $iEnd
-    if {$tflags=="document" && [info exists ::syn($w)]} {
-      foreach s $::syn($w) {
-        sqlite3_fts5_token -colo $s $iStart $iEnd
-      }
-    }
-  }
-}
 reset_db
-sqlite3_fts5_create_tokenizer db tcl tcl_create
+fts5_tclnum_register db
 
 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize = "tclnum document", detail=%DETAIL%
+  );
   INSERT INTO ft VALUES('one two three');
   INSERT INTO ft VALUES('four five six');
   INSERT INTO ft VALUES('eight nine ten');
@@ -95,6 +60,7 @@ foreach {tn expr res} {
   4 "1*" {1}
   5 "1 + 2" {1}
 } {
+  if {![fts5_expr_ok $expr ft]} continue
   do_execsql_test 2.1.$tn {
     SELECT rowid FROM ft WHERE ft MATCH $expr
   } $res
@@ -180,17 +146,7 @@ do_execsql_test 3.2.5 {
 # Check that expressions with synonyms can be parsed and executed.
 #
 reset_db
-sqlite3_fts5_create_tokenizer db tcl tcl_create
-proc tcl_tokenize {tflags text} {
-  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
-    sqlite3_fts5_token $w $iStart $iEnd
-    if {$tflags=="query" && [info exists ::syn($w)]} {
-      foreach s $::syn($w) {
-        sqlite3_fts5_token -colo $s $iStart $iEnd
-      }
-    }
-  }
-}
+fts5_tclnum_register db
 
 foreach {tn expr res} {
   1  {abc}                           {"abc"}
@@ -198,11 +154,13 @@ foreach {tn expr res} {
   3  {3}                             {"3"|"iii"|"three"}
   4  {3*}                            {"3"|"iii"|"three" *}
 } {
-  do_execsql_test 4.1.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res]
+  do_execsql_test 4.1.$tn {
+    SELECT fts5_expr($expr, 'tokenize=tclnum')
+  } [list $res]
 }
 
 do_execsql_test 4.2.1 {
-  CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tcl);
+  CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tclnum, detail=%DETAIL%);
   INSERT INTO xx VALUES('one two');
   INSERT INTO xx VALUES('three four');
 }
@@ -217,7 +175,7 @@ do_execsql_test 4.2.3 {
 
 do_test 5.0 {
   execsql { 
-    CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tcl)
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tclnum, detail=%DETAIL%)
   }
   foreach {rowid a b} {
     1 {four v 4 i three} {1 3 five five 4 one}
@@ -285,6 +243,7 @@ foreach {tn q res} {
     5 {three i v i four 4 1} {ii [five five five] iii}
   }
 } {
+  if {![fts5_expr_ok $q t1]} continue
   do_execsql_test 5.1.$tn {
     SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']')
     FROM t1 WHERE t1 MATCH $q
@@ -316,7 +275,6 @@ foreach {tn q res} {
   } $res
 }
 
-
 #-------------------------------------------------------------------------
 # Test terms with more than 4 synonyms.
 #
@@ -334,17 +292,19 @@ proc tcl_tokenize {tflags text} {
 }
 
 do_execsql_test 6.0.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl, detail=%DETAIL%);
   INSERT INTO t1 VALUES('yy xx qq');
   INSERT INTO t1 VALUES('yy xx xx');
 }
-do_execsql_test 6.0.2 {
-  SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)';
-} {{yy xx qq}}
+if {[fts5_expr_ok "NEAR(y q)" t1]} {
+  do_execsql_test 6.0.2 {
+    SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)';
+  } {{yy xx qq}}
+}
 
 do_test 6.0.3 {
   execsql { 
-    CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl)
+    CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl, detail=%DETAIL%)
   }
   foreach {rowid a b} {
     1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa}
@@ -387,6 +347,8 @@ foreach {tn q res} {
     2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]}
   }
 } {
+  if {![fts5_expr_ok $q t2]} continue
+
   do_execsql_test 6.1.$tn.asc {
     SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']')
     FROM t2 WHERE t2 MATCH $q
@@ -435,7 +397,7 @@ proc tcl_tokenize {tflags text} {
 }
 
 do_execsql_test 7.0.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl, detail=%DETAIL%);
   INSERT INTO t1 VALUES('0 2 3', '4 5 6 7');
   INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
   SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0';
@@ -446,7 +408,7 @@ do_execsql_test 7.0.2 {
 }
 
 do_execsql_test 7.1.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl, detail=%DETAIL%);
   INSERT INTO t2 VALUES('0 2 3', '4 5 6 7');
   INSERT INTO t2 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
   SELECT fts5_test_columnsize(t2) FROM t2 WHERE t2 MATCH '000 AND 00 AND 0';
@@ -456,5 +418,7 @@ do_execsql_test 7.1.2 {
   INSERT INTO t2(t2) VALUES('integrity-check');
 }
 
+} ;# foreach_detail_mode
+
 finish_test
 
diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test
index 96cacb293f..dddaa17af8 100644
--- a/ext/fts5/test/fts5synonym2.test
+++ b/ext/fts5/test/fts5synonym2.test
@@ -21,52 +21,22 @@ ifcapable !fts5 {
   return
 }
 
-#-------------------------------------------------------------------------
-# Code for a simple Tcl tokenizer that supports synonyms at query time.
-#
-foreach SYNDICT {
-  {zero  0}
-  {one   1 i}
-  {two   2 ii}
-  {three 3 iii}
-  {four  4 iv}
-  {five  5 v}
-  {six   6 vi}
-  {seven 7 vii}
-  {eight 8 viii}
-  {nine  9 ix}
-} {
-  foreach s $SYNDICT {
-    set o [list]
-    foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
-    set ::syn($s) $o
-  }
-}
-
-proc tcl_tokenize {tflags text} {
-  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
-    sqlite3_fts5_token $w $iStart $iEnd
-    if {$tflags == "query"} {
-      foreach s $::syn($w)  { sqlite3_fts5_token -colo $s $iStart $iEnd }
-    }
-  }
-}
-
-proc tcl_create {args} {
-  return "tcl_tokenize"
-}
-
-#
-# End of tokenizer code.
-#-------------------------------------------------------------------------
-
+foreach tok {query document} {
 foreach_detail_mode $testprefix {
 
-sqlite3_fts5_create_tokenizer db tcl tcl_create
+fts5_tclnum_register db
 fts5_aux_test_functions db
 
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%);
+proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] }
+sqlite3_fts5_create_function db fts5_rowid fts5_rowid
+
+do_execsql_test 1.$tok.0.1 "
+  CREATE VIRTUAL TABLE ss USING fts5(a, b, 
+       tokenize='tclnum $tok', detail=%DETAIL%);
+  INSERT INTO ss(ss, rank) VALUES('rank', 'fts5_rowid()');
+"
+
+do_execsql_test 1.$tok.0.2 {
   INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i');
   INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi');
   INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3');
@@ -137,17 +107,22 @@ foreach {tn expr} {
   4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)"
 } {
   if {[fts5_expr_ok $expr ss]==0} {
-    do_test 1.$tn.OMITTED { list } [list]
+    do_test 1.$tok.$tn.OMITTED { list } [list]
     continue
   }
 
-  set res [fts5_query_data $expr ss ASC ::syn]
-  breakpoint
-  do_execsql_test 1.$tn.[llength $res].asc {
+  set res [fts5_query_data $expr ss ASC ::tclnum_syn]
+  do_execsql_test 1.$tok.$tn.[llength $res].asc.1 {
     SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
   } $res
+
+  do_execsql_test 1.$tok.$tn.[llength $res].asc.2 {
+    SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
+    ORDER BY rank ASC
+  } $res
 }
 
+}
 }
 
 finish_test
diff --git a/manifest b/manifest
index fa6bddb8b1..8872e8fc50 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\ssqlite3VdbeAddOp4()\srather\sthan\sa\sseparate\ssqlite3VdbeChangeP4()\scall,\sfor\na\sslightly\ssmaller\sand\sfaster\sbinary.
-D 2016-01-16T15:12:35.739
+C Add\sfurther\stests\sfor\sfts5.\sFix\ssome\sproblems\swith\sdetail=col\smode\sand\sauxiliary\sfunctions.
+D 2016-01-16T18:58:51.767
 F Makefile.in a476545d0c8626224d0bacac85c6e2967474af81
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 01e855f958932d0d3ed62ec675fc63e2cef61fcb
@@ -99,14 +99,14 @@ F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
 F ext/fts5/fts5Int.h 313e3276ac9e0245ee722ef803253857a68722b9
 F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e
-F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1
-F ext/fts5/fts5_config.c 9c243d04ac0ca997d2d2e2252891f2a10fbd7217
-F ext/fts5/fts5_expr.c 510db45967ca359f64f2ba2c707ab57d740cad56
+F ext/fts5/fts5_buffer.c ba59964c95f760bd5ff1dbe173b85f197a13864e
+F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238
+F ext/fts5/fts5_expr.c 3ba4c9588ebb2e4f852d807869af7130b6362e5a
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
 F ext/fts5/fts5_index.c cd036089b22b0340fccef88a3ad62ac1016c7cbb
-F ext/fts5/fts5_main.c 1d116f5c44c6f06ec282d33dd6d041c8131b3d6a
+F ext/fts5/fts5_main.c 833db0a3df10ab26e0221a9baa40cf871c450df3
 F ext/fts5/fts5_storage.c fb2eaec3aa954b680d43096dc539f8270bd6390e
-F ext/fts5/fts5_tcl.c bcacc05dec0446e7b1a44d5d906057e677bd7ea4
+F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
 F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070
 F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be
 F ext/fts5/fts5_tokenize.c 504984ac6993323247221eebe3cd55bead01b5f8
@@ -115,7 +115,7 @@ F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
 F ext/fts5/fts5_vocab.c ee6df1a3be103414d7b7af833ae1885c7b83a9d0
 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl 393882afb225a21edf033043bbf936951e9198c1
+F ext/fts5/test/fts5_common.tcl 75b0af04898c18df289620dd4e7442d881e3ccc1
 F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084
 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test d5073ca7bd2d9fe8aab0c82c6c75a7e4b0d70ced
@@ -134,7 +134,7 @@ F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f
 F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e
 F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb
 F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07
-F ext/fts5/test/fts5config.test 42c1336cc6ed33d7e9c4a05dbce81721b765e7d0
+F ext/fts5/test/fts5config.test 83941309b94d002ed6f55d9cd814e0353c9ae013
 F ext/fts5/test/fts5conflict.test 26f4e46c4d31e16221794832a990dc4e30e18de5
 F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
@@ -148,13 +148,14 @@ F ext/fts5/test/fts5eb.test 021aa80b7ac09b964249aa32ced9ee908703e4aa
 F ext/fts5/test/fts5fault1.test 4b39c47ca3544615daa8a2f733b911fa08022c77
 F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341
 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
-F ext/fts5/test/fts5fault4.test 4864f2b5c2c083440dbe85aff60897bc1aa04603
+F ext/fts5/test/fts5fault4.test 532b6dacb963016cbf7003196bd87fb366540277
 F ext/fts5/test/fts5fault5.test 10c13a783de3f42a21e3e53e123b62ed0c3a1618
 F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda
 F ext/fts5/test/fts5fault7.test 01be274bfc8d9bf22451a3bf5892e9399d044f1b
 F ext/fts5/test/fts5fault8.test f2d8a2b673a5f72ca1fa0e85bdbfb2041ffd347d
+F ext/fts5/test/fts5fault9.test 964a3ee82b95053bd45d809fe2b7acf4a8c90ca9
 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
-F ext/fts5/test/fts5hash.test 7cf4607b8657c383f0b520668a99971e95d8b139
+F ext/fts5/test/fts5hash.test 00668f6fa9b9bffbd7c1be29f408aa2bdade0451
 F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65
 F ext/fts5/test/fts5matchinfo.test 86569026d20f1ed748236587ce798de8a96615f1
 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367
@@ -173,8 +174,8 @@ F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821
 F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9
 F ext/fts5/test/fts5simple2.test 7b51f8d411e9a77fa4519fb09ba5a3afda75c94d
-F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
-F ext/fts5/test/fts5synonym2.test d2d9099d9d105b55ea03fd52d61ae2847d534129
+F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48
+F ext/fts5/test/fts5synonym2.test eadb00c73ef0653258873e756b7e9102e0687539
 F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
@@ -1416,7 +1417,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 653ea15ad8e23b333e234eb5dde7b80134db2baf
-R 2155f55389b348e9a35fea6f93c01bbe
-U drh
-Z 32d5bf122f6c10496965338bc29a15a2
+P a4258cd4613c55acacb5c7b61faa3de7eb0759d2
+R 7d01724b879e1ef98656daadba5b9cfb
+U dan
+Z d745955e2f048d17831d9a126adffb23
diff --git a/manifest.uuid b/manifest.uuid
index 39ad2ac515..94a16261bf 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a4258cd4613c55acacb5c7b61faa3de7eb0759d2
\ No newline at end of file
+de77d6026e8035c505a704e7b8cfe5af6579d35f
\ No newline at end of file