From: dan <dan@noemail.net>
Date: Fri, 19 Dec 2014 20:53:51 +0000 (+0000)
Subject: Remove the fts5_test() aux function. Test aux functions using the tcl interface instead.
X-Git-Tag: version-3.8.11~114^2~122
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=1616d55153a7654bd7955d1ba0c5c08b786277bb;p=thirdparty%2Fsqlite.git

Remove the fts5_test() aux function. Test aux functions using the tcl interface instead.

FossilOrigin-Name: 67e3ffd950c5347d219a06b33ad51949cffa7d90
---

diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index aff871e9dd..a09487a369 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -841,273 +841,6 @@ static void fts5Bm25Function(
   }
 }
 
-static int fts5TestCallback(
-  void *pContext,                 /* Pointer to Fts5Buffer object */
-  const char *pToken,             /* Buffer containing token */
-  int nToken,                     /* Size of token in bytes */
-  int iStart,                     /* Start offset of token */
-  int iEnd,                       /* End offset of token */
-  int iPos                        /* Position offset of token */
-){
-  int rc = SQLITE_OK;
-  Fts5Buffer *pBuf = (Fts5Buffer*)pContext;
-  if( pBuf->n!=0 ){
-    sqlite3Fts5BufferAppendString(&rc, pBuf, " ");
-  }
-  sqlite3Fts5BufferAppendListElem(&rc, pBuf, pToken, nToken);
-  return rc;
-}
-
-
-static void fts5TestFunction(
-  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
-  Fts5Context *pFts,              /* First arg to pass to pApi functions */
-  sqlite3_context *pCtx,          /* Context for returning result/error */
-  int nVal,                       /* Number of values in apVal[] array */
-  sqlite3_value **apVal           /* Array of trailing arguments */
-){
-  Fts5Buffer s;                   /* Build up text to return here */
-  int nCol;                       /* Number of columns in table */
-  int nPhrase;                    /* Number of phrases in query */
-  i64 iRowid;                     /* Rowid of current row */
-  const char *zReq = 0;
-  int rc = SQLITE_OK;
-  int i;
-
-  if( nVal>=1 ){
-    zReq = (const char*)sqlite3_value_text(apVal[0]);
-  }
-
-  memset(&s, 0, sizeof(Fts5Buffer));
-  nCol = pApi->xColumnCount(pFts);
-
-  /*
-  ** xColumnTotalSize()
-  */
-  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "columntotalsize ");
-  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columntotalsize") ){
-    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
-    for(i=0; rc==SQLITE_OK && i<nCol; i++){
-      i64 colsz = 0;
-      rc = pApi->xColumnTotalSize(pFts, i, &colsz);
-      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
-    }
-    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
-  }
-
-  /*
-  ** xColumnCount()
-  */
-  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " columncount ");
-  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columncount") ){
-    nCol = pApi->xColumnCount(pFts);
-    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nCol);
-  }
-
-  /*
-  ** xColumnSize()
-  */
-  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " columnsize ");
-  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnsize") ){
-    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
-    for(i=0; rc==SQLITE_OK && i<nCol; i++){
-      int colsz = 0;
-      rc = pApi->xColumnSize(pFts, i, &colsz);
-      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
-    }
-    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
-  }
-
-  /*
-  ** xColumnText()
-  */
-  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " columntext ");
-  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columntext") ){
-    for(i=0; rc==SQLITE_OK && i<nCol; i++){
-      const char *z;
-      int n;
-      rc = pApi->xColumnText(pFts, i, &z, &n);
-      if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " ");
-      sqlite3Fts5BufferAppendListElem(&rc, &s, z, n);
-    }
-  }
-
-  /*
-  ** xInst()
-  */
-  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " inst ");
-  if( 0==zReq || 0==sqlite3_stricmp(zReq, "inst") ){
-    int nInst;
-    rc = pApi->xInstCount(pFts, &nInst);
-    for(i=0; rc==SQLITE_OK && i<nInst; i++){
-      int iPhrase, iCol, iOff;
-      rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
-      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d.%d.%d",
-          (i==0 ? "" : " "), iPhrase, iCol, iOff
-      );
-    }
-  }
-
-  /*
-  ** xPhraseCount()
-  */
-  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " phrasecount ");
-  nPhrase = pApi->xPhraseCount(pFts);
-  if( 0==zReq || 0==sqlite3_stricmp(zReq, "phrasecount") ){
-    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nPhrase);
-  }
-
-  /*
-  ** xPhraseSize()
-  */
-  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " phrasesize ");
-  if( 0==zReq || 0==sqlite3_stricmp(zReq, "phrasesize") ){
-    if( nPhrase==1 ){
-      int nSize = pApi->xPhraseSize(pFts, 0);
-      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nSize);
-    }else{
-      sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
-      for(i=0; i<nPhrase; i++){
-        int nSize = pApi->xPhraseSize(pFts, i);
-        sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", (i==0?"":" "), nSize);
-      }
-      sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
-    }
-  }
-
-  /*
-  ** xPoslist()
-  */
-  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " poslist ");
-  if( 0==zReq || 0==sqlite3_stricmp(zReq, "poslist") ){
-    int bParen = 0;
-    Fts5Buffer s3;
-    memset(&s3, 0, sizeof(s3));
-
-    for(i=0; i<nPhrase; i++){
-      Fts5Buffer s2;                  /* List of positions for phrase/column */
-      int j = 0;
-      i64 iPos = 0;
-      int nElem = 0;
-
-      memset(&s2, 0, sizeof(s2));
-      while( 0==pApi->xPoslist(pFts, i, &j, &iPos) ){
-        int iOff = FTS5_POS2OFFSET(iPos);
-        int iCol = FTS5_POS2COLUMN(iPos);
-        if( nElem!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s2, " ");
-        sqlite3Fts5BufferAppendPrintf(&rc, &s2, "%d.%d", iCol, iOff);
-        nElem++;
-      }
-
-      if( i!=0 ){
-        sqlite3Fts5BufferAppendPrintf(&rc, &s3, " ");
-      }
-      if( nElem==1 ){
-        sqlite3Fts5BufferAppendPrintf(&rc, &s3, "%s", (const char*)s2.p);
-      }else{
-        sqlite3Fts5BufferAppendPrintf(&rc, &s3, "{%s}", (const char*)s2.p);
-        bParen = 1;
-      }
-      sqlite3_free(s2.p);
-    }
-
-    if(zReq==0 && (nPhrase>1 || bParen) ){
-      sqlite3Fts5BufferAppendPrintf(&rc, &s, "{%s}", (const char*)s3.p);
-    }else{
-      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s", (const char*)s3.p);
-    }
-    sqlite3_free(s3.p);
-  }
-
-  if( zReq==0 ){
-    sqlite3Fts5BufferAppendPrintf(&rc, &s, " queryphrase ");
-  }
-  if( 0==zReq || 0==sqlite3_stricmp(zReq, "queryphrase") ){
-    int ic, ip;
-    int *anVal = 0;
-    Fts5Buffer buf1;
-    memset(&buf1, 0, sizeof(Fts5Buffer));
-
-    if( rc==SQLITE_OK ){
-      anVal = (int*)pApi->xGetAuxdata(pFts, 0);
-      if( anVal==0 ){
-        rc = fts5GatherTotals(pApi, pFts, &anVal);
-        if( rc==SQLITE_OK ){
-          rc = pApi->xSetAuxdata(pFts, (void*)anVal, sqlite3_free);
-        }
-      }
-    }
-
-    for(ip=0; rc==SQLITE_OK && ip<nPhrase; ip++){
-      if( ip>0 ) sqlite3Fts5BufferAppendString(&rc, &buf1, " ");
-      if( nCol>1 ) sqlite3Fts5BufferAppendString(&rc, &buf1, "{");
-      for(ic=0; ic<nCol; ic++){
-        int iVal = anVal[ip * nCol + ic];
-        sqlite3Fts5BufferAppendPrintf(&rc, &buf1, "%s%d", ic==0?"":" ", iVal);
-      }
-      if( nCol>1 ) sqlite3Fts5BufferAppendString(&rc, &buf1, "}");
-    }
-
-    if( zReq==0 ){
-      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf1.p, buf1.n);
-    }else{
-      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf1.p);
-    }
-    sqlite3_free(buf1.p);
-  }
-
-  if( zReq==0 ){
-    sqlite3Fts5BufferAppendString(&rc, &s, " rowid ");
-  }
-  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){
-    iRowid = pApi->xRowid(pFts);
-    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
-  }
-
-  if( zReq==0 ){
-    sqlite3Fts5BufferAppendString(&rc, &s, " rowcount ");
-  }
-  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowcount") ){
-    i64 nRow;
-    rc = pApi->xRowCount(pFts, &nRow);
-    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", nRow);
-  }
-
-  if( zReq==0 ){
-    sqlite3Fts5BufferAppendString(&rc, &s, " tokenize ");
-  }
-  if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){
-    Fts5Buffer buf;
-    memset(&buf, 0, sizeof(buf));
-    for(i=0; rc==SQLITE_OK && i<nCol; i++){
-      const char *z;
-      int n;
-      rc = pApi->xColumnText(pFts, i, &z, &n);
-      if( rc==SQLITE_OK ){
-        Fts5Buffer buf1;
-        memset(&buf1, 0, sizeof(Fts5Buffer));
-        rc = pApi->xTokenize(pFts, z, n, (void*)&buf1, fts5TestCallback);
-        if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &buf, " ");
-        sqlite3Fts5BufferAppendListElem(&rc, &buf, (const char*)buf1.p, buf1.n);
-        sqlite3_free(buf1.p);
-      }
-    }
-    if( zReq==0 ){
-      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf.p, buf.n);
-    }else{
-      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf.p);
-    }
-    sqlite3_free(buf.p);
-  }
-
-  if( rc==SQLITE_OK ){
-    sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT);
-  }else{
-    sqlite3_result_error_code(pCtx, rc);
-  }
-  sqlite3Fts5BufferFree(&s);
-}
-
 int sqlite3Fts5AuxInit(fts5_api *pApi){
   struct Builtin {
     const char *zFunc;            /* Function name (nul-terminated) */
@@ -1117,7 +850,6 @@ int sqlite3Fts5AuxInit(fts5_api *pApi){
   } aBuiltin [] = {
     { "bm25debug", (void*)1, fts5Bm25Function,    0 },
     { "snippet",   0, fts5SnippetFunction, 0 },
-    { "fts5_test", 0, fts5TestFunction,    0 },
     { "highlight", 0, fts5HighlightFunction, 0 },
     { "bm25",      0, fts5Bm25Function,    0 },
   };
diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c
index 28efe7109c..dd5ef60431 100644
--- a/ext/fts5/fts5_tcl.c
+++ b/ext/fts5/fts5_tcl.c
@@ -57,6 +57,57 @@ struct F5tApi {
   Fts5Context *pFts;
 };
 
+static int xTokenizeCb(
+  void *pCtx, 
+  const char *zToken, int nToken, 
+  int iStart, int iEnd, int iPos
+){
+  F5tFunction *p = (F5tFunction*)pCtx;
+  Tcl_Obj *pEval = Tcl_DuplicateObj(p->pScript);
+  int rc;
+
+  Tcl_IncrRefCount(pEval);
+  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zToken, nToken));
+  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iStart));
+  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iEnd));
+  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iPos));
+
+  rc = Tcl_EvalObjEx(p->interp, pEval, 0);
+  Tcl_DecrRefCount(pEval);
+
+  return rc;
+}
+
+static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);
+
+static int xQueryPhraseCb(
+  const Fts5ExtensionApi *pApi, 
+  Fts5Context *pFts, 
+  void *pCtx
+){
+  F5tFunction *p = (F5tFunction*)pCtx;
+  static sqlite3_int64 iCmd = 0;
+  Tcl_Obj *pEval;
+  int rc;
+
+  char zCmd[64];
+  F5tApi sApi;
+
+  sApi.pApi = pApi;
+  sApi.pFts = pFts;
+  sprintf(zCmd, "f5t_2_%lld", iCmd++);
+  Tcl_CreateObjCommand(p->interp, zCmd, xF5tApi, &sApi, 0);
+
+  pEval = Tcl_DuplicateObj(p->pScript);
+  Tcl_IncrRefCount(pEval);
+  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zCmd, -1));
+  rc = Tcl_EvalObjEx(p->interp, pEval, 0);
+  Tcl_DecrRefCount(pEval);
+  Tcl_DeleteCommand(p->interp, zCmd);
+
+  return rc;
+}
+
 /*
 **      api sub-command... 
 **
@@ -73,12 +124,21 @@ static int xF5tApi(
     int nArg;
     const char *zMsg;
   } aSub[] = {
-    { "xRowid",      0, "" },
-    { "xInstCount",  0, "" },
-    { "xInst",       1, "IDX" },
-    { "xColumnText", 1, "COL" },
-    { "xColumnSize", 1, "COL" },
+    { "xColumnCount",      0, "" },
+    { "xRowCount",         0, "" },
+    { "xColumnTotalSize",  1, "COL" },
+    { "xTokenize",         2, "TEXT SCRIPT" },
+    { "xPhraseCount",      0, "" },
+    { "xPhraseSize",       1, "PHRASE" },
+    { "xInstCount",        0, "" },
+    { "xInst",             1, "IDX" },
+    { "xRowid",            0, "" },
+    { "xColumnText",       1, "COL" },
+    { "xColumnSize",       1, "COL" },
+    { "xQueryPhrase",      2, "PHRASE SCRIPT" },
+    { 0, 0, 0}
   };
+
   int rc;
   int iSub = 0;
   F5tApi *p = (F5tApi*)clientData;
@@ -97,14 +157,67 @@ static int xF5tApi(
     return TCL_ERROR;
   }
 
+#define CASE(i,str) case i: assert( strcmp(aSub[i].zName, str)==0 );
   switch( iSub ){
-    case 0: { /* xRowid */
-      sqlite3_int64 iRowid = p->pApi->xRowid(p->pFts);
-      Tcl_SetObjResult(interp, Tcl_NewWideIntObj(iRowid));
+    CASE(0, "xColumnCount") {
+      int nCol;
+      nCol = p->pApi->xColumnCount(p->pFts);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewIntObj(nCol));
+      }
       break;
     }
-
-    case 1: { /* xInstCount */
+    CASE(1, "xRowCount") {
+      sqlite3_int64 nRow;
+      rc = p->pApi->xRowCount(p->pFts, &nRow);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nRow));
+      }
+      break;
+    }
+    CASE(2, "xColumnTotalSize") {
+      int iCol;
+      sqlite3_int64 nSize;
+      if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ) return TCL_ERROR;
+      rc = p->pApi->xColumnTotalSize(p->pFts, iCol, &nSize);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nSize));
+      }
+      break;
+    }
+    CASE(3, "xTokenize") {
+      int nText;
+      char *zText = Tcl_GetStringFromObj(objv[2], &nText);
+      F5tFunction ctx;
+      ctx.interp = interp;
+      ctx.pScript = objv[3];
+      rc = p->pApi->xTokenize(p->pFts, zText, nText, &ctx, xTokenizeCb);
+      if( rc==SQLITE_OK ){
+        Tcl_ResetResult(interp);
+      }
+      return rc;
+    }
+    CASE(4, "xPhraseCount") {
+      int nPhrase;
+      nPhrase = p->pApi->xPhraseCount(p->pFts);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewIntObj(nPhrase));
+      }
+      break;
+    }
+    CASE(5, "xPhraseSize") {
+      int iPhrase;
+      int sz;
+      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){
+        return TCL_ERROR;
+      }
+      sz = p->pApi->xPhraseSize(p->pFts, iPhrase);
+      if( rc==SQLITE_OK ){
+        Tcl_SetObjResult(interp, Tcl_NewIntObj(sz));
+      }
+      break;
+    }
+    CASE(6, "xInstCount") {
       int nInst;
       rc = p->pApi->xInstCount(p->pFts, &nInst);
       if( rc==SQLITE_OK ){
@@ -112,8 +225,7 @@ static int xF5tApi(
       }
       break;
     }
-
-    case 2: { /* xInst */
+    CASE(7, "xInst") {
       int iIdx, ip, ic, io;
       if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ){
         return TCL_ERROR;
@@ -128,8 +240,12 @@ static int xF5tApi(
       }
       break;
     }
-
-    case 3: { /* xColumnText */
+    CASE(8, "xRowid") {
+      sqlite3_int64 iRowid = p->pApi->xRowid(p->pFts);
+      Tcl_SetObjResult(interp, Tcl_NewWideIntObj(iRowid));
+      break;
+    }
+    CASE(9, "xColumnText") {
       const char *z = 0;
       int n = 0;
       int iCol;
@@ -142,8 +258,7 @@ static int xF5tApi(
       }
       break;
     }
-
-    case 4: { /* xColumnSize */
+    CASE(10, "xColumnSize") {
       int n = 0;
       int iCol;
       if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
@@ -155,11 +270,26 @@ static int xF5tApi(
       }
       break;
     }
+    CASE(11, "xQueryPhrase") {
+      int iPhrase;
+      F5tFunction ctx;
+      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){
+        return TCL_ERROR;
+      }
+      ctx.interp = interp;
+      ctx.pScript = objv[3];
+      rc = p->pApi->xQueryPhrase(p->pFts, iPhrase, &ctx, xQueryPhraseCb);
+      if( rc==SQLITE_OK ){
+        Tcl_ResetResult(interp);
+      }
+      break;
+    }
 
     default: 
       assert( 0 );
       break;
   }
+#undef CASE
 
   if( rc!=SQLITE_OK ){
     Tcl_AppendResult(interp, "error in api call", 0);
diff --git a/manifest b/manifest
index 9c8098afeb..6b558dbd3f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\swith\sprefix\squeries\sand\sthe\sAND\soperator.
-D 2014-12-18T20:01:15.691
+C Remove\sthe\sfts5_test()\saux\sfunction.\sTest\saux\sfunctions\susing\sthe\stcl\sinterface\sinstead.
+D 2014-12-19T20:53:51.092
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -107,14 +107,14 @@ F ext/fts5/extract_api_docs.tcl 6320db4a1d0722a4e2069e661381ad75e9889786
 F ext/fts5/fts5.c d1c1722eb661da3e8e3a19909958b97beff7d243
 F ext/fts5/fts5.h 72fc1e9995b1ddc254a487b9528614a83bd3dfb6
 F ext/fts5/fts5Int.h 36054b1dfc4881a9b94f945b348ab6cc01c0c7a5
-F ext/fts5/fts5_aux.c 0e3e5fea6bf5772805afe14c95cb5f16e03e4b3f
+F ext/fts5/fts5_aux.c b8e5660a05b86dab059c9989835b5df0ac5e3c55
 F ext/fts5/fts5_buffer.c 1bc5c762bb2e9b4a40b2e8a820a31b809e72eec1
 F ext/fts5/fts5_config.c 5caeb4e77680d635be25b899f97a29cf26fb45ce
 F ext/fts5/fts5_expr.c 27d3d2deebae277c34ae2bb3d501dd879c442ba5
 F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279
 F ext/fts5/fts5_index.c 4a8e8535b4303400ddb5f6fb08152da0d88ebf6f
 F ext/fts5/fts5_storage.c bfeedb83b095a1018f4f531c3cc3f9099e9f9081
-F ext/fts5/fts5_tcl.c 5272224faf9be129679da5e19d788f0307afc375
+F ext/fts5/fts5_tcl.c d56484fd5cc3b02d268ee11fa4918e98ce3b1d03
 F ext/fts5/fts5_tokenize.c 8360c0d1ae0d4696f3cc13f7c67a2db6011cdc5b
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
@@ -599,13 +599,14 @@ F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b
 F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849
 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36
+F test/fts5_common.tcl 2488117cd80b7a4de7c20054b89f082b77b4189c
 F test/fts5aa.test 27c7d3c865e144a0501dcbfbd6d2ae87f77602ea
 F test/fts5ab.test 52f6b9223372ff70b0edb5a3054fbd7bc7fcfefc
-F test/fts5ac.test 60302196b7711176ce872fe2e4c73c75ac2c4038
+F test/fts5ac.test 021e175b809d2baa23792807caae5dfc6bc706f4
 F test/fts5ad.test ff518db6b0d7750b51ee6531ffebf82e57094bfd
-F test/fts5ae.test 5de775469d45a2f8218fc89b8d6d5176c226d05e
+F test/fts5ae.test 0877873a2b9df6b3a2d832ed5ea928f838d19faf
 F test/fts5af.test d24e3b0f879998ef5f60087272f8ab7b3a8fd4dc
-F test/fts5ag.test 1c6c188d1bdc41b2277db3f4ddfea7d90bf44ceb
+F test/fts5ag.test 8b2bb67cf2a3245eaad5e49ab8daa6be6e64332b
 F test/fts5ah.test 788e923e60b5e7a559f672cfbf262b8b260ea176
 F test/fts5ai.test aa2b5fd0f8d2cf59ac0211111e63cbca3b40ed7d
 F test/fts5aj.test bc3d91bd012c7ca175cdf266c2074920bb5fa5ba
@@ -888,7 +889,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43
 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c
 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1
-F test/tester.tcl f31bea1483ea1d39620f982130026e76f872d744
+F test/tester.tcl 7d2c97b43e51abde7a35f9d3bc57e886c8503e55
 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5
 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58
 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7
@@ -1208,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P e358c3de5c916f2c851ab9324ceaae4e4e7a0fbd
-R d93acda5ed1642f2b9a85bcc5fb79da9
+P 38b3c65e3ee95eb7afadb76e0110570fbbc41e1b
+R a4b205e397ac161e65b87e55d29e5aba
 U dan
-Z ee6f25fbcb6953dd4dd423bed7e3ab13
+Z b9d8ce93a014115b155c701f123b4810
diff --git a/manifest.uuid b/manifest.uuid
index 82d7a8e415..3656228956 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-38b3c65e3ee95eb7afadb76e0110570fbbc41e1b
\ No newline at end of file
+67e3ffd950c5347d219a06b33ad51949cffa7d90
\ No newline at end of file
diff --git a/test/fts5_common.tcl b/test/fts5_common.tcl
new file mode 100644
index 0000000000..78f561ac95
--- /dev/null
+++ b/test/fts5_common.tcl
@@ -0,0 +1,114 @@
+# 2014 Dec 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+
+
+
+proc fts5_test_poslist {cmd} {
+  set res [list]
+  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
+    lappend res [string map {{ } .} [$cmd xInst $i]]
+  }
+  set res
+}
+
+proc fts5_test_columnsize {cmd} {
+  set res [list]
+  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
+    lappend res [$cmd xColumnSize $i]
+  }
+  set res
+}
+
+proc fts5_test_columntext {cmd} {
+  set res [list]
+  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
+    lappend res [$cmd xColumnText $i]
+  }
+  set res
+}
+
+proc fts5_test_columntotalsize {cmd} {
+  set res [list]
+  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
+    lappend res [$cmd xColumnTotalSize $i]
+  }
+  set res
+}
+
+proc test_append_token {varname token iStart iEnd iPos} {
+  upvar $varname var
+  lappend var $token
+}
+proc fts5_test_tokenize {cmd} {
+  set res [list]
+  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
+    set tokens [list]
+    $cmd xTokenize [$cmd xColumnText $i] [list test_append_token tokens]
+    lappend res $tokens
+  }
+  set res
+}
+
+proc fts5_test_rowcount {cmd} {
+  $cmd xRowCount
+}
+
+proc test_queryphrase_cb {cnt cmd} {
+  upvar $cnt L 
+  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
+    foreach {ip ic io} [$cmd xInst $i] break
+    set A($ic) 1
+  }
+  foreach ic [array names A] {
+    lset L $ic [expr {[lindex $L $ic] + 1}]
+  }
+}
+proc fts5_test_queryphrase {cmd} {
+  set res [list]
+  for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
+    set cnt [list]
+    for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 }
+    $cmd xQueryPhrase $i [list test_queryphrase_cb cnt]
+    lappend res $cnt
+  }
+  set res
+}
+
+proc fts5_test_all {cmd} {
+  set res [list]
+  lappend res columnsize      [fts5_test_columnsize $cmd]
+  lappend res columntext      [fts5_test_columntext $cmd]
+  lappend res columntotalsize [fts5_test_columntotalsize $cmd]
+  lappend res poslist         [fts5_test_poslist $cmd]
+  lappend res tokenize        [fts5_test_tokenize $cmd]
+  lappend res rowcount        [fts5_test_rowcount $cmd]
+  set res
+}
+
+proc fts5_aux_test_functions {db} {
+  foreach f {
+    fts5_test_columnsize
+    fts5_test_columntext
+    fts5_test_columntotalsize
+    fts5_test_poslist
+    fts5_test_tokenize
+    fts5_test_rowcount
+    fts5_test_all
+
+    fts5_test_queryphrase
+  } {
+    sqlite3_fts5_create_function $db $f $f
+  }
+}
+
+
diff --git a/test/fts5ac.test b/test/fts5ac.test
index 1b56c8b0e3..1044a81932 100644
--- a/test/fts5ac.test
+++ b/test/fts5ac.test
@@ -139,11 +139,12 @@ do_test 1.1 {
 
 # Usage:
 #
-#   poslist aCol ?-near N? ?-col C? -- phrase1 phrase2...
+#   poslist aCol ?-pc VARNAME? ?-near N? ?-col C? -- phrase1 phrase2...
 #
 proc poslist {aCol args} {
   set O(-near) 10
   set O(-col)  -1
+  set O(-pc)   ""
 
   set nOpt [lsearch -exact $args --]
   if {$nOpt<0} { error "no -- option" }
@@ -153,6 +154,12 @@ proc poslist {aCol args} {
     set O($k) $v
   }
 
+  if {$O(-pc) == ""} {
+    set counter 0
+  } else {
+    upvar $O(-pc) counter
+  }
+
   # Set $phraselist to be a list of phrases. $nPhrase its length.
   set phraselist [lrange $args [expr $nOpt+1] end]
   set nPhrase [llength $phraselist]
@@ -197,14 +204,15 @@ proc poslist {aCol args} {
   }
 
   set res [list]
+#puts [array names A]
+
   for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
-    set plist [list]
     for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
       foreach a $A($iCol,$iPhrase) {
-        lappend plist "$iCol.$a"
+        lappend res "$counter.$iCol.$a"
       }
     }
-    lappend res $plist
+    incr counter
   }
 
     #puts $res
@@ -220,6 +228,17 @@ proc nearset {args} {
   return [expr [llength [lindex $plist 0]]>0]
 }
 
+proc instcompare {lhs rhs} {
+  foreach {p1 c1 o1} [split $lhs .] {}
+  foreach {p2 c2 o2} [split $rhs .] {}
+
+  set res [expr $c1 - $c2]
+  if {$res==0} { set res [expr $o1 - $o2] }
+  if {$res==0} { set res [expr $p1 - $p2] }
+
+  return $res
+}
+
 # Argument $expr is an FTS5 match expression designed to be executed against
 # an FTS5 table with the following schema:
 # 
@@ -247,11 +266,14 @@ proc matchdata {bPos expr {bAsc 0}} {
       if {$bPos} {
         set N [regexp -all -inline {\[nearset [^\]]*\]} $tclexpr]
         set rowres [list]
+        set cnt 0
         foreach phrase $N {
-          set cmd "poslist [string range $phrase 9 end-1]"
+          set arglist [string range $phrase 9 end-1]
+          set cmd "poslist [lindex $arglist 0] -pc cnt [lrange $arglist 1 end]"
           set pos [eval $cmd]
           set rowres [concat $rowres $pos]
         }
+        set rowres [lsort -command instcompare $rowres]
         lappend res [list $id $rowres]
       } else {
         lappend res $id
@@ -272,6 +294,16 @@ proc matchdata {bPos expr {bAsc 0}} {
 # End of test code
 #-------------------------------------------------------------------------
 
+proc fts5_test_poslist {cmd} {
+  set res [list]
+  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
+    lappend res [string map {{ } .} [$cmd xInst $i]]
+  }
+  set res
+}
+
+sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist
+
 #-------------------------------------------------------------------------
 # Test phrase queries.
 #
@@ -291,7 +323,7 @@ foreach {tn phrase} {
   set res [matchdata 1 $expr]
 
   do_execsql_test 1.2.$tn.[llength $res] { 
-    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
+    SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   } $res
 }
 
@@ -313,7 +345,7 @@ foreach {tn expr} {
 } {
   set res [matchdata 1 $expr]
   do_execsql_test 2.$tn.[llength $res] { 
-    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
+    SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   } $res
 }
 
@@ -328,7 +360,7 @@ foreach {tn expr} {
 } {
   set res [matchdata 1 $expr]
   do_execsql_test 3.$tn.[llength $res] { 
-    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
+    SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   } $res
 }
 
@@ -349,12 +381,12 @@ foreach {tn expr} {
 } {
   set res [matchdata 1 $expr]
   do_execsql_test 4.1.$tn.[llength $res] { 
-    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
+    SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   } $res
 }
 
-do_test 4.1  { poslist {{a b c}} -- a } {0.0}
-do_test 4.2  { poslist {{a b c}} -- c } {0.2}
+do_test 4.1  { poslist {{a b c}} -- a } {0.0.0}
+do_test 4.2  { poslist {{a b c}} -- c } {0.0.2}
 
 foreach {tn expr tclexpr} {
   1 {a b} {[N $x -- {a}] && [N $x -- {b}]}
diff --git a/test/fts5ae.test b/test/fts5ae.test
index 07b1891618..b6475d8bbf 100644
--- a/test/fts5ae.test
+++ b/test/fts5ae.test
@@ -51,6 +51,7 @@ do_execsql_test 1.4.2 {
   SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
 } {1 2 4}
 
+fts5_aux_test_functions db
 
 #-------------------------------------------------------------------------
 # 
@@ -61,26 +62,26 @@ do_execsql_test 2.0 {
 }
 
 do_execsql_test 2.1 {
-  SELECT rowid, fts5_test(t2, 'poslist') FROM t2 
+  SELECT rowid, fts5_test_poslist(t2) FROM t2 
   WHERE t2 MATCH 'm' ORDER BY rowid;
 } {
-  1 {{0.5 1.0 1.2}} 
-  2 {{0.7 1.5}}
+  1 {0.0.5 0.1.0 0.1.2} 
+  2 {0.0.7 0.1.5}
 }
 
 do_execsql_test 2.2 {
-  SELECT rowid, fts5_test(t2, 'poslist') FROM t2 
+  SELECT rowid, fts5_test_poslist(t2) FROM t2 
   WHERE t2 MATCH 'u OR q' ORDER BY rowid;
 } {
-  1 {0.0 {}}
-  2 {{} {0.2 0.10}}
+  1 {0.0.0}
+  2 {1.0.2 1.0.10}
 }
 
 do_execsql_test 2.3 {
-  SELECT rowid, fts5_test(t2, 'poslist') FROM t2 
+  SELECT rowid, fts5_test_poslist(t2) FROM t2 
   WHERE t2 MATCH 'y:o' ORDER BY rowid;
 } {
-  1 {{1.3 1.7}}
+  1 {0.1.3 0.1.7}
 }
 
 #-------------------------------------------------------------------------
@@ -92,25 +93,25 @@ do_execsql_test 3.0 {
 }
 
 do_execsql_test 3.1 {
-  SELECT rowid, fts5_test(t3, 'poslist') FROM t3 WHERE t3 MATCH 'NEAR(a b)';
+  SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)';
 } {
-  1 {{0.6 0.10 0.12} {0.9 0.15}}
+  1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15}
 }
 
 do_execsql_test 3.2 {
-  SELECT rowid, fts5_test(t3, 'poslist') FROM t3 WHERE t3 MATCH 'NEAR(r c)';
+  SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)';
 } {
-  2 {0.0 0.1}
+  2 {0.0.0 1.0.1}
 }
 
 do_execsql_test 3.3 {
   INSERT INTO t3 
   VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
-  SELECT rowid, fts5_test(t3, 'poslist') 
+  SELECT rowid, fts5_test_poslist(t3) 
   FROM t3 WHERE t3 MATCH 'a OR b AND c';
 } {
-  3 {0.5 {} {}} 
-  1 {{0.6 0.10 0.12} {0.9 0.15} 1.2}
+  3 0.0.5 
+  1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15 2.1.2}
 }
 
 #-------------------------------------------------------------------------
@@ -122,16 +123,17 @@ do_execsql_test 4.0 {
 }
 
 do_execsql_test 4.1 {
-  SELECT rowid, fts5_test(t4, 'poslist') FROM t4 WHERE t4 MATCH 'a OR b AND c';
+  SELECT rowid, fts5_test_poslist(t4) FROM t4 WHERE t4 MATCH 'a OR b AND c';
 } {
-  1 {0.5 {} {}} 
+  1 0.0.5
 }
 
 #-------------------------------------------------------------------------
 # Test that the xColumnSize() and xColumnAvgsize() APIs work.
 #
-
 reset_db
+fts5_aux_test_functions db
+
 do_execsql_test 5.1 {
   CREATE VIRTUAL TABLE t5 USING fts5(x, y);
   INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
@@ -139,7 +141,7 @@ do_execsql_test 5.1 {
   INSERT INTO t5 VALUES('a', '');
 }
 do_execsql_test 5.2 {
-  SELECT rowid, fts5_test(t5, 'columnsize') FROM t5 WHERE t5 MATCH 'a'
+  SELECT rowid, fts5_test_columnsize(t5) FROM t5 WHERE t5 MATCH 'a'
   ORDER BY rowid DESC;
 } {
   3 {1 0}
@@ -148,7 +150,7 @@ do_execsql_test 5.2 {
 }
 
 do_execsql_test 5.2 {
-  SELECT rowid, fts5_test(t5, 'columntext') FROM t5 WHERE t5 MATCH 'a'
+  SELECT rowid, fts5_test_columntext(t5) FROM t5 WHERE t5 MATCH 'a'
   ORDER BY rowid DESC;
 } {
   3 {a {}}
@@ -157,7 +159,7 @@ do_execsql_test 5.2 {
 }
 
 do_execsql_test 5.3 {
-  SELECT rowid, fts5_test(t5, 'columntotalsize') FROM t5 WHERE t5 MATCH 'a'
+  SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
   ORDER BY rowid DESC;
 } {
   3 {5 7}
@@ -167,7 +169,7 @@ do_execsql_test 5.3 {
 
 do_execsql_test 5.4 {
   INSERT INTO t5 VALUES('x y z', 'v w x y z');
-  SELECT rowid, fts5_test(t5, 'columntotalsize') FROM t5 WHERE t5 MATCH 'a'
+  SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
   ORDER BY rowid DESC;
 } {
   3 {8 12}
@@ -179,6 +181,7 @@ do_execsql_test 5.4 {
 # Test the xTokenize() API
 #
 reset_db
+fts5_aux_test_functions db
 do_execsql_test 6.1 {
   CREATE VIRTUAL TABLE t6 USING fts5(x, y);
   INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
@@ -186,7 +189,7 @@ do_execsql_test 6.1 {
 }
 
 do_execsql_test 6.2 {
-  SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*'
+  SELECT rowid, fts5_test_tokenize(t6) FROM t6 WHERE t6 MATCH 't*'
 } {
   2 {{horatio than are} {dreamt of in your philosophy}}
   1 {{there are more} {things in heaven and earth}}
@@ -196,6 +199,7 @@ do_execsql_test 6.2 {
 # Test the xQueryPhrase() API
 #
 reset_db
+fts5_aux_test_functions db
 do_execsql_test 7.1 {
   CREATE VIRTUAL TABLE t7 USING fts5(x, y);
 }
@@ -221,12 +225,12 @@ foreach {tn q res} {
   6 {a OR b OR c OR d} {{4 2} {3 4} {2 1} {2 2}}
 } {
   do_execsql_test 7.3.$tn { 
-    SELECT fts5_test(t7, 'queryphrase') FROM t7 WHERE t7 MATCH $q LIMIT 1
+    SELECT fts5_test_queryphrase(t7) FROM t7 WHERE t7 MATCH $q LIMIT 1
   } [list $res]
 }
 
 do_execsql_test 7.4 {
-  SELECT fts5_test(t7, 'rowcount') FROM t7 WHERE t7 MATCH 'a';
+  SELECT fts5_test_rowcount(t7) FROM t7 WHERE t7 MATCH 'a';
 } {5 5 5 5}
 
 #do_execsql_test 7.4 {
diff --git a/test/fts5ag.test b/test/fts5ag.test
index 647604ef64..52b4774d55 100644
--- a/test/fts5ag.test
+++ b/test/fts5ag.test
@@ -96,9 +96,11 @@ do_test 1.1 {
   set {} {}
 } {}
 
+fts5_aux_test_functions db
+
 proc do_fts5ag_test {tn E} {
-  set q1 {SELECT fts5_test(t1) FROM t1 WHERE t1 MATCH $E ORDER BY rank}
-  set q2 {SELECT fts5_test(t1) FROM t1 WHERE t1 MATCH $E ORDER BY bm25(t1)}
+  set q1 {SELECT fts5_test_all(t1) FROM t1 WHERE t1 MATCH $E ORDER BY rank}
+  set q2 {SELECT fts5_test_all(t1) FROM t1 WHERE t1 MATCH $E ORDER BY bm25(t1)}
 
   set res [execsql $q1]
   set expected [execsql $q2]
diff --git a/test/tester.tcl b/test/tester.tcl
index 1c4e93937c..2b5d871d5f 100644
--- a/test/tester.tcl
+++ b/test/tester.tcl
@@ -1918,3 +1918,4 @@ database_never_corrupt
 
 source $testdir/thread_common.tcl
 source $testdir/malloc_common.tcl
+source $testdir/fts5_common.tcl