From: dan <dan@noemail.net>
Date: Tue, 9 Jun 2015 20:58:39 +0000 (+0000)
Subject: Add the "columnsize=" option to fts5, similar to fts4's "matchinfo=fts3".
X-Git-Tag: version-3.8.11~114^2~13
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=bcc2f04c6818d30c85b8c8d240d2aaa8815b1056;p=thirdparty%2Fsqlite.git

Add the "columnsize=" option to fts5, similar to fts4's "matchinfo=fts3".

FossilOrigin-Name: aa12f9d9b79c2f523fd6b00e47bcb66dba09ce0c
---

diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c
index f5d6a64acb..2c397c92b8 100644
--- a/ext/fts5/fts5.c
+++ b/ext/fts5/fts5.c
@@ -1039,6 +1039,7 @@ static int fts5FilterMethod(
   sqlite3_value **apVal           /* Arguments for the indexing scheme */
 ){
   Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
+  Fts5Config *pConfig = pTab->pConfig;
   Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
   int rc = SQLITE_OK;             /* Error code */
   int iVal = 0;                   /* Counter for apVal[] */
@@ -1049,7 +1050,7 @@ static int fts5FilterMethod(
   sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
   sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
   sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
-  char **pzErrmsg = pTab->pConfig->pzErrmsg;
+  char **pzErrmsg = pConfig->pzErrmsg;
 
   assert( pCsr->pStmt==0 );
   assert( pCsr->pExpr==0 );
@@ -1059,7 +1060,7 @@ static int fts5FilterMethod(
   assert( pCsr->zRankArgs==0 );
 
   assert( pzErrmsg==0 || pzErrmsg==&pTab->base.zErrMsg );
-  pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg;
+  pConfig->pzErrmsg = &pTab->base.zErrMsg;
 
   /* Decode the arguments passed through to this function.
   **
@@ -1107,7 +1108,7 @@ static int fts5FilterMethod(
   }else if( pMatch ){
     const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
 
-    rc = fts5CursorParseRank(pTab->pConfig, pCsr, pRank);
+    rc = fts5CursorParseRank(pConfig, pCsr, pRank);
     if( rc==SQLITE_OK ){
       if( zExpr[0]=='*' ){
         /* The user has issued a query of the form "MATCH '*...'". This
@@ -1116,7 +1117,7 @@ static int fts5FilterMethod(
         rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]);
       }else{
         char **pzErr = &pTab->base.zErrMsg;
-        rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
+        rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr);
         if( rc==SQLITE_OK ){
           if( bOrderByRank ){
             pCsr->ePlan = FTS5_PLAN_SORTED_MATCH;
@@ -1128,6 +1129,11 @@ static int fts5FilterMethod(
         }
       }
     }
+  }else if( pConfig->zContent==0 ){
+    *pConfig->pzErrmsg = sqlite3_mprintf(
+        "%s: table does not support scanning", pConfig->zName
+    );
+    rc = SQLITE_ERROR;
   }else{
     /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup
     ** by rowid (ePlan==FTS5_PLAN_ROWID).  */
@@ -1146,7 +1152,7 @@ static int fts5FilterMethod(
     }
   }
 
-  pTab->pConfig->pzErrmsg = pzErrmsg;
+  pConfig->pzErrmsg = pzErrmsg;
   return rc;
 }
 
@@ -1621,23 +1627,58 @@ static int fts5ApiColumnText(
   return rc;
 }
 
+static int fts5ColumnSizeCb(
+  void *pContext,                 /* Pointer to int */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStart,                     /* Start offset of token */
+  int iEnd                        /* End offset of token */
+){
+  int *pCnt = (int*)pContext;
+  *pCnt = *pCnt + 1;
+  return SQLITE_OK;
+}
+
 static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
   Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
   Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
+  Fts5Config *pConfig = pTab->pConfig;
   int rc = SQLITE_OK;
 
   if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
-    i64 iRowid = fts5CursorRowid(pCsr);
-    rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
+    if( pConfig->bColumnsize ){
+      i64 iRowid = fts5CursorRowid(pCsr);
+      rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
+    }else if( pConfig->zContent==0 ){
+      int i;
+      for(i=0; i<pConfig->nCol; i++){
+        if( pConfig->abUnindexed[i]==0 ){
+          pCsr->aColumnSize[i] = -1;
+        }
+      }
+    }else{
+      int i;
+      for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
+        if( pConfig->abUnindexed[i]==0 ){
+          const char *z; int n;
+          void *p = (void*)(&pCsr->aColumnSize[i]);
+          pCsr->aColumnSize[i] = 0;
+          rc = fts5ApiColumnText(pCtx, i, &z, &n);
+          if( rc==SQLITE_OK ){
+            rc = sqlite3Fts5Tokenize(pConfig, z, n, p, fts5ColumnSizeCb);
+          }
+        }
+      }
+    }
     CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
   }
   if( iCol<0 ){
     int i;
     *pnToken = 0;
-    for(i=0; i<pTab->pConfig->nCol; i++){
+    for(i=0; i<pConfig->nCol; i++){
       *pnToken += pCsr->aColumnSize[i];
     }
-  }else if( iCol<pTab->pConfig->nCol ){
+  }else if( iCol<pConfig->nCol ){
     *pnToken = pCsr->aColumnSize[iCol];
   }else{
     *pnToken = 0;
@@ -1956,7 +1997,7 @@ static int fts5FindFunctionMethod(
 }
 
 /*
-** Implementation of FTS3 xRename method. Rename an fts5 table.
+** Implementation of FTS5 xRename method. Rename an fts5 table.
 */
 static int fts5RenameMethod(
   sqlite3_vtab *pVtab,            /* Virtual table handle */
diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 54c23df629..b3fb4611fc 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -105,6 +105,9 @@ typedef struct Fts5Config Fts5Config;
 **   decent error message if it encounters a file-format version it does
 **   not understand.
 **
+** bColumnsize:
+**   True if the %_docsize table is created.
+**
 */
 struct Fts5Config {
   sqlite3 *db;                    /* Database handle */
@@ -118,6 +121,7 @@ struct Fts5Config {
   int eContent;                   /* An FTS5_CONTENT value */
   char *zContent;                 /* content table */ 
   char *zContentRowid;            /* "content_rowid=" option value */ 
+  int bColumnsize;                /* "columnsize=" option value (dflt==1) */
   char *zContentExprlist;
   Fts5Tokenizer *pTok;
   fts5_tokenizer *pTokApi;
@@ -196,6 +200,8 @@ void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
 void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
 void sqlite3Fts5BufferAppend32(int*, Fts5Buffer*, int);
 
+char *sqlite3Fts5Mprintf(int *pRc, char *zFmt, ...);
+
 #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
 #define fts5BufferGrow(a,b,c)         sqlite3Fts5BufferGrow(a,b,c)
 #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c
index 5c4e518bcc..b900ada744 100644
--- a/ext/fts5/fts5_buffer.c
+++ b/ext/fts5/fts5_buffer.c
@@ -125,6 +125,21 @@ void sqlite3Fts5BufferAppendPrintf(
   }
 }
 
+char *sqlite3Fts5Mprintf(int *pRc, char *zFmt, ...){
+  char *zRet = 0;
+  if( *pRc==SQLITE_OK ){
+    va_list ap;
+    va_start(ap, zFmt);
+    zRet = sqlite3_vmprintf(zFmt, ap);
+    va_end(ap);
+    if( zRet==0 ){
+      *pRc = SQLITE_NOMEM; 
+    }
+  }
+  return zRet;
+}
+ 
+
 /*
 ** Free any buffer allocated by pBuf. Zero the structure before returning.
 */
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index 4b35ac8b2c..04366fcea7 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -196,7 +196,7 @@ void sqlite3Fts5Dequote(char *z){
 }
 
 /*
-** Parse the "special" CREATE VIRTUAL TABLE directive and update
+** Parse a "special" CREATE VIRTUAL TABLE directive and update
 ** configuration object pConfig as appropriate.
 **
 ** If successful, object pConfig is updated and SQLITE_OK returned. If
@@ -211,10 +211,10 @@ static int fts5ConfigParseSpecial(
   const char *zArg,               /* Argument to parse */
   char **pzErr                    /* OUT: Error message */
 ){
+  int rc = SQLITE_OK;
   int nCmd = strlen(zCmd);
   if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
     const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
-    int rc = SQLITE_OK;
     const char *p;
     if( pConfig->aPrefix ){
       *pzErr = sqlite3_mprintf("multiple prefix=... directives");
@@ -248,7 +248,6 @@ static int fts5ConfigParseSpecial(
   }
 
   if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
-    int rc = SQLITE_OK;
     const char *p = (const char*)zArg;
     int nArg = strlen(zArg) + 1;
     char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
@@ -293,7 +292,6 @@ static int fts5ConfigParseSpecial(
   }
 
   if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){
-    int rc = SQLITE_OK;
     if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
       *pzErr = sqlite3_mprintf("multiple content=... directives");
       rc = SQLITE_ERROR;
@@ -301,19 +299,15 @@ static int fts5ConfigParseSpecial(
       if( zArg[0] ){
         pConfig->eContent = FTS5_CONTENT_EXTERNAL;
         pConfig->zContent = sqlite3_mprintf("%Q.%Q", pConfig->zDb, zArg);
+        if( pConfig->zContent==0 ) rc = SQLITE_NOMEM;
       }else{
         pConfig->eContent = FTS5_CONTENT_NONE;
-        pConfig->zContent = sqlite3_mprintf(
-            "%Q.'%q_docsize'", pConfig->zDb, pConfig->zName
-        );
       }
-      if( pConfig->zContent==0 ) rc = SQLITE_NOMEM;
     }
     return rc;
   }
 
   if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
-    int rc = SQLITE_OK;
     if( pConfig->zContentRowid ){
       *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
       rc = SQLITE_ERROR;
@@ -323,6 +317,16 @@ static int fts5ConfigParseSpecial(
     return rc;
   }
 
+  if( sqlite3_strnicmp("columnsize", zCmd, nCmd)==0 ){
+    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
+      *pzErr = sqlite3_mprintf("malformed columnsize=... directive");
+      rc = SQLITE_ERROR;
+    }else{
+      pConfig->bColumnsize = (zArg[0]=='1');
+    }
+    return rc;
+  }
+
   *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
   return SQLITE_ERROR;
 }
@@ -477,6 +481,7 @@ int sqlite3Fts5ConfigParse(
   pRet->abUnindexed = (u8*)&pRet->azCol[nArg];
   pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
   pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
+  pRet->bColumnsize = 1;
   if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
     *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
     rc = SQLITE_ERROR;
@@ -530,15 +535,24 @@ int sqlite3Fts5ConfigParse(
   }
 
   /* If no zContent option was specified, fill in the default values. */
-  if( rc==SQLITE_OK && pRet->eContent==FTS5_CONTENT_NORMAL ){
-    pRet->zContent = sqlite3_mprintf("%Q.'%q_content'", pRet->zDb, pRet->zName);
-    if( pRet->zContent==0 ){
-      rc = SQLITE_NOMEM;
-    }else{
-      sqlite3_free(pRet->zContentRowid);
-      pRet->zContentRowid = 0;
+  if( rc==SQLITE_OK && pRet->zContent==0 ){
+    const char *zTail = 0;
+    assert( pRet->eContent==FTS5_CONTENT_NORMAL 
+         || pRet->eContent==FTS5_CONTENT_NONE 
+    );
+    if( pRet->eContent==FTS5_CONTENT_NORMAL ){
+      zTail = "content";
+    }else if( pRet->bColumnsize ){
+      zTail = "docsize";
+    }
+
+    if( zTail ){
+      pRet->zContent = sqlite3Fts5Mprintf(
+          &rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail
+      );
     }
   }
+
   if( rc==SQLITE_OK && pRet->zContentRowid==0 ){
     pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1);
   }
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index cd677c2449..4dd72c2d41 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -39,15 +39,11 @@ struct Fts5Storage {
 
 #define FTS5_STMT_INSERT_CONTENT  3
 #define FTS5_STMT_REPLACE_CONTENT 4
-
 #define FTS5_STMT_DELETE_CONTENT  5
 #define FTS5_STMT_REPLACE_DOCSIZE  6
 #define FTS5_STMT_DELETE_DOCSIZE  7
-
 #define FTS5_STMT_LOOKUP_DOCSIZE  8
-
 #define FTS5_STMT_REPLACE_CONFIG 9
-
 #define FTS5_STMT_SCAN 10
 
 /*
@@ -64,6 +60,14 @@ static int fts5StorageGetStmt(
 ){
   int rc = SQLITE_OK;
 
+  /* If there is no %_docsize table, there should be no requests for 
+  ** statements to operate on it.  */
+  assert( p->pConfig->bColumnsize || (
+        eStmt!=FTS5_STMT_REPLACE_DOCSIZE 
+     && eStmt!=FTS5_STMT_DELETE_DOCSIZE 
+     && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE 
+  ));
+
   assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
   if( p->aStmt[eStmt]==0 ){
     const char *azStmt[] = {
@@ -175,12 +179,16 @@ static int fts5ExecPrintf(
 int sqlite3Fts5DropAll(Fts5Config *pConfig){
   int rc = fts5ExecPrintf(pConfig->db, 0, 
       "DROP TABLE IF EXISTS %Q.'%q_data';"
-      "DROP TABLE IF EXISTS %Q.'%q_docsize';"
       "DROP TABLE IF EXISTS %Q.'%q_config';",
       pConfig->zDb, pConfig->zName,
-      pConfig->zDb, pConfig->zName,
       pConfig->zDb, pConfig->zName
   );
+  if( rc==SQLITE_OK && pConfig->bColumnsize ){
+    rc = fts5ExecPrintf(pConfig->db, 0, 
+        "DROP TABLE IF EXISTS %Q.'%q_docsize';",
+        pConfig->zDb, pConfig->zName
+    );
+  }
   if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
     rc = fts5ExecPrintf(pConfig->db, 0, 
         "DROP TABLE IF EXISTS %Q.'%q_content';",
@@ -266,7 +274,7 @@ int sqlite3Fts5StorageOpen(
       sqlite3_free(zDefn);
     }
 
-    if( rc==SQLITE_OK ){
+    if( rc==SQLITE_OK && pConfig->bColumnsize ){
       rc = sqlite3Fts5CreateTable(
           pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
       );
@@ -374,19 +382,25 @@ static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
 ** Insert a record into the %_docsize table. Specifically, do:
 **
 **   INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
+**
+** If there is no %_docsize table (as happens if the columnsize=0 option
+** is specified when the FTS5 table is created), this function is a no-op.
 */
 static int fts5StorageInsertDocsize(
   Fts5Storage *p,                 /* Storage module to write to */
   i64 iRowid,                     /* id value */
   Fts5Buffer *pBuf                /* sz value */
 ){
-  sqlite3_stmt *pReplace = 0;
-  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
-  if( rc==SQLITE_OK ){
-    sqlite3_bind_int64(pReplace, 1, iRowid);
-    sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
-    sqlite3_step(pReplace);
-    rc = sqlite3_reset(pReplace);
+  int rc = SQLITE_OK;
+  if( p->pConfig->bColumnsize ){
+    sqlite3_stmt *pReplace = 0;
+    rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
+    if( rc==SQLITE_OK ){
+      sqlite3_bind_int64(pReplace, 1, iRowid);
+      sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
+      sqlite3_step(pReplace);
+      rc = sqlite3_reset(pReplace);
+    }
   }
   return rc;
 }
@@ -455,6 +469,7 @@ static int fts5StorageSaveTotals(Fts5Storage *p){
 ** Remove a row from the FTS table.
 */
 int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
+  Fts5Config *pConfig = p->pConfig;
   int rc;
   sqlite3_stmt *pDel;
 
@@ -466,7 +481,7 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
   }
 
   /* Delete the %_docsize record */
-  if( rc==SQLITE_OK ){
+  if( rc==SQLITE_OK && pConfig->bColumnsize ){
     rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
   }
   if( rc==SQLITE_OK ){
@@ -528,13 +543,15 @@ int sqlite3Fts5StorageSpecialDelete(
   }
 
   /* Delete the %_docsize record */
-  if( rc==SQLITE_OK ){
-    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
-  }
-  if( rc==SQLITE_OK ){
-    sqlite3_bind_int64(pDel, 1, iDel);
-    sqlite3_step(pDel);
-    rc = sqlite3_reset(pDel);
+  if( pConfig->bColumnsize ){
+    if( rc==SQLITE_OK ){
+      rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
+    }
+    if( rc==SQLITE_OK ){
+      sqlite3_bind_int64(pDel, 1, iDel);
+      sqlite3_step(pDel);
+      rc = sqlite3_reset(pDel);
+    }
   }
 
   /* Write the averages record */
@@ -554,11 +571,15 @@ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
 
   /* Delete the contents of the %_data and %_docsize tables. */
   rc = fts5ExecPrintf(pConfig->db, 0,
-      "DELETE FROM %Q.'%q_data';"
-      "DELETE FROM %Q.'%q_docsize';",
-      pConfig->zDb, pConfig->zName,
+      "DELETE FROM %Q.'%q_data';",
       pConfig->zDb, pConfig->zName
   );
+  if( rc==SQLITE_OK && pConfig->bColumnsize ){
+    rc = fts5ExecPrintf(pConfig->db, 0,
+        "DELETE FROM %Q.'%q_docsize';",
+        pConfig->zDb, pConfig->zName
+    );
+  }
 
   /* Reinitialize the %_data table. This call creates the initial structure
   ** and averages records.  */
@@ -635,18 +656,24 @@ int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
 ** a NULL value is inserted into the rowid column. The new rowid is allocated
 ** by inserting a dummy row into the %_docsize table. The dummy will be
 ** overwritten later.
+**
+** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In
+** this case the user is required to provide a rowid explicitly.
 */
 static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
-  sqlite3_stmt *pReplace = 0;
-  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
-  if( rc==SQLITE_OK ){
-    sqlite3_bind_null(pReplace, 1);
-    sqlite3_bind_null(pReplace, 2);
-    sqlite3_step(pReplace);
-    rc = sqlite3_reset(pReplace);
-  }
-  if( rc==SQLITE_OK ){
-    *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
+  int rc = SQLITE_MISMATCH;
+  if( p->pConfig->bColumnsize ){
+    sqlite3_stmt *pReplace = 0;
+    rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
+    if( rc==SQLITE_OK ){
+      sqlite3_bind_null(pReplace, 1);
+      sqlite3_bind_null(pReplace, 2);
+      sqlite3_step(pReplace);
+      rc = sqlite3_reset(pReplace);
+    }
+    if( rc==SQLITE_OK ){
+      *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
+    }
   }
   return rc;
 }
@@ -958,6 +985,7 @@ int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
       rc = FTS5_CORRUPT;
     }
   }
+
   return rc;
 }
 
diff --git a/ext/fts5/test/fts5ae.test b/ext/fts5/test/fts5ae.test
index d310e723be..e32e1a3b6e 100644
--- a/ext/fts5/test/fts5ae.test
+++ b/ext/fts5/test/fts5ae.test
@@ -148,7 +148,7 @@ do_execsql_test 5.2 {
   1 {4 6}
 }
 
-do_execsql_test 5.2 {
+do_execsql_test 5.3 {
   SELECT rowid, fts5_test_columntext(t5) FROM t5 WHERE t5 MATCH 'a'
   ORDER BY rowid DESC;
 } {
@@ -157,7 +157,7 @@ do_execsql_test 5.2 {
   1 {{a b c d} {e f g h i j}}
 }
 
-do_execsql_test 5.3 {
+do_execsql_test 5.4 {
   SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
   ORDER BY rowid DESC;
 } {
@@ -166,7 +166,7 @@ do_execsql_test 5.3 {
   1 {5 7}
 }
 
-do_execsql_test 5.4 {
+do_execsql_test 5.5 {
   INSERT INTO t5 VALUES('x y z', 'v w x y z');
   SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
   ORDER BY rowid DESC;
diff --git a/ext/fts5/test/fts5columnsize.test b/ext/fts5/test/fts5columnsize.test
new file mode 100644
index 0000000000..4dead06f62
--- /dev/null
+++ b/ext/fts5/test/fts5columnsize.test
@@ -0,0 +1,112 @@
+# 2015 Jun 10
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on fts5 tables with the columnsize=0 option.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5columnsize
+
+#-------------------------------------------------------------------------
+# Check that the option can be parsed and that the %_docsize table is
+# only created if it is set to true.
+#
+foreach {tn outcome stmt} {
+  1 0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0) }
+  2 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=1) }
+  3 0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='0') }
+  4 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='1') }
+  5 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='') }
+  6 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=2) }
+  7 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0, columnsize=1) }
+  8 1 { CREATE VIRTUAL TABLE t1 USING fts5(x) }
+} {
+  execsql { 
+    DROP TABLE IF EXISTS t1;
+  }
+  if {$outcome==2} {
+    do_catchsql_test 1.$tn.1 $stmt {1 {malformed columnsize=... directive}}
+  } else {
+    do_execsql_test 1.$tn.2 $stmt
+    do_execsql_test 1.$tn.3 {
+      SELECT count(*) FROM sqlite_master WHERE name = 't1_docsize'
+    } $outcome
+  }
+}
+
+#-------------------------------------------------------------------------
+# Run tests on a table with no %_content or %_docsize backing store.
+#
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x, columnsize=0, content='');
+}
+do_catchsql_test 2.1 {
+  INSERT INTO t2 VALUES('a b c d e f');
+} {1 {datatype mismatch}}
+do_execsql_test 2.2 {
+  INSERT INTO t2(rowid, x) VALUES(1, 'c d e f');
+  INSERT INTO t2(rowid, x) VALUES(2, 'c d e f g h');
+  INSERT INTO t2(rowid, x) VALUES(3, 'a b c d e f g h');
+} {}
+do_execsql_test 2.3 {
+  SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::';
+  SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::';
+  SELECT rowid FROM t2 WHERE t2 MATCH 'h'; 
+} {3 :: 1 2 3 :: 2 3}
+do_execsql_test 2.4 {
+  INSERT INTO t2(t2, rowid, x) VALUES('delete', 2, 'c d e f g h');
+  SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::';
+  SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::';
+  SELECT rowid FROM t2 WHERE t2 MATCH 'h'; 
+} {3 :: 1 3 :: 3}
+do_execsql_test 2.5 {
+  INSERT INTO t2(t2) VALUES('delete-all');
+  SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::';
+  SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::';
+  SELECT rowid FROM t2 WHERE t2 MATCH 'h'; 
+} {:: ::}
+do_execsql_test 2.6 {
+  INSERT INTO t2(rowid, x) VALUES(1, 'o t t f');
+  INSERT INTO t2(rowid, x) VALUES(2, 'f s s e');
+  INSERT INTO t2(rowid, x) VALUES(3, 'n t e t');
+}
+
+do_catchsql_test 2.7.1 {
+  SELECT rowid FROM t2
+} {1 {t2: table does not support scanning}}
+do_catchsql_test 2.7.2 {
+  SELECT rowid FROM t2 WHERE rowid=2
+} {1 {t2: table does not support scanning}}
+do_catchsql_test 2.7.3 {
+  SELECT rowid FROM t2 WHERE rowid BETWEEN 1 AND 3
+} {1 {t2: table does not support scanning}}
+
+do_execsql_test 2.X {
+  DROP TABLE t2
+}
+
+#-------------------------------------------------------------------------
+# Test the xColumnSize() API
+#
+fts5_aux_test_functions db
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t3 USING fts5(x, y UNINDEXED, z, columnsize=0);
+  INSERT INTO t3 VALUES('a a', 'b b b', 'c');
+  INSERT INTO t3 VALUES('x a x', 'b b b y', '');
+}
+do_execsql_test 3.1 {
+  SELECT rowid, fts5_test_columnsize(t3) FROM t3 WHERE t3 MATCH 'a'
+} {
+  1 {2 0 1} 2 {3 0 0}
+}
+
+finish_test
diff --git a/manifest b/manifest
index 8040ab30e4..3b31e16d5b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\scomment\sin\sfts5.h.
-D 2015-06-06T19:23:32.945
+C Add\sthe\s"columnsize="\soption\sto\sfts5,\ssimilar\sto\sfts4's\s"matchinfo=fts3".
+D 2015-06-09T20:58:39.182
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in d272f8755b464f20e02dd7799bfe16794c9574c4
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -105,16 +105,16 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl ed0534dd51efce39878bce33944c6073d37a1e20
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
-F ext/fts5/fts5.c 1c7424b9ba39f1e244f776556a4fface71abe772
+F ext/fts5/fts5.c 8af8014b40c382a987998a27f72490b339ce3726
 F ext/fts5/fts5.h 81d1a92fc2b4bd477af7e4e0b38b456f3e199fba
-F ext/fts5/fts5Int.h 3de83c9639bd8332eb84a13c1eb2387e83e128bf
+F ext/fts5/fts5Int.h a6d1c30e1655bd91484cb98661581e35a130b87b
 F ext/fts5/fts5_aux.c d53f00f31ad615ca4f139dd8751f9041afa00971
-F ext/fts5/fts5_buffer.c 9ec57c75c81e81dca118568876b1caead0aadadf
-F ext/fts5/fts5_config.c 11f969ed711a0a8b611d47431d74c372ad78c713
+F ext/fts5/fts5_buffer.c be0dc80a9406151b350be27c7ec2956722578771
+F ext/fts5/fts5_config.c 6ae691e36f90185896f4db0a819ae2394f880ca1
 F ext/fts5/fts5_expr.c 549bda1f7edcf10365fbfbc002bdea1be3c287bb
 F ext/fts5/fts5_hash.c c1cfdb2cae0fad00b06fae38a40eaf9261563ccc
 F ext/fts5/fts5_index.c 7cea402924cd3d8cd5943a7f9514c9153696571b
-F ext/fts5/fts5_storage.c 770ab52377c18a9aa4dc843ee79388febdb184d4
+F ext/fts5/fts5_storage.c 684ef9575dd1709c3faacbfd1765e623fb1d0505
 F ext/fts5/fts5_tcl.c 7ea165878e4ae3598e89acd470a0ee1b5a00e33c
 F ext/fts5/fts5_tokenize.c 97251d68d7a6a9415bde1203f9382864dfc1f989
 F ext/fts5/fts5_unicode2.c da3cf712f05cd8347c8c5bc00964cc0361c88da9
@@ -127,7 +127,7 @@ F ext/fts5/test/fts5aa.test 0be21c89fd66b588db355a6398911fd875bdcc6c
 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
 F ext/fts5/test/fts5ac.test 0990ae7497ebaea2ab5f7fd5caedd93a71a905fc
 F ext/fts5/test/fts5ad.test 312f3c8ed9592533499c5b94d2059ae6382913a0
-F ext/fts5/test/fts5ae.test 9175201baf8c885fc1cbb2da11a0c61fd11224db
+F ext/fts5/test/fts5ae.test ddc558e3e3b52db0101f7541b2e3849b77052c92
 F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a
 F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505
 F ext/fts5/test/fts5ah.test b9e78fa986a7bd564ebadfb244de02c84d7ac3ae
@@ -139,6 +139,7 @@ F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca
 F ext/fts5/test/fts5aux.test e5631607bbc05ac1c38cf7d691000509aca71ef3
 F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
 F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b
+F ext/fts5/test/fts5columnsize.test c7333cf079022c1ad25d04538b8f279fad4c2f8d
 F ext/fts5/test/fts5config.test c9cc535f3b36cde1e5a32bf579f3f5962a9e82b2
 F ext/fts5/test/fts5content.test e46904decd896e38c848ad4f38fa4e80251a028b
 F ext/fts5/test/fts5corrupt.test 35bfdbbb3cdcea46ae7385f6432e9b5c574e70a1
@@ -1357,7 +1358,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 3a9cb648138a059862fb438c0787fdd888f5e88e
-R 01f96e31f05863ae01feddea5f82854e
+P e964b5877497b16cf985d3d847e82529bb3fa4a3
+R 26fe69b53869c7d4cebecf3b1c47f607
 U dan
-Z 479453162469a9d47c2a3d0d93cd23e2
+Z 7aaa26406a2384f1f8538b962eb28bef
diff --git a/manifest.uuid b/manifest.uuid
index 4112b379fe..da2e7a4579 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e964b5877497b16cf985d3d847e82529bb3fa4a3
\ No newline at end of file
+aa12f9d9b79c2f523fd6b00e47bcb66dba09ce0c
\ No newline at end of file