From: dan <dan@noemail.net>
Date: Wed, 23 Jul 2014 19:31:56 +0000 (+0000)
Subject: Add a snippet() function to fts5.
X-Git-Tag: version-3.8.11~114^2~157
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b30860abe2056bd1d47c23227a0b1d35cbc27b13;p=thirdparty%2Fsqlite.git

Add a snippet() function to fts5.

FossilOrigin-Name: bdc58fd28a63ac9632c3df6c7768a9a236566605
---

diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c
index a3876671a3..14da56ec40 100644
--- a/ext/fts5/fts5.c
+++ b/ext/fts5/fts5.c
@@ -686,13 +686,12 @@ static int fts5ApiPoslist(
   Fts5Context *pCtx, 
   int iPhrase, 
   int *pi, 
-  int *piCol, 
-  int *piOff
+  i64 *piPos 
 ){
   Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
   const u8 *a; int n;             /* Poslist for phrase iPhrase */
   n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, &a);
-  return sqlite3Fts5PoslistNext(a, n, pi, piCol, piOff);
+  return sqlite3Fts5PoslistNext64(a, n, pi, piPos);
 }
 
 static void fts5ApiCallback(
diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h
index 572aef03c8..4d45ee60b0 100644
--- a/ext/fts5/fts5.h
+++ b/ext/fts5/fts5.h
@@ -69,6 +69,8 @@ typedef void (*fts5_extension_function)(
 ** xPoslist:
 **   Iterate through instances of phrase iPhrase in the current row. 
 **
+**   At EOF, a non-zero value is returned and output variable iPos set to -1.
+**
 ** xTokenize:
 **   Tokenize text using the tokenizer belonging to the FTS5 table.
 */
@@ -91,9 +93,12 @@ struct Fts5ExtensionApi {
   sqlite3_int64 (*xRowid)(Fts5Context*);
   int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
   int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
-  int (*xPoslist)(Fts5Context*, int iPhrase, int *pi, int *piCol, int *piOff);
+  int (*xPoslist)(Fts5Context*, int iPhrase, int *pi, sqlite3_int64 *piPos);
 };
 
+#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
+#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)
+
 /* 
 ** CUSTOM AUXILIARY FUNCTIONS
 *************************************************************************/
diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index 25a1747fdb..faee34c82d 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -13,12 +13,413 @@
 
 #include "fts5Int.h"
 
+typedef struct SnippetPhrase SnippetPhrase;
+typedef struct SnippetIter SnippetIter;
+typedef struct SnippetCtx SnippetCtx;
+
+struct SnippetPhrase {
+  u64 mask;                       /* Current mask */
+  int nToken;                     /* Tokens in this phrase */
+  int i;                          /* Current offset in phrase poslist */
+  i64 iPos;                       /* Next position in phrase (-ve -> EOF) */
+};
+
+struct SnippetIter {
+  i64 iLast;                      /* Last token position of current snippet */
+  int nScore;                     /* Score of current snippet */
+
+  const Fts5ExtensionApi *pApi;
+  Fts5Context *pFts;
+  u64 szmask;                     /* Mask used to on SnippetPhrase.mask */
+  int nPhrase;                    /* Number of phrases */
+  SnippetPhrase aPhrase[0];       /* Array of size nPhrase */
+};
+
+struct SnippetCtx {
+  int iFirst;                     /* Offset of first token to record */
+  int nToken;                     /* Size of aiStart[] and aiEnd[] arrays */
+  int iSeen;                      /* Set to largest offset seen */
+  int *aiStart; 
+  int *aiEnd;
+};
+
+static int fts5SnippetCallback(
+  void *pContext,                 /* Pointer to Fts5Buffer object */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStart,                     /* Start offset of token */
+  int iEnd,                       /* End offset of token */
+  int iPos                        /* Position offset of token */
+){
+  int rc = SQLITE_OK;
+  SnippetCtx *pCtx = (SnippetCtx*)pContext;
+  int iOff = iPos - pCtx->iFirst;
+
+  if( iOff>=0 ){
+    if( iOff < pCtx->nToken ){
+      pCtx->aiStart[iOff] = iStart;
+      pCtx->aiEnd[iOff] = iEnd;
+    }
+    pCtx->iSeen = iPos;
+    if( iOff>=pCtx->nToken ) rc = SQLITE_DONE;
+  }
+
+  return rc;
+}
+
+/*
+** Set pIter->nScore to the score for the current entry.
+*/
+static void fts5SnippetCalculateScore(SnippetIter *pIter){
+  int i;
+  int nScore = 0;
+  assert( pIter->iLast>=0 );
+
+  for(i=0; i<pIter->nPhrase; i++){
+    SnippetPhrase *p = &pIter->aPhrase[i];
+    u64 mask = p->mask;
+    if( mask ){
+      u64 j;
+      nScore += 1000;
+      for(j=1; j & pIter->szmask; j<<=1){
+        if( mask & j ) nScore++;
+      }
+    }
+  }
+
+  pIter->nScore = nScore;
+}
+
+/*
+** Allocate a new snippet iter.
+*/
+static int fts5SnippetIterNew(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  int nToken,                     /* Number of tokens in snippets */
+  SnippetIter **ppIter            /* OUT: New object */
+){
+  int i;                          /* Counter variable */
+  SnippetIter *pIter;             /* New iterator object */
+  int nByte;                      /* Bytes of space to allocate */
+  int nPhrase;                    /* Number of phrases in query */
+
+  *ppIter = 0;
+  nPhrase = pApi->xPhraseCount(pFts);
+  nByte = sizeof(SnippetIter) + nPhrase * sizeof(SnippetPhrase);
+  pIter = (SnippetIter*)sqlite3_malloc(nByte);
+  if( pIter==0 ) return SQLITE_NOMEM;
+  memset(pIter, 0, nByte);
+
+  pIter->nPhrase = nPhrase;
+  pIter->pApi = pApi;
+  pIter->pFts = pFts;
+  pIter->szmask = ((u64)1 << nToken) - 1;
+  assert( nToken<=63 );
+
+  for(i=0; i<nPhrase; i++){
+    pIter->aPhrase[i].nToken = pApi->xPhraseSize(pFts, i);
+  }
+
+  *ppIter = pIter;
+  return SQLITE_OK;
+}
+
+/*
+** Set the iterator to point to the first candidate snippet.
+*/
+static void fts5SnippetIterFirst(SnippetIter *pIter){
+  const Fts5ExtensionApi *pApi = pIter->pApi;
+  Fts5Context *pFts = pIter->pFts;
+  int i;                          /* Used to iterate through phrases */
+  SnippetPhrase *pMin = 0;        /* Phrase with first match */
+
+  memset(pIter->aPhrase, 0, sizeof(SnippetPhrase) * pIter->nPhrase);
+
+  for(i=0; i<pIter->nPhrase; i++){
+    SnippetPhrase *p = &pIter->aPhrase[i];
+    p->nToken = pApi->xPhraseSize(pFts, i);
+    pApi->xPoslist(pFts, i, &p->i, &p->iPos);
+    if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ){
+      pMin = p;
+    }
+  }
+  assert( pMin );
+
+  pIter->iLast = pMin->iPos + pMin->nToken - 1;
+  pMin->mask = 0x01;
+  pApi->xPoslist(pFts, pMin - pIter->aPhrase, &pMin->i, &pMin->iPos);
+  fts5SnippetCalculateScore(pIter);
+}
+
+/*
+** Advance the snippet iterator to the next candidate snippet.
+*/
+static void fts5SnippetIterNext(SnippetIter *pIter){
+  const Fts5ExtensionApi *pApi = pIter->pApi;
+  Fts5Context *pFts = pIter->pFts;
+  int nPhrase = pIter->nPhrase;
+  int i;                          /* Used to iterate through phrases */
+  SnippetPhrase *pMin = 0;
+
+  for(i=0; i<nPhrase; i++){
+    SnippetPhrase *p = &pIter->aPhrase[i];
+    if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ) pMin = p;
+  }
+
+  if( pMin==0 ){
+    /* pMin==0 indicates that the SnippetIter is at EOF. */
+    pIter->iLast = -1;
+  }else{
+    i64 nShift = pMin->iPos - pIter->iLast;
+    assert( nShift>=0 );
+    for(i=0; i<nPhrase; i++){
+      SnippetPhrase *p = &pIter->aPhrase[i];
+      if( nShift>=63 ){
+        p->mask = 0;
+      }else{
+        p->mask = p->mask << (int)nShift;
+        p->mask &= pIter->szmask;
+      }
+    }
+
+    pIter->iLast = pMin->iPos;
+    pMin->mask |= 0x01;
+    fts5SnippetCalculateScore(pIter);
+    pApi->xPoslist(pFts, pMin - pIter->aPhrase, &pMin->i, &pMin->iPos);
+  }
+}
+
+static void fts5SnippetIterFree(SnippetIter *pIter){
+  if( pIter ){
+    sqlite3_free(pIter);
+  }
+}
+
+static int fts5SnippetText(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  SnippetIter *pIter,             /* Snippet to write to buffer */
+  int nToken,                     /* Size of desired snippet in tokens */
+  const char *zStart,
+  const char *zFinal,
+  const char *zEllip,
+  Fts5Buffer *pBuf                /* Write output to this buffer */
+){
+  SnippetCtx ctx;
+  int i;
+  u64 all = 0;
+  const char *zCol;               /* Column text to extract snippet from */
+  int nCol;                       /* Size of column text in bytes */
+  int rc;
+  int nShift;
+
+  rc = pApi->xColumnText(pFts, FTS5_POS2COLUMN(pIter->iLast), &zCol, &nCol);
+  if( rc!=SQLITE_OK ) return rc;
+
+  /* At this point pIter->iLast is the offset of the last token in the
+  ** proposed snippet. However, in all cases pIter->iLast contains the
+  ** final token of one of the phrases. This makes the snippet look
+  ** unbalanced. For example:
+  **
+  **     "...x x x x x <b>term</b>..."
+  **
+  ** It is better to increase iLast a little so that the snippet looks
+  ** more like:
+  **
+  **     "...x x x <b>term</b> y y..."
+  **
+  ** The problem is that there is no easy way to discover whether or not
+  ** how many tokens are present in the column following "term". 
+  */
+
+  /* Set variable nShift to the number of tokens by which the snippet
+  ** should be shifted, assuming there are sufficient tokens to the right
+  ** of iLast in the column value.  */
+  for(i=0; i<pIter->nPhrase; i++){
+    int iToken;
+    for(iToken=0; iToken<pIter->aPhrase[i].nToken; iToken++){
+      all |= (pIter->aPhrase[i].mask << iToken);
+    }
+  }
+  for(i=nToken-1; i>=0; i--){
+    if( all & ((u64)1 << i) ) break;
+  }
+  assert( i>=0 );
+  nShift = (nToken - i) / 2;
+
+  memset(&ctx, 0, sizeof(SnippetCtx));
+  ctx.nToken = nToken + nShift;
+  ctx.iFirst = FTS5_POS2OFFSET(pIter->iLast) - nToken + 1;
+  if( ctx.iFirst<0 ){
+    nShift += ctx.iFirst;
+    if( nShift<0 ) nShift = 0;
+    ctx.iFirst = 0;
+  }
+  ctx.aiStart = (int*)sqlite3_malloc(sizeof(int) * ctx.nToken * 2);
+  if( ctx.aiStart==0 ) return SQLITE_NOMEM;
+  ctx.aiEnd = &ctx.aiStart[ctx.nToken];
+
+  rc = pApi->xTokenize(pFts, zCol, nCol, (void*)&ctx, fts5SnippetCallback);
+  if( rc==SQLITE_OK ){
+    int i1;                       /* First token from input to include */
+    int i2;                       /* Last token from input to include */
+
+    int iPrint;
+    int iMatchto;
+    int iBit0;
+    int iLast;
+
+    int *aiStart = ctx.aiStart - ctx.iFirst;
+    int *aiEnd = ctx.aiEnd - ctx.iFirst;
+
+    /* Ideally we want to start the snippet with token (ctx.iFirst + nShift).
+    ** However, this is only possible if there are sufficient tokens within
+    ** the column. This block sets variables i1 and i2 to the first and last
+    ** input tokens to include in the snippet.  */
+    if( (ctx.iFirst + nShift + nToken)<=ctx.iSeen ){
+      i1 = ctx.iFirst + nShift;
+      i2 = i1 + nToken - 1;
+    }else{
+      i2 = ctx.iSeen;
+      i1 = ctx.iSeen - nToken + 1;
+      assert( i1>=0 || ctx.iFirst==0 );
+      if( i1<0 ) i1 = 0;
+    }
+
+    /* If required, append the preceding ellipsis. */
+    if( i1>0 ) sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%s", zEllip);
+
+    iLast = FTS5_POS2OFFSET(pIter->iLast);
+    iPrint = i1;
+    iMatchto = -1;
+
+    for(i=i1; i<=i2; i++){
+
+      /* Check if this is the first token of any phrase match. */
+      int ip;
+      for(ip=0; ip<pIter->nPhrase; ip++){
+        SnippetPhrase *pPhrase = &pIter->aPhrase[ip];
+        u64 m = (1 << (iLast - i - pPhrase->nToken + 1));
+
+        if( i<=iLast && (pPhrase->mask & m) ){
+          if( iMatchto<0 ){
+            sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s%s",
+                aiStart[i] - aiStart[iPrint],
+                &zCol[aiStart[iPrint]],
+                zStart
+            );
+            iPrint = i;
+          }
+          if( i>iMatchto ) iMatchto = i + pPhrase->nToken - 1;
+        }
+      }
+
+      if( i==iMatchto ){
+        sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s%s",
+            aiEnd[i] - aiStart[iPrint],
+            &zCol[aiStart[iPrint]],
+            zFinal
+        );
+        iMatchto = -1;
+        iPrint = i+1;
+
+        if( i<i2 ){
+          sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s",
+              aiStart[i+1] - aiEnd[i],
+              &zCol[aiEnd[i]]
+          );
+        }
+      }
+    }
+
+    if( iPrint<=i2 ){
+      sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s", 
+          aiEnd[i2] - aiStart[iPrint], 
+          &zCol[aiStart[iPrint]]
+      );
+      if( iMatchto>=0 ){
+        sqlite3Fts5BufferAppendString(&rc, pBuf, zFinal);
+      }
+    }
+
+    /* If required, append the trailing ellipsis. */
+    if( i2<ctx.iSeen ) sqlite3Fts5BufferAppendString(&rc, pBuf, zEllip);
+  }
+
+  sqlite3_free(ctx.aiStart);
+  return rc;
+}
+
+/*
+** A default snippet() implementation. This is compatible with the FTS3
+** snippet() function.
+*/
 static void fts5SnippetFunction(
   const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
   Fts5Context *pFts,              /* First arg to pass to pApi functions */
   sqlite3_context *pCtx,          /* Context for returning result/error */
   int nVal,                       /* Number of values in apVal[] array */
   sqlite3_value **apVal           /* Array of trailing arguments */
+){
+  const char *zStart = "<b>";
+  const char *zFinal = "</b>";
+  const char *zEllip = "<b>...</b>";
+  int nToken = -15;
+  int nAbs;
+  int nFrag;                      /* Number of fragments to return */
+  int rc;
+  SnippetIter *pIter = 0;
+
+  if( nVal>=1 ) zStart = (const char*)sqlite3_value_text(apVal[0]);
+  if( nVal>=2 ) zFinal = (const char*)sqlite3_value_text(apVal[1]);
+  if( nVal>=3 ) zEllip = (const char*)sqlite3_value_text(apVal[2]);
+  if( nVal>=4 ){
+    nToken = sqlite3_value_int(apVal[3]);
+    if( nToken==0 ) nToken = -15;
+  }
+  nAbs = nToken * (nToken<0 ? -1 : 1);
+
+  rc = fts5SnippetIterNew(pApi, pFts, nAbs, &pIter);
+  if( rc==SQLITE_OK ){
+    Fts5Buffer buf;               /* Result buffer */
+    int nBestScore = 0;           /* Score of best snippet found */
+    int n;                        /* Size of column snippet is from in bytes */
+    int i;                        /* Used to iterate through phrases */
+
+    for(fts5SnippetIterFirst(pIter); 
+        pIter->iLast>=0; 
+        fts5SnippetIterNext(pIter)
+    ){
+      if( pIter->nScore>nBestScore ) nBestScore = pIter->nScore;
+    }
+    for(fts5SnippetIterFirst(pIter); 
+        pIter->iLast>=0; 
+        fts5SnippetIterNext(pIter)
+    ){
+      if( pIter->nScore==nBestScore ) break;
+    }
+
+    memset(&buf, 0, sizeof(Fts5Buffer));
+    rc = fts5SnippetText(pApi, pFts, pIter, nAbs, zStart, zFinal, zEllip, &buf);
+    if( rc==SQLITE_OK ){
+      sqlite3_result_text(pCtx, (const char*)buf.p, buf.n, SQLITE_TRANSIENT);
+    }
+    sqlite3_free(buf.p);
+  }
+
+  fts5SnippetIterFree(pIter);
+  if( rc!=SQLITE_OK ){
+    sqlite3_result_error_code(pCtx, rc);
+  }
+}
+
+static void fts5Bm25Function(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  sqlite3_context *pCtx,          /* Context for returning result/error */
+  int nVal,                       /* Number of values in apVal[] array */
+  sqlite3_value **apVal           /* Array of trailing arguments */
 ){
   assert( 0 );
 }
@@ -146,12 +547,13 @@ static void fts5TestFunction(
     for(i=0; i<nPhrase; i++){
       Fts5Buffer s2;                  /* List of positions for phrase/column */
       int j = 0;
-      int iOff = 0;
-      int iCol = 0;
+      i64 iPos = 0;
       int nElem = 0;
 
       memset(&s2, 0, sizeof(s2));
-      while( 0==pApi->xPoslist(pFts, i, &j, &iCol, &iOff) ){
+      while( 0==pApi->xPoslist(pFts, i, &j, &iPos) ){
+        int iOff = FTS5_POS2OFFSET(iPos);
+        int iCol = FTS5_POS2COLUMN(iPos);
         if( nElem!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s2, " ");
         sqlite3Fts5BufferAppendPrintf(&rc, &s2, "%d.%d", iCol, iOff);
         nElem++;
diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c
index b798ad2119..bea316eda4 100644
--- a/ext/fts5/fts5_buffer.c
+++ b/ext/fts5/fts5_buffer.c
@@ -146,6 +146,7 @@ int sqlite3Fts5PoslistNext64(
   int i = *pi;
   if( i>=n ){
     /* EOF */
+    *piOff = -1;
     return 1;  
   }else{
     i64 iOff = *piOff;
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 06faf7ebff..9eea4552bf 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -389,7 +389,6 @@ static int fts5ExprNearIsMatch(Fts5ExprNearset *pNear, int *pbMatch){
   int i;
   int rc = SQLITE_OK;
   int bMatch;
-  i64 iMax;
 
   assert( pNear->nPhrase>1 );
 
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index 13e1f08082..c56c2d038a 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -486,6 +486,29 @@ int sqlite3Fts5StorageInsert(
   return rc;
 }
 
+static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){
+  Fts5Config *pConfig = p->pConfig;
+  char *zSql;
+  int rc;
+
+  zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'", 
+      pConfig->zDb, pConfig->zName, zSuffix
+  );
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    sqlite3_stmt *pCnt = 0;
+    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0);
+    if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCnt) ){
+      *pnRow = sqlite3_column_int64(pCnt, 0);
+    }
+    rc = sqlite3_finalize(pCnt);
+  }
+
+  sqlite3_free(zSql);
+  return rc;
+}
+
 /*
 ** Context object used by sqlite3Fts5StorageIntegrity().
 */
@@ -493,6 +516,7 @@ typedef struct Fts5IntegrityCtx Fts5IntegrityCtx;
 struct Fts5IntegrityCtx {
   i64 iRowid;
   int iCol;
+  int szCol;
   u64 cksum;
   Fts5Config *pConfig;
 };
@@ -512,6 +536,7 @@ static int fts5StorageIntegrityCallback(
   pCtx->cksum ^= sqlite3Fts5IndexCksum(
       pCtx->pConfig, pCtx->iRowid, pCtx->iCol, iPos, pToken, nToken
   );
+  pCtx->szCol = iPos+1;
   return SQLITE_OK;
 }
 
@@ -524,11 +549,17 @@ static int fts5StorageIntegrityCallback(
 int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
   Fts5Config *pConfig = p->pConfig;
   int rc;                         /* Return code */
+  int *aColSize;                  /* Array of size pConfig->nCol */
+  i64 *aTotalSize;                /* Array of size pConfig->nCol */
   Fts5IntegrityCtx ctx;
   sqlite3_stmt *pScan;
 
   memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
   ctx.pConfig = p->pConfig;
+  aTotalSize = (i64*)sqlite3_malloc(pConfig->nCol * (sizeof(int)+sizeof(i64)));
+  if( !aTotalSize ) return SQLITE_NOMEM;
+  aColSize = (int*)&aTotalSize[pConfig->nCol];
+  memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
 
   /* Generate the expected index checksum based on the contents of the
   ** %_content table. This block stores the checksum in ctx.cksum. */
@@ -538,6 +569,8 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
     while( SQLITE_ROW==sqlite3_step(pScan) ){
       int i;
       ctx.iRowid = sqlite3_column_int64(pScan, 0);
+      ctx.szCol = 0;
+      rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
       for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
         ctx.iCol = i;
         rc = sqlite3Fts5Tokenize(
@@ -547,12 +580,37 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
             (void*)&ctx,
             fts5StorageIntegrityCallback
         );
+        if( ctx.szCol!=aColSize[i] ) rc = SQLITE_CORRUPT_VTAB;
+        aTotalSize[i] += ctx.szCol;
       }
+      if( rc!=SQLITE_OK ) break;
     }
     rc2 = sqlite3_reset(pScan);
     if( rc==SQLITE_OK ) rc = rc2;
   }
 
+  /* Test that the "totals" (sometimes called "averages") record looks Ok */
+  if( rc==SQLITE_OK ){
+    int i;
+    rc = fts5StorageLoadTotals(p);
+    for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
+      if( p->aTotalSize[i]!=aTotalSize[i] ) rc = SQLITE_CORRUPT_VTAB;
+    }
+  }
+
+  /* Check that the %_docsize and %_content tables contain the expected
+  ** number of rows.  */
+  if( rc==SQLITE_OK ){
+    i64 nRow;
+    rc = fts5StorageCount(p, "content", &nRow);
+    if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = SQLITE_CORRUPT_VTAB;
+  }
+  if( rc==SQLITE_OK ){
+    i64 nRow;
+    rc = fts5StorageCount(p, "docsize", &nRow);
+    if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = SQLITE_CORRUPT_VTAB;
+  }
+
   /* Pass the expected checksum down to the FTS index module. It will
   ** verify, amongst other things, that it matches the checksum generated by
   ** inspecting the index itself.  */
@@ -560,6 +618,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
     rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum);
   }
 
+  sqlite3_free(aTotalSize);
   return rc;
 }
 
diff --git a/manifest b/manifest
index a9ec4b7137..ec56664116 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sDELETE\sand\sUPDATE\soperations\son\sfts5\stables.
-D 2014-07-21T15:45:26.584
+C Add\sa\ssnippet()\sfunction\sto\sfts5.
+D 2014-07-23T19:31:56.454
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -103,15 +103,15 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
-F ext/fts5/fts5.c 35124fe8a49868808604c6a5264bf4f23587ac99
-F ext/fts5/fts5.h c77b6a4a56d80f70fc4f0444030c88724397ed10
+F ext/fts5/fts5.c 6f859d444eb8be46cb3f7aba3aaae369c5b26809
+F ext/fts5/fts5.h 57325b418b26dcd60be5bc8aab05b33767d81590
 F ext/fts5/fts5Int.h 12d03496152b716e63a5380e396b776fbefa2065
-F ext/fts5/fts5_aux.c 14961135231dd50e6c17894e649c3bbc8c042829
-F ext/fts5/fts5_buffer.c 00361d4a70040ebd2c32bc349ab708ff613a1749
+F ext/fts5/fts5_aux.c cba929fb13931c9b8be7d572991e648b98f14cf2
+F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00
 F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
-F ext/fts5/fts5_expr.c 288b3e016253eab69ea8cefbff346a4697b44291
+F ext/fts5/fts5_expr.c 2911813db174afa28b69ccc7031b6dd80293b241
 F ext/fts5/fts5_index.c 68d2d41b5c6d2f8838c3d6ebdc8b242718b8e997
-F ext/fts5/fts5_storage.c 7bb34138d134841cbe0a809467070d07013d8d7d
+F ext/fts5/fts5_storage.c 9a2744f492413395a0e75f20c19b797c801a7308
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb
@@ -600,6 +600,7 @@ F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397
 F test/fts5ac.test 9be418d037763f4cc5d86f4239db41fc86bb4f85
 F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07
 F test/fts5ae.test fe9db78201bbb87c6f82b72a14b946d0f7fc3026
+F test/fts5af.test a2980528a04b67ac4690e6c02ebe9455f45c9a35
 F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
@@ -767,7 +768,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d
 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54
-F test/permutations.test c3eb62a88337d9a5046c509dd90ba6d43debc76d
+F test/permutations.test e6e951c816199693676d8c3d22d6bf54bcd719fc
 F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0
 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13
 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
@@ -1195,7 +1196,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 8c6b0aff3443fae4b7f0b9adcbf1514992b70653
-R ceadf07a7e21667d721f7d8d3cf93a59
+P d44d3a8518ff7a1a3e2c0ab97493aa590676ee8c
+R 638d6826a594d773b5778bd6943c3d96
 U dan
-Z 2d0ad058324af420450f942c4237ae4f
+Z 30db824dafb73f9c4c6895383aa25ed9
diff --git a/manifest.uuid b/manifest.uuid
index f831f113b7..990c947a9e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d44d3a8518ff7a1a3e2c0ab97493aa590676ee8c
\ No newline at end of file
+bdc58fd28a63ac9632c3df6c7768a9a236566605
\ No newline at end of file
diff --git a/test/fts5af.test b/test/fts5af.test
new file mode 100644
index 0000000000..cd5f91f13e
--- /dev/null
+++ b/test/fts5af.test
@@ -0,0 +1,138 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+# 
+# More specifically, the tests in this file focus on the built-in 
+# snippet() function.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts5af
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y);
+}
+
+
+foreach {tn doc res} {
+
+  1.1 {X o o o o o o} {[X] o o o o o o}
+  1.2 {o X o o o o o} {o [X] o o o o o}
+  1.3 {o o X o o o o} {o o [X] o o o o}
+  1.4 {o o o X o o o} {o o o [X] o o o}
+  1.5 {o o o o X o o} {o o o o [X] o o}
+  1.6 {o o o o o X o} {o o o o o [X] o}
+  1.7 {o o o o o o X} {o o o o o o [X]}
+
+  2.1 {X o o o o o o o} {[X] o o o o o o...}
+  2.2 {o X o o o o o o} {o [X] o o o o o...}
+  2.3 {o o X o o o o o} {o o [X] o o o o...}
+  2.4 {o o o X o o o o} {o o o [X] o o o...}
+  2.5 {o o o o X o o o} {...o o o [X] o o o}
+  2.6 {o o o o o X o o} {...o o o o [X] o o}
+  2.7 {o o o o o o X o} {...o o o o o [X] o}
+  2.8 {o o o o o o o X} {...o o o o o o [X]}
+
+  3.1 {X o o o o o o o o} {[X] o o o o o o...}
+  3.2 {o X o o o o o o o} {o [X] o o o o o...}
+  3.3 {o o X o o o o o o} {o o [X] o o o o...}
+  3.4 {o o o X o o o o o} {o o o [X] o o o...}
+  3.5 {o o o o X o o o o} {...o o o [X] o o o...}
+  3.6 {o o o o o X o o o} {...o o o [X] o o o}
+  3.7 {o o o o o o X o o} {...o o o o [X] o o}
+  3.8 {o o o o o o o X o} {...o o o o o [X] o}
+  3.9 {o o o o o o o o X} {...o o o o o o [X]}
+
+  4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
+  4.2 {o X o o o o o X o} {...[X] o o o o o [X]...}
+  4.3 {o o X o o o o o X} {...[X] o o o o o [X]}
+
+  5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
+  5.2 {o X o o o o X o o} {...[X] o o o o [X] o...}
+  5.3 {o o X o o o o X o} {...[X] o o o o [X] o}
+  5.4 {o o o X o o o o X} {...o [X] o o o o [X]}
+
+  6.1 {X o o o X o o o} {[X] o o o [X] o o...}
+  6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
+  6.3 {o o X o o o X o o} {...o [X] o o o [X] o...}
+  6.4 {o o o X o o o X o} {...o [X] o o o [X] o}
+  6.5 {o o o o X o o o X} {...o o [X] o o o [X]}
+
+  7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
+  7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
+  7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
+  7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
+  7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
+  7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
+} {
+  do_execsql_test 1.$tn.1 {
+    DELETE FROM t1;
+    INSERT INTO t1 VALUES($doc, NULL);
+    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH 'X';
+  } [list $res]
+
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t1;
+    INSERT INTO t1 VALUES(NULL, $doc);
+    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH 'X';
+  } [list $res]
+}
+
+foreach {tn doc res} {
+  1.1 {X Y o o o o o} {[X Y] o o o o o}
+  1.2 {o X Y o o o o} {o [X Y] o o o o}
+  1.3 {o o X Y o o o} {o o [X Y] o o o}
+  1.4 {o o o X Y o o} {o o o [X Y] o o}
+  1.5 {o o o o X Y o} {o o o o [X Y] o}
+  1.6 {o o o o o X Y} {o o o o o [X Y]}
+
+  2.1 {X Y o o o o o o} {[X Y] o o o o o...}
+  2.2 {o X Y o o o o o} {o [X Y] o o o o...}
+  2.3 {o o X Y o o o o} {o o [X Y] o o o...}
+  2.4 {o o o X Y o o o} {...o o [X Y] o o o}
+  2.5 {o o o o X Y o o} {...o o o [X Y] o o}
+  2.6 {o o o o o X Y o} {...o o o o [X Y] o}
+  2.7 {o o o o o o X Y} {...o o o o o [X Y]}
+
+  3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
+  3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
+  3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
+  3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
+  3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
+  3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
+  3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
+  3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
+
+} {
+  do_execsql_test 2.$tn.1 {
+    DELETE FROM t1;
+    INSERT INTO t1 VALUES($doc, NULL);
+    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH 'X+Y';
+  } [list $res]
+
+  do_execsql_test 2.$tn.2 {
+    DELETE FROM t1;
+    INSERT INTO t1 VALUES(NULL, $doc);
+    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH 'X+Y';
+  } [list $res]
+}
+
+finish_test
+
diff --git a/test/permutations.test b/test/permutations.test
index 308d521f0e..38c8321e3a 100644
--- a/test/permutations.test
+++ b/test/permutations.test
@@ -226,6 +226,7 @@ test_suite "fts5" -prefix "" -description {
   All FTS5 tests.
 } -files {
   fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ae.test fts5ea.test
+  fts5af.test
 }
 
 test_suite "nofaultsim" -prefix "" -description {