Optimizations for the matchinfo() function, particularly the 'y' flag.

FossilOrigin-Name: dddd7e182943a1d3a9d32830e819a63f1a228d6d

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index e7958d9c4dec3fcea65d9c4c6f058345a037de04..8673ab7c3eedcec05296243eadae82dd2ab57936 100644 (file)
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -1675,7 +1675,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
   sqlite3Fts3ExprFree(pCsr->pExpr);
   sqlite3Fts3FreeDeferredTokens(pCsr);
   sqlite3_free(pCsr->aDoclist);
-  sqlite3_free(pCsr->aMatchinfo);
+  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
   assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
   sqlite3_free(pCsr);
   return SQLITE_OK;
@@ -3176,7 +3176,7 @@ static int fts3FilterMethod(
   /* In case the cursor has been used before, clear it now. */
   sqlite3_finalize(pCsr->pStmt);
   sqlite3_free(pCsr->aDoclist);
-  sqlite3_free(pCsr->aMatchinfo);
+  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
   sqlite3Fts3ExprFree(pCsr->pExpr);
   memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
 

diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h
index 0748d916d048d27f81bedd22b92eca93feddd1be..9e98c0410f82123ac09ebe7572b2afbd016e220b 100644 (file)
--- a/ext/fts3/fts3Int.h
+++ b/ext/fts3/fts3Int.h
@@ -197,6 +197,8 @@ typedef struct Fts3DeferredToken Fts3DeferredToken;
 typedef struct Fts3SegReader Fts3SegReader;
 typedef struct Fts3MultiSegReader Fts3MultiSegReader;
 
+typedef struct MatchinfoBuffer MatchinfoBuffer;
+
 /*
 ** A connection to a fulltext index is an instance of the following
 ** structure. The xCreate and xConnect methods create an instance
@@ -306,9 +308,7 @@ struct Fts3Cursor {
   i64 iMinDocid;                  /* Minimum docid to return */
   i64 iMaxDocid;                  /* Maximum docid to return */
   int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
-  u32 *aMatchinfo;                /* Information about most recent match */
-  int nMatchinfo;                 /* Number of elements in aMatchinfo[] */
-  char *zMatchinfo;               /* Matchinfo specification */
+  MatchinfoBuffer *pMIBuffer;     /* Buffer for matchinfo data */
 };
 
 #define FTS3_EVAL_FILTER    0
@@ -564,6 +564,7 @@ void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
   const char *, const char *, int, int
 );
 void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
+void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p);
 
 /* fts3_expr.c */
 int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,

diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c
index 6abb169ebfaf7a89867394b2e9edfd202d54d1d3..1eceafa050c8b915f16f9729952e96a5ca6a807b 100644 (file)
--- a/ext/fts3/fts3_snippet.c
+++ b/ext/fts3/fts3_snippet.c
@@ -92,6 +92,18 @@ struct MatchInfo {
   u32 *aMatchinfo;                /* Pre-allocated buffer */
 };
 
+/*
+** An instance of this structure is used to manage a pair of buffers, each
+** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below
+** for details.
+*/
+struct MatchinfoBuffer {
+  u8 aRef[3];
+  int nElem;
+  int bGlobal;                    /* Set if global data is loaded */
+  char *zMatchinfo;
+  u32 aMatchinfo[0];
+};
 
 
 /*
@@ -107,6 +119,97 @@ struct StrBuffer {
 };
 
 
+/*************************************************************************
+** Start of MatchinfoBuffer code.
+*/
+
+/*
+** Allocate a two-slot MatchinfoBuffer object.
+*/
+static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){
+  MatchinfoBuffer *pRet;
+  int nByte = sizeof(u32) * (2*nElem + 2) + sizeof(MatchinfoBuffer);
+  int nStr = strlen(zMatchinfo);
+
+  pRet = sqlite3_malloc(nByte + nStr+1);
+  if( pRet ){
+    memset(pRet, 0, nByte);
+    pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet;
+    pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1);
+    pRet->nElem = nElem;
+    pRet->zMatchinfo = ((char*)pRet) + nByte;
+    memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1);
+    pRet->aRef[0] = 1;
+  }
+
+  return pRet;
+}
+
+static void fts3MIBufferFree(void *p){
+  MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]);
+
+  assert( (u32*)p==&pBuf->aMatchinfo[1] 
+       || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] 
+  );
+  if( (u32*)p==&pBuf->aMatchinfo[1] ){
+    pBuf->aRef[1] = 0;
+  }else{
+    pBuf->aRef[2] = 0;
+  }
+
+  if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){
+    sqlite3_free(pBuf);
+  }
+}
+
+static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){
+  void (*xRet)(void*) = 0;
+  u32 *aOut = 0;
+
+  if( p->aRef[1]==0 ){
+    p->aRef[1] = 1;
+    aOut = &p->aMatchinfo[1];
+    xRet = fts3MIBufferFree;
+  }
+  else if( p->aRef[2]==0 ){
+    p->aRef[2] = 1;
+    aOut = &p->aMatchinfo[p->nElem+2];
+    xRet = fts3MIBufferFree;
+  }else{
+    aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32));
+    if( aOut ){
+      xRet = sqlite3_free;
+      if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32));
+    }
+  }
+
+  *paOut = aOut;
+  return xRet;
+}
+
+static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){
+  p->bGlobal = 1;
+  memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32));
+}
+
+/*
+** Free a MatchinfoBuffer object allocated using fts3MIBufferNew()
+*/
+void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){
+  if( p ){
+    assert( p->aRef[0]==1 );
+    p->aRef[0] = 0;
+    if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){
+      sqlite3_free(p);
+    }
+  }
+}
+
+/* 
+** End of MatchinfoBuffer code.
+*************************************************************************/
+
+
 /*
 ** This function is used to help iterate through a position-list. A position
 ** list is a list of unique integers, sorted from smallest to largest. Each
@@ -133,6 +236,8 @@ static void fts3GetDeltaPosition(char **pp, int *piPos){
   *piPos += (iVal-2);
 }
 
+static int fts3ExprLHitsCb(Fts3Expr*,int,void*);
+
 /*
 ** Helper function for fts3ExprIterate() (see below).
 */
@@ -143,17 +248,21 @@ static int fts3ExprIterate2(
   void *pCtx                      /* Second argument to pass to callback */
 ){
   int rc;                         /* Return code */
-  int eType = pExpr->eType;       /* Type of expression node pExpr */
 
-  if( eType!=FTSQUERY_PHRASE ){
-    assert( pExpr->pLeft && pExpr->pRight );
-    rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx);
-    if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){
-      rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx);
-    }
+  if( x==fts3ExprLHitsCb && pExpr->bEof ){
+    rc = SQLITE_OK;
   }else{
-    rc = x(pExpr, *piPhrase, pCtx);
-    (*piPhrase)++;
+    int eType = pExpr->eType;     /* Type of expression node pExpr */
+    if( eType!=FTSQUERY_PHRASE ){
+      assert( pExpr->pLeft && pExpr->pRight );
+      rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx);
+      if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){
+        rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx);
+      }
+    }else{
+      rc = x(pExpr, *piPhrase, pCtx);
+      (*piPhrase)++;
+    }
   }
   return rc;
 }
@@ -819,23 +928,15 @@ static int fts3ExprLHitsCb(
   int iPhrase,                    /* Phrase number */
   void *pCtx                      /* Pointer to MatchInfo structure */
 ){
-  MatchInfo *p = (MatchInfo *)pCtx;
-  Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
   int rc = SQLITE_OK;
-  int iStart = iPhrase * p->nCol;
-  Fts3Expr *pEof;                 /* Ancestor node already at EOF */
+  MatchInfo *p = (MatchInfo *)pCtx;
   
   /* This must be a phrase */
   assert( pExpr->pPhrase );
 
-  /* Initialize all output integers to zero. */
-  memset(&p->aMatchinfo[iStart], 0, sizeof(u32) * p->nCol);
-
-  /* Check if this or any parent node is at EOF. If so, then all output
-  ** values are zero.  */
-  for(pEof=pExpr; pEof && pEof->bEof==0; pEof=pEof->pParent);
-
-  if( pEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
+  if( pExpr->iDocid==p->pCursor->iPrevId ){
+    Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
+    int iStart = iPhrase * p->nCol;
     Fts3Phrase *pPhrase = pExpr->pPhrase;
     char *pIter = pPhrase->doclist.pList;
     int iCol = 0;
@@ -1149,9 +1250,12 @@ static int fts3MatchinfoValues(
         }
         break;
 
-      case FTS3_MATCHINFO_LHITS:
+      case FTS3_MATCHINFO_LHITS: {
+        int nZero = fts3MatchinfoSize(pInfo, FTS3_MATCHINFO_LHITS)*sizeof(u32);
+        memset(pInfo->aMatchinfo, 0, nZero);
         (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLHitsCb, (void*)pInfo);
         break;
+      }
 
       default: {
         Fts3Expr *pExpr;
@@ -1185,6 +1289,7 @@ static int fts3MatchinfoValues(
 ** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
 */
 static int fts3GetMatchinfo(
+  sqlite3_context *pCtx,        /* Return results here */
   Fts3Cursor *pCsr,               /* FTS3 Cursor object */
   const char *zArg                /* Second argument to matchinfo() function */
 ){
@@ -1193,6 +1298,9 @@ static int fts3GetMatchinfo(
   int rc = SQLITE_OK;
   int bGlobal = 0;                /* Collect 'global' stats as well as local */
 
+  u32 *aOut = 0;
+  void (*xDestroyOut)(void*) = 0;
+
   memset(&sInfo, 0, sizeof(MatchInfo));
   sInfo.pCursor = pCsr;
   sInfo.nCol = pTab->nColumn;
@@ -1200,19 +1308,17 @@ static int fts3GetMatchinfo(
   /* If there is cached matchinfo() data, but the format string for the 
   ** cache does not match the format string for this request, discard 
   ** the cached data. */
-  if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){
-    assert( pCsr->aMatchinfo );
-    sqlite3_free(pCsr->aMatchinfo);
-    pCsr->zMatchinfo = 0;
-    pCsr->aMatchinfo = 0;
+  if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){
+    sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
+    pCsr->pMIBuffer = 0;
   }
 
-  /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
+  /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the
   ** matchinfo function has been called for this query. In this case 
   ** allocate the array used to accumulate the matchinfo data and
   ** initialize those elements that are constant for every row.
   */
-  if( pCsr->aMatchinfo==0 ){
+  if( pCsr->pMIBuffer==0 ){
     int nMatchinfo = 0;           /* Number of u32 elements in match-info */
     int nArg;                     /* Bytes in zArg */
     int i;                        /* Used to iterate through zArg */
@@ -1227,23 +1333,35 @@ static int fts3GetMatchinfo(
     }
 
     /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
-    nArg = (int)strlen(zArg);
-    pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1);
-    if( !pCsr->aMatchinfo ) return SQLITE_NOMEM;
-
-    pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo];
-    pCsr->nMatchinfo = nMatchinfo;
-    memcpy(pCsr->zMatchinfo, zArg, nArg+1);
-    memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
+    pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg);
+    if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM;
+
     pCsr->isMatchinfoNeeded = 1;
     bGlobal = 1;
   }
 
-  sInfo.aMatchinfo = pCsr->aMatchinfo;
-  sInfo.nPhrase = pCsr->nPhrase;
-  if( pCsr->isMatchinfoNeeded ){
+  if( rc==SQLITE_OK ){
+    xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut);
+    if( xDestroyOut==0 ){
+      rc = SQLITE_NOMEM;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    sInfo.aMatchinfo = aOut;
+    sInfo.nPhrase = pCsr->nPhrase;
     rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
-    pCsr->isMatchinfoNeeded = 0;
+    if( bGlobal ){
+      fts3MIBufferSetGlobal(pCsr->pMIBuffer);
+    }
+  }
+
+  if( rc!=SQLITE_OK ){
+    sqlite3_result_error_code(pCtx, rc);
+    if( xDestroyOut ) xDestroyOut(aOut);
+  }else{
+    int n = pCsr->pMIBuffer->nElem * sizeof(u32);
+    sqlite3_result_blob(pCtx, aOut, n, xDestroyOut);
   }
 
   return rc;
@@ -1568,15 +1686,8 @@ void sqlite3Fts3Matchinfo(
   }
 
   /* Retrieve matchinfo() data. */
-  rc = fts3GetMatchinfo(pCsr, zFormat);
+  rc = fts3GetMatchinfo(pContext, pCsr, zFormat);
   sqlite3Fts3SegmentsClose(pTab);
-
-  if( rc!=SQLITE_OK ){
-    sqlite3_result_error_code(pContext, rc);
-  }else{
-    int n = pCsr->nMatchinfo * sizeof(u32);
-    sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
-  }
 }
 
 #endif

diff --git a/manifest b/manifest
index 7b6957d36aa952f6b5df2281eaf2854adf3d955c..738c7e07a1596d15cfbb535961020aa8a3a488af 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\s#ifdef\sstatements\sto\stest_blob.c\sso\sthat\sit\swill\sbuild\nwith\sSQLITE_OMIT_INCRBLOB.
-D 2015-05-05T11:08:02.278
+C Optimizations\sfor\sthe\smatchinfo()\sfunction,\sparticularly\sthe\s'y'\sflag.
+D 2015-05-05T19:37:07.819
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 08728ecbeddca339c77bfd564d3484b523dffdb1
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -78,16 +78,16 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 2fb98467f4b670c8934cdd97d1ba3ffa7382764c
+F ext/fts3/fts3.c 341e9d9a3c7615bac8e815a8937d576265b22f78
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h 59ecaa2d7af0da44c70b6aeaebdcfc070d14abab
+F ext/fts3/fts3Int.h bf61766eeb57a6922a8458b894d85e50d1cfb04e
 F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1
 F ext/fts3/fts3_expr.c 71c063da9c2a4167fb54aec089dd5ef33a58c9cb
 F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60
 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf
 F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5
 F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009
-F ext/fts3/fts3_snippet.c 40a96ba78e90aba7d7d6d014a18049bb218060fd
+F ext/fts3/fts3_snippet.c a1c62f1b7c55d14a13385689ce11aa0e1ada55b8
 F ext/fts3/fts3_term.c 88c55a6fa1a51ab494e33dced0401a6c28791fd7
 F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038
 F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860
@@ -1256,7 +1256,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P d2cb1becc07fad5cbd48c206c676493ba90cada1
-R e1db2a6afd30f4051a55266b8de5e117
-U drh
-Z df7e75264a470014004f32463950df05
+P b8f090e65d010c62df335d0520a36a24904e8bc6
+R db4b4f916c43503b3d5880cc35083941
+T *branch * fts3-matchinfo-y
+T *sym-fts3-matchinfo-y *
+T -sym-trunk *
+U dan
+Z 2cd9b208c5989e901293be093105f297

diff --git a/manifest.uuid b/manifest.uuid
index c9040b853e27d47a5a54ea9df1ffd8a833ee8a9e..c76ab92b5067d77c528f91f0ebe29408a5540a87 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b8f090e65d010c62df335d0520a36a24904e8bc6
\ No newline at end of file
+dddd7e182943a1d3a9d32830e819a63f1a228d6d
\ No newline at end of file