From: dan <dan@noemail.net>
Date: Fri, 27 Feb 2015 07:23:26 +0000 (+0000)
Subject: Fix suffix and prefix compression of terms in top-level fts5 segments. And a crash... 
X-Git-Tag: version-3.8.11~114^2~83
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=5db7e2ca83d6650bb480870cc40e87b68d8154e1;p=thirdparty%2Fsqlite.git

Fix suffix and prefix compression of terms in top-level fts5 segments. And a crash that could follow an OOM condition.

FossilOrigin-Name: bb104b3646c6f07ed002be7360b08433ee7980d4
---

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 90df81a016..f31548ee21 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -400,7 +400,7 @@ int sqlite3Fts5HashQuery(
   int *pnDoclist                  /* OUT: Size of doclist in bytes */
 );
 
-void sqlite3Fts5HashScanInit(
+int sqlite3Fts5HashScanInit(
   Fts5Hash*,                      /* Hash table to query */
   const char *pTerm, int nTerm    /* Query prefix */
 );
diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c
index eb1f3d1b1a..7c63fad19c 100644
--- a/ext/fts5/fts5_hash.c
+++ b/ext/fts5/fts5_hash.c
@@ -421,11 +421,11 @@ int sqlite3Fts5HashQuery(
   return SQLITE_OK;
 }
 
-void sqlite3Fts5HashScanInit(
+int sqlite3Fts5HashScanInit(
   Fts5Hash *p,                    /* Hash table to query */
   const char *pTerm, int nTerm    /* Query prefix */
 ){
-  fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
+  return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
 }
 
 void sqlite3Fts5HashScanNext(Fts5Hash *p){
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 481cd75fa7..79c6d007de 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -2079,9 +2079,10 @@ static void fts5SegIterHashInit(
   int n = 0;
 
   assert( pHash );
+  assert( p->rc==SQLITE_OK );
 
   if( pTerm==0 || (iIdx==0 && (flags & FTS5INDEX_QUERY_PREFIX)) ){
-    sqlite3Fts5HashScanInit(pHash, (const char*)pTerm, nTerm);
+    p->rc = sqlite3Fts5HashScanInit(pHash, (const char*)pTerm, nTerm);
     sqlite3Fts5HashScanEntry(pHash, (const char**)&z, &pList, &nList);
     n = (z ? strlen((const char*)z) : 0);
   }else{
@@ -3424,6 +3425,7 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
     Fts5StructureSegment *pSeg;   /* New segment within pStruct */
     int nHeight;                  /* Height of new segment b-tree */
     Fts5Buffer *pBuf;             /* Buffer in which to assemble leaf page */
+    const char *zPrev = 0;
 
     Fts5SegWriter writer;
     fts5WriteInit(p, &writer, iHash, iSegid);
@@ -3438,14 +3440,15 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
     if( p->rc==SQLITE_OK ){
       memset(pBuf->p, 0, 4);
       pBuf->n = 4;
-      sqlite3Fts5HashScanInit(pHash, 0, 0);
+      p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0);
     }
 
-    while( 0==sqlite3Fts5HashScanEof(pHash) ){
+    while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){
       const char *zTerm;
       int nTerm;
       const u8 *pDoclist;
       int nDoclist;
+      int nSuffix;                /* Size of term suffix */
 
       sqlite3Fts5HashScanEntry(pHash, &zTerm,(const char**)&pDoclist,&nDoclist);
       nTerm = strlen(zTerm);
@@ -3462,17 +3465,24 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
 
       /* Write the term to the leaf. And push it up into the b-tree hierarchy */
       if( writer.bFirstTermInPage==0 ){
-        pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], 0);
+        int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, zTerm);
+        pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nPre);
+        nSuffix = nTerm - nPre;
       }else{
         fts5PutU16(&pBuf->p[2], pBuf->n);
         writer.bFirstTermInPage = 0;
         if( writer.aWriter[0].pgno!=1 ){
-          fts5WriteBtreeTerm(p, &writer, nTerm, (const u8*)zTerm);
+          int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, zTerm);
+          fts5WriteBtreeTerm(p, &writer, nPre+1, (const u8*)zTerm);
           pBuf = &writer.aWriter[0].buf;
+          assert( nPre<nTerm );
         }
+        nSuffix = nTerm;
       }
-      pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nTerm);
-      fts5BufferAppendBlob(&p->rc, pBuf, nTerm, (const u8*)zTerm);
+      pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nSuffix);
+      fts5BufferAppendBlob(&p->rc, pBuf, 
+          nSuffix, (const u8*)&zTerm[nTerm-nSuffix]
+      );
 
       if( pgsz>=(pBuf->n + nDoclist + 1) ){
         /* The entire doclist will fit on the current leaf. */
@@ -3536,6 +3546,7 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
 
       pBuf->p[pBuf->n++] = '\0';
       assert( pBuf->n<=pBuf->nSpace );
+      zPrev = zTerm;
       sqlite3Fts5HashScanNext(pHash);
     }
     sqlite3Fts5HashClear(pHash);
diff --git a/manifest b/manifest
index c27cd5f2c0..9484610f04 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Optimize\scopying\sdata\sfrom\sfts5\sin-memory\shash\stables\sto\stop\slevel\ssegments.
-D 2015-02-26T20:49:09.566
+C Fix\ssuffix\sand\sprefix\scompression\sof\sterms\sin\stop-level\sfts5\ssegments.\sAnd\sa\scrash\sthat\scould\sfollow\san\sOOM\scondition.
+D 2015-02-27T07:23:26.074
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -106,13 +106,13 @@ F ext/fts3/unicode/mkunicode.tcl 159c1194da0bc72f51b3c2eb71022568006dc5ad
 F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
 F ext/fts5/fts5.c f2e899fba27ca33c8897635752c4c83a40dcb18d
 F ext/fts5/fts5.h f931954065693898d26c51f23f1d27200184a69a
-F ext/fts5/fts5Int.h 7c2af493177b0e4e0290b869f19cd6d1d671d5ac
+F ext/fts5/fts5Int.h 5c8efea3d0a1ccc70194225f8c402a1732ed5ad5
 F ext/fts5/fts5_aux.c 549aef152b0fd46020f5595d861b1fd60b3f9b4f
 F ext/fts5/fts5_buffer.c b92ba0eb67532d174934087f93716caf9a2168c7
 F ext/fts5/fts5_config.c e3421a76c2abd33a05ac09df0c97c64952d1e700
 F ext/fts5/fts5_expr.c eee52c9df84eade48eaa3f50c8876f44b552ff9b
-F ext/fts5/fts5_hash.c 323099a445bf8f608af069e2d8ff4bb93db9904c
-F ext/fts5/fts5_index.c 7a9de0c033a8f702f8e3659a23c2ea31bbbb789b
+F ext/fts5/fts5_hash.c 63ad0066ec83525f0dad5b416d9db6e06f7d39ac
+F ext/fts5/fts5_index.c 14549572551b60d99413f9bd2043ed2be004a328
 F ext/fts5/fts5_storage.c f7c12c9f454b2a525827b3d85fd222789236f548
 F ext/fts5/fts5_tcl.c 1293fac2bb26903fd3d5cdee59c5885ba7e620d5
 F ext/fts5/fts5_tokenize.c 0d108148c26132448487926fe683425002aee369
@@ -1284,7 +1284,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 208e3cb6b6dc8c7d824b64dec2034004c9fcbba5
-R 2ae35fa8ad1fd2b74f86acf33cc74d9f
+P 8e3ca6323a2beab5f04250e24ae15b159d2aa0ac
+R f04df10b48e4ca5c813b34f1122b4215
 U dan
-Z 84061e5477aeb4ed2552f0a9aa275eda
+Z 4695fccf9ba60f7f53ddc85dbb7a72b7
diff --git a/manifest.uuid b/manifest.uuid
index 3538b76cd7..b080cab9ec 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8e3ca6323a2beab5f04250e24ae15b159d2aa0ac
\ No newline at end of file
+bb104b3646c6f07ed002be7360b08433ee7980d4
\ No newline at end of file