From: dan <dan@noemail.net>
Date: Wed, 27 Oct 2010 10:55:54 +0000 (+0000)
Subject: In fts4, store the total number of bytes of for all records in the table in the ... 
X-Git-Tag: version-3.7.4~86^2~1
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=8c30711208134a8cadcc5dd3e732b0762fb41bb1;p=thirdparty%2Fsqlite.git

In fts4, store the total number of bytes of for all records in the table in the %_stat table.

FossilOrigin-Name: 941647d121ac60e2eabc998cfe79b157fb918d7e
---

diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index 49bede68cb..2bb5fb9c13 100644
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -583,6 +583,7 @@ static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){
         return rc;
       }
     }
+    aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
   }
   return SQLITE_OK;
 }
@@ -680,7 +681,7 @@ static int fts3DeleteAll(Fts3Table *p){
 ** (an integer) of a row about to be deleted. Remove all terms from the
 ** full-text index.
 */
-static void fts3DeleteTerms(
+static void fts3DeleteTerms( 
   int *pRC,               /* Result code */
   Fts3Table *p,           /* The FTS table to delete from */
   sqlite3_value **apVal,  /* apVal[] contains the docid to be deleted */
@@ -702,6 +703,7 @@ static void fts3DeleteTerms(
           *pRC = rc;
           return;
         }
+        aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
       }
     }
     rc = sqlite3_reset(pSelect);
@@ -1033,9 +1035,7 @@ int sqlite3Fts3SegReaderCost(
           const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
           a += sqlite3Fts3GetVarint(a, &nDoc);
           while( a<pEnd ){
-            sqlite3_int64 nVarint;
-            a += sqlite3Fts3GetVarint(a, &nVarint);
-            nByte += nVarint;
+            a += sqlite3Fts3GetVarint(a, &nByte);
           }
         }
 
@@ -2517,16 +2517,26 @@ static void fts3InsertDocsize(
 }
 
 /*
-** Update the 0 record of the %_stat table so that it holds a blob
-** which contains the document count followed by the cumulative
-** document sizes for all columns.
+** Record 0 of the %_stat table contains a blob consisting of N varints,
+** where N is the number of user defined columns in the fts3 table plus
+** two. If nCol is the number of user defined columns, then values of the 
+** varints are set as follows:
+**
+**   Varint 0:       Total number of rows in the table.
+**
+**   Varint 1..nCol: For each column, the total number of tokens stored in
+**                   the column for all rows of the table.
+**
+**   Varint 1+nCol:  The total size, in bytes, of all text values in all
+**                   columns of all rows of the table.
+**
 */
 static void fts3UpdateDocTotals(
-  int *pRC,       /* The result code */
-  Fts3Table *p,   /* Table being updated */
-  u32 *aSzIns,    /* Size increases */
-  u32 *aSzDel,    /* Size decreases */
-  int nChng       /* Change in the number of documents */
+  int *pRC,                       /* The result code */
+  Fts3Table *p,                   /* Table being updated */
+  u32 *aSzIns,                    /* Size increases */
+  u32 *aSzDel,                    /* Size decreases */
+  int nChng                       /* Change in the number of documents */
 ){
   char *pBlob;             /* Storage for BLOB written into %_stat */
   int nBlob;               /* Size of BLOB written into %_stat */
@@ -2535,13 +2545,15 @@ static void fts3UpdateDocTotals(
   int i;                   /* Loop counter */
   int rc;                  /* Result code from subfunctions */
 
+  const int nStat = p->nColumn+2;
+
   if( *pRC ) return;
-  a = sqlite3_malloc( (sizeof(u32)+10)*(p->nColumn+1) );
+  a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
   if( a==0 ){
     *pRC = SQLITE_NOMEM;
     return;
   }
-  pBlob = (char*)&a[p->nColumn+1];
+  pBlob = (char*)&a[nStat];
   rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
   if( rc ){
     sqlite3_free(a);
@@ -2549,11 +2561,11 @@ static void fts3UpdateDocTotals(
     return;
   }
   if( sqlite3_step(pStmt)==SQLITE_ROW ){
-    fts3DecodeIntArray(p->nColumn+1, a,
+    fts3DecodeIntArray(nStat, a,
          sqlite3_column_blob(pStmt, 0),
          sqlite3_column_bytes(pStmt, 0));
   }else{
-    memset(a, 0, sizeof(u32)*(p->nColumn+1) );
+    memset(a, 0, sizeof(u32)*(nStat) );
   }
   sqlite3_reset(pStmt);
   if( nChng<0 && a[0]<(u32)(-nChng) ){
@@ -2561,7 +2573,7 @@ static void fts3UpdateDocTotals(
   }else{
     a[0] += nChng;
   }
-  for(i=0; i<p->nColumn; i++){
+  for(i=0; i<p->nColumn+1; i++){
     u32 x = a[i+1];
     if( x+aSzIns[i] < aSzDel[i] ){
       x = 0;
@@ -2570,7 +2582,7 @@ static void fts3UpdateDocTotals(
     }
     a[i+1] = x;
   }
-  fts3EncodeIntArray(p->nColumn+1, a, pBlob, &nBlob);
+  fts3EncodeIntArray(nStat, a, pBlob, &nBlob);
   rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0);
   if( rc ){
     sqlite3_free(a);
@@ -2791,10 +2803,10 @@ int sqlite3Fts3UpdateMethod(
   assert( p->pSegments==0 );
 
   /* Allocate space to hold the change in document sizes */
-  aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 );
+  aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
   if( aSzIns==0 ) return SQLITE_NOMEM;
-  aSzDel = &aSzIns[p->nColumn];
-  memset(aSzIns, 0, sizeof(aSzIns[0])*p->nColumn*2);
+  aSzDel = &aSzIns[p->nColumn+1];
+  memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
 
   /* If this is a DELETE or UPDATE operation, remove the old record. */
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
diff --git a/manifest b/manifest
index 79d95349c1..055fab5938 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Structural\scoverage\stests\sfor\svdbeblob.c.\sIncluding\sexperimental\snew\sAPI\ssqlite3_blob_reopen().
-D 2010-10-26T18:42:52
+C In\sfts4,\sstore\sthe\stotal\snumber\sof\sbytes\sof\sfor\sall\srecords\sin\sthe\stable\sin\sthe\s%_stat\stable.
+D 2010-10-27T10:55:54
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 2c8cefd962eca0147132c7cf9eaa4bb24c656f3f
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -73,7 +73,7 @@ F ext/fts3/fts3_snippet.c 300c12b7f0a2a6ae0491bb2d00e2d5ff9c28f685
 F ext/fts3/fts3_tokenizer.c b4f2d01c24573852755bc92864816785dae39318
 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
-F ext/fts3/fts3_write.c 1b9211904f8157ebca8e17034e1150f3653e6c3f
+F ext/fts3/fts3_write.c 943216b144447a1fbadef373cbd2b7eb3fd17a65
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@@ -433,7 +433,7 @@ F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
 F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
-F test/fts3defer.test 1f75427fa35d7715595c91575b7c5d449fd28089
+F test/fts3defer.test eab4f24c8402fb4e1e6aad44bcdfbe5bf42160b2
 F test/fts3defer2.test 1a9f213ca79509b60d81460febc7e4e5b64af95c
 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
@@ -879,7 +879,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P d1cc5c93f09c9092ec478c04e8d9a8b1f9c0cb04
-R 1bc240279e5d7a2dcb9378295e259554
+P 97c6b2616ddcce2337778c6ee88a973cc4fe999d
+R 8864817f6f3963775cfdd60519bb3822
 U dan
-Z c7b142127578014c4915d0546d65c2ad
+Z 890501f0776b5167d933a1fceff6796c
diff --git a/manifest.uuid b/manifest.uuid
index 4cfa0c76ea..9e8bb976fe 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-97c6b2616ddcce2337778c6ee88a973cc4fe999d
\ No newline at end of file
+941647d121ac60e2eabc998cfe79b157fb918d7e
\ No newline at end of file
diff --git a/test/fts3defer.test b/test/fts3defer.test
index d3b0a0023e..f623143568 100644
--- a/test/fts3defer.test
+++ b/test/fts3defer.test
@@ -189,6 +189,15 @@ lappend data {*}{
   "srwwnezqk csjqxhgj rbwzuf nvfasfh jcpiwj xldlpy nvfasfh jk vgsld wjybxmieki"
 }
 
+proc add_empty_records {n} {
+  execsql BEGIN
+  for {set i 0} {$i < $n} {incr i} {
+    execsql { INSERT INTO t1 VALUES('') }
+  }
+  execsql COMMIT
+}
+
+
 #set e [list]
 #foreach d $data {set e [concat $e $d]}
 #puts [lsort -unique $e]
@@ -213,12 +222,14 @@ foreach {tn setup} {
     set dmt_modes {0 1 2}
     execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
     foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
+    add_empty_records 1000
     execsql $zero_long_doclists
   }
   4 {
     set dmt_modes 0
     execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
     foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
+    add_empty_records 1000
     execsql "INSERT INTO t1(t1) VALUES('optimize')"
     execsql $zero_long_doclists
   }
@@ -239,6 +250,7 @@ foreach {tn setup} {
   do_select_test 1.2 {
     SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh'
   } {100}
+if {$tn==3} breakpoint
   do_select_test 1.3 {
     SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf'
   } {7 70 98}