From: dan <dan@noemail.net>
Date: Sat, 31 Jan 2015 15:23:44 +0000 (+0000)
Subject: Minor optimizations to fts5 writes.
X-Git-Tag: version-3.8.11~114^2~90
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=50821403113ce3e2289a549c4c9e57484943bd40;p=thirdparty%2Fsqlite.git

Minor optimizations to fts5 writes.

FossilOrigin-Name: 1fffe51fa92f1784365140d5b163ab6c690981ae
---

diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c
index 78ab2083c3..841c4d3709 100644
--- a/ext/fts5/fts5_hash.c
+++ b/ext/fts5/fts5_hash.c
@@ -77,6 +77,17 @@ static void fts5Put4ByteVarint(u8 *a, int iVal){
   a[3] = (0x7F & (u8)(iVal));
 }
 
+static int fts5Get4ByteVarint(u8 *a, int *pnVarint){
+  int iRet = ((int)(a[0] & 0x7F) << 21) + ((int)(a[1] & 0x7F) << 14)
+       + ((int)(a[2] & 0x7F) <<  7) + ((int)(a[3]));
+  *pnVarint = (
+      (iRet & 0xFFFFFF80)==0 ? 1 : 
+      (iRet & 0xFFFFC000)==0 ? 2 :
+      (iRet & 0xFFE00000)==0 ? 3 : 4
+  );
+  return iRet;
+}
+
 /*
 ** Allocate a new hash table.
 */
@@ -398,10 +409,12 @@ int sqlite3Fts5HashIterate(
         while( rc==SQLITE_OK && iOff<pList->nData ){
           i64 iDelta;             /* Rowid delta value */
           int nPoslist;           /* Size of position list in bytes */
+          int nVarint;
           iOff += getVarint(&pPtr[iOff], (u64*)&iDelta);
           iRowid += iDelta;
-          iOff += fts5GetVarint32(&pPtr[iOff], nPoslist);
-          rc = xEntry(pCtx, iRowid, &pPtr[iOff], nPoslist);
+          nPoslist = fts5Get4ByteVarint(&pPtr[iOff], &nVarint);
+          iOff += 4;
+          rc = xEntry(pCtx, iRowid, &pPtr[iOff-nVarint], nPoslist+nVarint);
           iOff += nPoslist;
         }
 
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 1731d3ed70..ed7692bef6 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -3392,12 +3392,17 @@ static int fts5FlushNewEntry(
   Fts5FlushCtx *p = (Fts5FlushCtx*)pCtx;
   Fts5Index *pIdx = p->pIdx;
 
+#ifdef SQLITE_DEBUG
+  /* The poslist-size varint should already be at the start of the 
+  ** aPoslist/nPoslist buffer. This assert verifies that. */
+  int n, i;
+  i = fts5GetVarint32(aPoslist, n);
+  assert( nPoslist==(n+i) );
+#endif
+
   /* Append the rowid itself */
   fts5WriteAppendRowid(pIdx, &p->writer, iRowid);
 
-  /* Append the size of the position list in bytes */
-  fts5WriteAppendPoslistInt(pIdx, &p->writer, nPoslist);
-
   /* And the poslist data */
   fts5WriteAppendPoslistData(pIdx, &p->writer, aPoslist, nPoslist);
   return pIdx->rc;
diff --git a/ext/fts5/tool/loadfts5.tcl b/ext/fts5/tool/loadfts5.tcl
index 034286bc60..97353ffa3b 100644
--- a/ext/fts5/tool/loadfts5.tcl
+++ b/ext/fts5/tool/loadfts5.tcl
@@ -8,6 +8,8 @@ proc loadfile {f} {
 }
 
 set ::nRow 0
+set ::nRowPerDot 1000
+
 proc load_hierachy {dir} {
   foreach f [glob -nocomplain -dir $dir *] {
     if {$::O(limit) && $::nRow>=$::O(limit)} break
@@ -16,6 +18,13 @@ proc load_hierachy {dir} {
     } else {
       db eval { INSERT INTO t1 VALUES($f, loadfile($f)) }
       incr ::nRow
+
+      if {($::nRow % $::nRowPerDot)==0} {
+        puts -nonewline .
+        if {($::nRow % (65*$::nRowPerDot))==0} { puts "" }
+        flush stdout
+      }
+
     }
   }
 }
@@ -81,7 +90,9 @@ sqlite3 db [lindex $argv end-1]
 db func loadfile loadfile
 
 db transaction {
-  db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))"
+  catch {
+    db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))"
+  }
   if {$O(automerge)>=0} {
     if {$O(vtab) == "fts5"} {
       db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) }
diff --git a/manifest b/manifest
index d807680b8c..980bed6d09 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\ssome\sproblems\swith\stransactions\sthat\sboth\sread\sand\swrite\san\sfts5\stable.
-D 2015-01-29T20:59:34.380
+C Minor\soptimizations\sto\sfts5\swrites.
+D 2015-01-31T15:23:44.132
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -111,8 +111,8 @@ F ext/fts5/fts5_aux.c 549aef152b0fd46020f5595d861b1fd60b3f9b4f
 F ext/fts5/fts5_buffer.c b92ba0eb67532d174934087f93716caf9a2168c7
 F ext/fts5/fts5_config.c e3421a76c2abd33a05ac09df0c97c64952d1e700
 F ext/fts5/fts5_expr.c 473e3428a9a637fa6e61d64d8ca3796ec57a58e9
-F ext/fts5/fts5_hash.c b54822ca901fb76d79c6a09daecbc464e5fe02c1
-F ext/fts5/fts5_index.c 1550befd9622d009520fdadfa0b42154e0ac54c0
+F ext/fts5/fts5_hash.c 57febfb06e59ae419ee9ba31667635f70d7c4dd0
+F ext/fts5/fts5_index.c 3239fa41b002230dd47f92937b72e7fa2caa0f90
 F ext/fts5/fts5_storage.c f7c12c9f454b2a525827b3d85fd222789236f548
 F ext/fts5/fts5_tcl.c 1293fac2bb26903fd3d5cdee59c5885ba7e620d5
 F ext/fts5/fts5_tokenize.c 7c61d5c35c3449597bdeaa54dd48afe26852c7b0
@@ -147,7 +147,7 @@ F ext/fts5/test/fts5rowid.test a1b2a6d76648c734c1aab11ee1a619067e8d90e6
 F ext/fts5/test/fts5tokenizer.test b34ae592db66f6e89546d791ce1f905ba0b3395c
 F ext/fts5/test/fts5unicode.test 79b3e34eb29ce4929628aa514a40cb467fdabe4d
 F ext/fts5/test/fts5unicode2.test 64a5267fd6082fcb46439892ebd0cbaa5c38acee
-F ext/fts5/tool/loadfts5.tcl 0d39b916550097a3b714060bfc1164a4a9b73f4c
+F ext/fts5/tool/loadfts5.tcl 310cb6556b74eda5ce7829a539298c3f35003523
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
@@ -1283,7 +1283,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P f704bc059e06b01f1d68fa7dad89e33eace6c389
-R f45496311c450f4a551203517eb9c071
+P 0e225b15357765f132c3364b222f9931a608a5b2
+R bf6974c43532ef38693ab05d4629d2ec
 U dan
-Z d89173c476e3f912e9f3a6ccba8c9b1b
+Z 8f2a2071b0090e43c31fbea210c6dce0
diff --git a/manifest.uuid b/manifest.uuid
index a041d59a2d..734bdb3e9c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0e225b15357765f132c3364b222f9931a608a5b2
\ No newline at end of file
+1fffe51fa92f1784365140d5b163ab6c690981ae
\ No newline at end of file