From: drh <drh@noemail.net>
Date: Tue, 25 Oct 2016 13:57:40 +0000 (+0000)
Subject: First attempt at a utility program to compute sqlite_stat1 without doing
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5fba856bbf0245dce545fad8c3d40c33d5037fb0;p=thirdparty%2Fsqlite.git

First attempt at a utility program to compute sqlite_stat1 without doing
a full table scan.

FossilOrigin-Name: 7b83581a43384fe81dc319482e03be0df45ab25d
---

diff --git a/main.mk b/main.mk
index a5a50d205d..118843e38d 100644
--- a/main.mk
+++ b/main.mk
@@ -497,6 +497,10 @@ dbhash$(EXE):	$(TOP)/tool/dbhash.c sqlite3.c sqlite3.h
 	$(TCCX) -o dbhash$(EXE) -DSQLITE_THREADSAFE=0 \
 		$(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) $(THREADLIB)
 
+faststat1$(EXE):	$(TOP)/tool/faststat1.c sqlite3.c sqlite3.h
+	$(TCCX) -o faststat1$(EXE) -DSQLITE_THREADSAFE=0 \
+		$(TOP)/tool/faststat1.c sqlite3.c $(TLIBS) $(THREADLIB)
+
 scrub$(EXE):	$(TOP)/ext/misc/scrub.c sqlite3.o
 	$(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB)
 
diff --git a/manifest b/manifest
index 624111029c..a38df93cf5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\sreading\sfrom\san\sindex,\sthe\sshared-cache\slock\smust\sbe\son\sthe\scorresponding\ntable.
-D 2016-10-21T18:01:40.991
+C First\sattempt\sat\sa\sutility\sprogram\sto\scompute\ssqlite_stat1\swithout\sdoing\na\sfull\stable\sscan.
+D 2016-10-25T13:57:40.551
 F Makefile.in 6fd48ffcf7c2deea7499062d1f3747f986c19678
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5151cc64c4c05f3455f4f692ad11410a810d937f
@@ -311,7 +311,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 06dc0b1a9c9e2d05c9275937dd5b894bfe7d17d8
+F main.mk 1e4a0bdbe86b94b46df89e241949dc3156243f0f
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1449,6 +1449,7 @@ F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2
 F tool/dbhash.c a06228aa21ebc4e6ea8daa486601d938499238a5
 F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2
 F tool/fast_vacuum.c 5ba0d6f5963a0a63bdc42840f678bad75b2ebce1
+F tool/faststat1.c 2178e42e0206ac80a49834454e716cfc21060cfa
 F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439
 F tool/fuzzershell.c f294ca67a10e87db76af130d75b2c94be36359c6
 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
@@ -1527,7 +1528,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0c8a5b8844df3389881aecd8e853eb3c78edd954
-R d660078687b657495181c871ccc08c5d
+P 04fe12b590ab67e40cd079b5e614b787f5f525ad
+R bd47d064e2ed9e25c0899a770a248208
 U drh
-Z 6dae85bfa44e07a1b59d3382cd897280
+Z d3e8fe3b1d555eec50042b333c79640e
diff --git a/manifest.uuid b/manifest.uuid
index 231aa71907..19d3391591 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-04fe12b590ab67e40cd079b5e614b787f5f525ad
\ No newline at end of file
+7b83581a43384fe81dc319482e03be0df45ab25d
\ No newline at end of file
diff --git a/tool/faststat1.c b/tool/faststat1.c
new file mode 100644
index 0000000000..233cfd28a8
--- /dev/null
+++ b/tool/faststat1.c
@@ -0,0 +1,423 @@
+/*
+** 2016-10-24
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This is a utility program that uses the est_count and btree_sample
+** pragmas to try to approximate the content of the sqlite_stat1 table
+** without doing a full table scan.
+**
+** To compile, simply link against SQLite.
+**
+** See the showHelp() routine below for a brief description of how to
+** run the utility.
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <string.h>
+#include <assert.h>
+#include "sqlite3.h"
+
+/*
+** All global variables are gathered into the "g" singleton.
+*/
+struct GlobalVars {
+  const char *zArgv0;       /* Name of program */
+  unsigned fDebug;          /* Debug flags */
+  sqlite3 *db;              /* The database connection */
+} g;
+
+/*
+** Allowed values for g.fDebug
+*/
+#define DEBUG_NONE          0
+
+  
+/*
+** Print an error resulting from faulting command-line arguments and
+** abort the program.
+*/
+static void cmdlineError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stderr, "%s: ", g.zArgv0);
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0);
+  exit(1);
+}
+
+/*
+** Print an error message for an error that occurs at runtime, then
+** abort the program.
+*/
+static void runtimeError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stderr, "%s: ", g.zArgv0);
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n");
+  exit(1);
+}
+
+/*
+** Prepare a new SQL statement.  Print an error and abort if anything
+** goes wrong.
+*/
+static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){
+  char *zSql;
+  int rc;
+  sqlite3_stmt *pStmt;
+
+  zSql = sqlite3_vmprintf(zFormat, ap);
+  if( zSql==0 ) runtimeError("out of memory");
+  rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0);
+  if( rc ){
+    runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db),
+                 zSql);
+  }
+  sqlite3_free(zSql);
+  return pStmt;
+}
+static sqlite3_stmt *db_prepare(const char *zFormat, ...){
+  va_list ap;
+  sqlite3_stmt *pStmt;
+  va_start(ap, zFormat);
+  pStmt = db_vprepare(zFormat, ap);
+  va_end(ap);
+  return pStmt;
+}
+
+/*
+** Estimate the number of rows in the given table or index.
+*/
+static sqlite3_int64 estEntryCount(const char *zTabIdx){
+  double sum = 0.0;
+  int i;
+  int n = 0;
+  sqlite3_stmt *pStmt;
+# define N_CNT_SAMPLE 10
+  for(i=0; i<=N_CNT_SAMPLE; i++){
+    pStmt = db_prepare("PRAGMA est_count(\"%w\",%g)", 
+                       zTabIdx, ((double)i)/(double)(N_CNT_SAMPLE));
+    if( sqlite3_step(pStmt)==SQLITE_ROW ){
+      sum += sqlite3_column_double(pStmt, 0);
+      n++;
+    }
+    sqlite3_finalize(pStmt);
+  }
+  return n==0 ? 0 : (sqlite3_int64)(sum/n);
+}
+
+/*
+** Stat1 for a table.
+*/
+static void analyzeTable(const char *zTab){
+  sqlite3_int64 n = estEntryCount(zTab);
+  sqlite3_stmt *pStmt;
+  if( n==0 ){
+    printf("-- empty table: %s\n", zTab);
+    return;
+  }
+  pStmt = db_prepare(
+     "INSERT INTO temp.est_stat1(tbl,idx,stat)"
+     "VALUES(\"%w\",NULL,'%lld')", zTab, n
+  );
+  sqlite3_step(pStmt);
+  sqlite3_finalize(pStmt);
+}
+
+/*
+** Compare the i-th column of pStmt against pValue.  Return true if they
+** are different.
+*/
+static int columnNotEqual(sqlite3_stmt *pStmt, int i, sqlite3_value *pValue){
+  int n1, n2, n;
+  if( sqlite3_column_type(pStmt,i)!=sqlite3_value_type(pValue) ) return 1;
+  switch( sqlite3_column_type(pStmt,i) ){
+    case SQLITE_NULL:
+      return 0;  /* Nulls compare equal to one another in this context */
+
+    case SQLITE_INTEGER:
+      return sqlite3_column_int64(pStmt,i)!=sqlite3_value_int64(pValue);
+
+    case SQLITE_FLOAT:
+      return sqlite3_column_double(pStmt,i)!=sqlite3_value_double(pValue);
+
+    case SQLITE_BLOB:
+      n1 = sqlite3_column_bytes(pStmt,i);
+      n2 = sqlite3_value_bytes(pValue);
+      n = n1<n2 ? n1 : n2;
+      if( memcmp(sqlite3_column_blob(pStmt,i), sqlite3_value_blob(pValue),n) ){
+        return 1;
+      }
+      return n1!=n2;
+
+    case SQLITE_TEXT:
+      n1 = sqlite3_column_bytes(pStmt,i);
+      n2 = sqlite3_value_bytes(pValue);
+      n = n1<n2 ? n1 : n2;
+      if( memcmp(sqlite3_column_text(pStmt,i), sqlite3_value_text(pValue),n) ){
+        return 1;
+      }
+      return n1!=n2;
+ 
+  }
+  return 1;
+}
+
+/*
+** Stat1 for an index;
+*/
+static void analyzeIndex(const char *zTab, const char *zIdx){
+  sqlite3_int64 n = estEntryCount(zIdx);
+  sqlite3_stmt *pStmt;
+  sqlite3_uint64 *aCnt;
+  sqlite3_value **apValue;
+  int nCol = 0;
+  int nByte;
+  int i, j, k;
+  int iLimit;
+  int nRow = 0;
+  char *zRes;
+  int szRes;
+  int rc;
+
+# define N_SPAN  5
+  if( n==0 ) return;
+  pStmt = db_prepare("PRAGMA index_xinfo=\"%w\"", zIdx);
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    const char *zColl = (const char*)sqlite3_column_text(pStmt,4);
+    if( sqlite3_stricmp(zColl,"binary")!=0 ){
+      printf("-- cannot analyze index \"%s\" because column \"%s\" uses"
+             " collating sequence \"%s\".\n",
+             zIdx, sqlite3_column_text(pStmt, 2), zColl);
+      sqlite3_finalize(pStmt);
+      return;
+    }
+    if( sqlite3_column_int(pStmt, 5)==0 ) break;
+    nCol++;
+  }
+  sqlite3_finalize(pStmt);
+  nByte = (sizeof(aCnt[0]) + sizeof(apValue[0]))*nCol + 30*(nCol+1);
+  aCnt = sqlite3_malloc( nByte );
+  if( aCnt==0 ){
+    runtimeError("out of memory");
+  }
+  memset(aCnt, 0, nByte);
+  apValue = (sqlite3_value**)&aCnt[nCol];
+  zRes = (char*)&apValue[nCol];
+  szRes = 30*(nCol+1);
+
+  iLimit = n>10000 ? 100 : 20000;
+  pStmt = db_prepare("PRAGMA btree_sample(\"%w\",0.0,%lld)",
+                     zIdx, n*2);
+  for(i=0; i<N_SPAN; i++){
+    k = 0;
+    while( k<iLimit && (rc = sqlite3_step(pStmt))==SQLITE_ROW ){
+      int iFirst;
+      for(iFirst=0; iFirst<nCol; iFirst++){
+        if( apValue[iFirst]==0 ) break;
+        if( columnNotEqual(pStmt, iFirst, apValue[iFirst]) ) break;
+      }
+      for(j=iFirst; j<nCol; j++){
+        aCnt[j]++;
+        sqlite3_value_free(apValue[j]);
+        apValue[j] = sqlite3_value_dup(sqlite3_column_value(pStmt,j));
+      }
+      nRow++;
+      k++;
+    }
+    sqlite3_finalize(pStmt);
+    if( rc!=SQLITE_ROW || i==N_SPAN-1 ) break;
+    pStmt = db_prepare("PRAGMA btree_sample(\"%w\",%g,%lld)",
+                       zIdx, ((double)i)/(double)N_SPAN, n*2);
+  }  
+  for(j=0; j<nCol; j++) sqlite3_value_free(apValue[j]);
+  sqlite3_snprintf(szRes, zRes, "%lld", n);
+  k = (int)strlen(zRes);
+  for(j=0; j<nCol; j++){
+    sqlite3_snprintf(szRes-k, zRes+k, " %d", nRow/aCnt[j]);
+    k += (int)strlen(zRes+k);
+  }
+  pStmt = db_prepare(
+     "INSERT INTO temp.est_stat1(tbl,idx,stat)"
+     "VALUES(\"%w\",\"%w\",'%s')", zTab, zIdx, zRes
+  );
+  sqlite3_step(pStmt);
+  sqlite3_finalize(pStmt);
+}
+
+/*
+** Print the sqlite3_value X as an SQL literal.
+*/
+static void printQuoted(FILE *out, sqlite3_value *X){
+  switch( sqlite3_value_type(X) ){
+    case SQLITE_FLOAT: {
+      double r1;
+      char zBuf[50];
+      r1 = sqlite3_value_double(X);
+      sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1);
+      fprintf(out, "%s", zBuf);
+      break;
+    }
+    case SQLITE_INTEGER: {
+      fprintf(out, "%lld", sqlite3_value_int64(X));
+      break;
+    }
+    case SQLITE_BLOB: {
+      const unsigned char *zBlob = sqlite3_value_blob(X);
+      int nBlob = sqlite3_value_bytes(X);
+      if( zBlob ){
+        int i;
+        fprintf(out, "x'");
+        for(i=0; i<nBlob; i++){
+          fprintf(out, "%02x", zBlob[i]);
+        }
+        fprintf(out, "'");
+      }else{
+        /* Could be an OOM, could be a zero-byte blob */
+        fprintf(out, "X''");
+      }
+      break;
+    }
+    case SQLITE_TEXT: {
+      const unsigned char *zArg = sqlite3_value_text(X);
+      int i, j;
+
+      if( zArg==0 ){
+        fprintf(out, "NULL");
+      }else{
+        fprintf(out, "'");
+        for(i=j=0; zArg[i]; i++){
+          if( zArg[i]=='\'' ){
+            fprintf(out, "%.*s'", i-j+1, &zArg[j]);
+            j = i+1;
+          }
+        }
+        fprintf(out, "%s'", &zArg[j]);
+      }
+      break;
+    }
+    case SQLITE_NULL: {
+      fprintf(out, "NULL");
+      break;
+    }
+  }
+}
+
+/*
+** Output SQL that will recreate the aux.zTab table.
+*/
+static void dump_table(const char *zTab, const char *zAlias){
+  int i;                    /* Loop counter */
+  int nCol;                 /* Number of result columns */
+  sqlite3_stmt *pStmt;      /* SQL statement */
+  const char *zSep;         /* Separator string */
+
+  pStmt = db_prepare("SELECT * FROM %s", zTab);
+  nCol = sqlite3_column_count(pStmt);
+  while( SQLITE_ROW==sqlite3_step(pStmt) ){
+    printf("INSERT INTO %s VALUES", zAlias);
+    zSep = "(";
+    for(i=0; i<nCol; i++){
+      fprintf(stdout, "%s",zSep);
+      printQuoted(stdout, sqlite3_column_value(pStmt,i));
+      zSep = ",";
+    }
+    fprintf(stdout, ");\n");
+  }
+  sqlite3_finalize(pStmt);
+}
+
+
+/*
+** Print sketchy documentation for this utility program
+*/
+static void showHelp(void){
+  printf("Usage: %s [options] DBFILE\n", g.zArgv0);
+  printf(
+"Generate an approximate sqlite_stat1 table for the database in the DBFILE\n"
+"file. Write the result to standard output.\n"
+"Options:\n"
+"  (none yet....)\n"
+  );
+}
+
+int main(int argc, char **argv){
+  const char *zDb = 0;
+  int i;
+  int rc;
+  char *zErrMsg = 0;
+  sqlite3_stmt *pStmt;
+
+  g.zArgv0 = argv[0];
+  sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
+  for(i=1; i<argc; i++){
+    const char *z = argv[i];
+    if( z[0]=='-' ){
+      z++;
+      if( z[0]=='-' ) z++;
+      if( strcmp(z,"debug")==0 ){
+        if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
+        g.fDebug = strtol(argv[++i], 0, 0);
+      }else
+      if( strcmp(z,"help")==0 ){
+        showHelp();
+        return 0;
+      }else
+      {
+        cmdlineError("unknown option: %s", argv[i]);
+      }
+    }else if( zDb==0 ){
+      zDb = argv[i];
+    }else{
+      cmdlineError("unknown argument: %s", argv[i]);
+    }
+  }
+  if( zDb==0 ){
+    cmdlineError("database filename required");
+  }
+  rc = sqlite3_open(zDb, &g.db);
+  if( rc ){
+    cmdlineError("cannot open database file \"%s\"", zDb);
+  }
+  rc = sqlite3_exec(g.db, "SELECT * FROM sqlite_master", 0, 0, &zErrMsg);
+  if( rc || zErrMsg ){
+    cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb);
+  }
+  rc = sqlite3_exec(g.db, "CREATE TEMP TABLE est_stat1(tbl,idx,stat);",
+                    0, 0, &zErrMsg);
+  if( rc || zErrMsg ){
+    cmdlineError("Cannot CREATE TEMP TABLE");
+  }
+  pStmt = db_prepare("SELECT type, name, tbl_name FROM sqlite_master"
+                     " WHERE type IN ('table','index')"
+                     "   AND rootpage>0"
+                     "   AND (type='index' OR name NOT LIKE 'sqlite_%%')"
+                     " ORDER BY tbl_name, type DESC, name");
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    const char *zType = (const char*)sqlite3_column_text(pStmt, 0);
+    const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
+    const char *zTblName = (const char*)sqlite3_column_text(pStmt, 2);
+    if( zType[0]=='t' ){
+      analyzeTable(zName);
+    }else{
+      analyzeIndex(zTblName, zName);
+    }
+  }
+  sqlite3_finalize(pStmt);
+  dump_table("temp.est_stat1","sqlite_stat1");
+  sqlite3_close(g.db);
+  return 0;
+}