From: drh <drh@noemail.net>
Date: Sat, 3 Jun 2017 17:24:04 +0000 (+0000)
Subject: In kvtest, add the ability to work with a hierarchy of files on disk,
X-Git-Tag: version-3.20.0~219
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=1346fdad650e1896d4aa47dcf556b4ffa38d01f6;p=thirdparty%2Fsqlite.git

In kvtest, add the ability to work with a hierarchy of files on disk,
in addition to having all files in the same directory.

FossilOrigin-Name: f568f666c85ab9b80592927dc033cfd65bd4415576cf5b3beaf300d68a8e074e
---

diff --git a/manifest b/manifest
index 394f8cc3a8..729fb5a1fd 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s--nocheckpoint\sand\s--multitrans\soptions\sto\skvtest.
-D 2017-06-03T15:17:21.324
+C In\skvtest,\sadd\sthe\sability\sto\swork\swith\sa\shierarchy\sof\sfiles\son\sdisk,\nin\saddition\sto\shaving\sall\sfiles\sin\sthe\ssame\sdirectory.
+D 2017-06-03T17:24:04.003
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc
@@ -925,7 +925,7 @@ F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb2848
 F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0
 F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa
 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff
-F test/kvtest.c a9496cda31fad0998f8c135ae1b02f5a7cf42702a69885084dc2e973744bfba8
+F test/kvtest.c 4870db237a4a0e760cb508675fd612895b32030ad64998e8ff76675afa79d01f
 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63
 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200
 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302
@@ -1582,7 +1582,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7fdc78a672b2ea6187dcb5fdf32f809bb8e4d501e2434f2233edc3bc2e3acc7c
-R 042e3a76ff3a30833c7d24fbea5e960d
+P 5828633c2392274b6863b50eaffbb2a176a4d892e83542824a9a3f0d1b62c967
+R e8478f9019018f65e9cf63cf97245ef3
 U drh
-Z db3574b40a9ba9c25a835775eb0162a1
+Z eb09df8f5bb56b30d7c67488f8144ab1
diff --git a/manifest.uuid b/manifest.uuid
index 058637f374..0c4275099b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5828633c2392274b6863b50eaffbb2a176a4d892e83542824a9a3f0d1b62c967
\ No newline at end of file
+f568f666c85ab9b80592927dc033cfd65bd4415576cf5b3beaf300d68a8e074e
\ No newline at end of file
diff --git a/test/kvtest.c b/test/kvtest.c
index dbf311501e..39101ff4db 100644
--- a/test/kvtest.c
+++ b/test/kvtest.c
@@ -71,10 +71,15 @@ static const char zHelp[] =
 "\n"
 "           --variance V           Randomly vary M by plus or minus V\n"
 "\n"
-"   kvtest export DBFILE DIRECTORY\n"
+"   kvtest export DBFILE DIRECTORY [--tree]\n"
 "\n"
 "        Export all the blobs in the kv table of DBFILE into separate\n"
-"        files in DIRECTORY.\n"
+"        files in DIRECTORY.  DIRECTORY is created if it does not previously\n"
+"        exist.  If the --tree option is used, then the blobs are written\n"
+"        into a hierarchy of directories, using names like 00/00/00,\n"
+"        00/00/01, 00/00/02, and so forth.  Without the --tree option, all\n"
+"        files are in the top-level directory with names like 000000, 000001,\n"
+"        000002, and so forth.\n"
 "\n"
 "   kvtest stat DBFILE\n"
 "\n"
@@ -117,6 +122,7 @@ static const char zHelp[] =
 # include <unistd.h>
 #else
   /* Provide Windows equivalent for the needed parts of unistd.h */
+# include <direct.h>
 # include <io.h>
 # define R_OK 2
 # define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
@@ -239,13 +245,23 @@ static int integerValue(const char *zArg){
 /*
 ** Check the filesystem object zPath.  Determine what it is:
 **
-**    PATH_DIR     A directory
+**    PATH_DIR     A single directory holding many files
+**    PATH_TREE    A directory hierarchy with files at the leaves
 **    PATH_DB      An SQLite database
 **    PATH_NEXIST  Does not exist
 **    PATH_OTHER   Something else
+**
+** PATH_DIR means all of the separate files are grouped together
+** into a single directory with names like 000000, 000001, 000002, and
+** so forth.  PATH_TREE means there is a hierarchy of directories so
+** that no single directory has too many entries.  The files have names
+** like 00/00/00, 00/00/01, 00/00/02 and so forth.  The decision between
+** PATH_DIR and PATH_TREE is determined by the presence of a subdirectory
+** named "00" at the top-level.
 */
 #define PATH_DIR     1
-#define PATH_DB      2
+#define PATH_TREE    2
+#define PATH_DB      3
 #define PATH_NEXIST  0
 #define PATH_OTHER   99
 static int pathType(const char *zPath){
@@ -255,7 +271,15 @@ static int pathType(const char *zPath){
   memset(&x, 0, sizeof(x));
   rc = stat(zPath, &x);
   if( rc<0 ) return PATH_OTHER;
-  if( S_ISDIR(x.st_mode) ) return PATH_DIR;
+  if( S_ISDIR(x.st_mode) ){
+    char *zLayer1 = sqlite3_mprintf("%s/00", zPath);
+    memset(&x, 0, sizeof(x));
+    rc = stat(zLayer1, &x);
+    sqlite3_free(zLayer1);
+    if( rc<0 ) return PATH_DIR;
+    if( S_ISDIR(x.st_mode) ) return PATH_TREE;
+    return PATH_DIR;
+  }
   if( (x.st_size%512)==0 ) return PATH_DB;
   return PATH_OTHER;
 }
@@ -416,37 +440,6 @@ static int statMain(int argc, char **argv){
   return 0;
 }
 
-/*
-** Implementation of the "writefile(X,Y)" SQL function.  The argument Y
-** is written into file X.  The number of bytes written is returned.  Or
-** NULL is returned if something goes wrong, such as being unable to open
-** file X for writing.
-*/
-static void writefileFunc(
-  sqlite3_context *context,
-  int argc,
-  sqlite3_value **argv
-){
-  FILE *out;
-  const char *z;
-  sqlite3_int64 rc;
-  const char *zFile;
-
-  zFile = (const char*)sqlite3_value_text(argv[0]);
-  if( zFile==0 ) return;
-  out = fopen(zFile, "wb");
-  if( out==0 ) return;
-  z = (const char*)sqlite3_value_blob(argv[1]);
-  if( z==0 ){
-    rc = 0;
-  }else{
-    rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
-  }
-  fclose(out);
-  printf("\r%s   ", zFile); fflush(stdout);
-  sqlite3_result_int64(context, rc);
-}
-
 /*
 **      remember(V,PTR)
 **
@@ -467,6 +460,17 @@ static void rememberFunc(
   sqlite3_result_int64(pCtx, v);
 }
 
+/*
+** Make sure a directory named zDir exists.
+*/
+static void kvtest_mkdir(const char *zDir){
+#if defined(_WIN32)
+  (void)mkdir(zDir);
+#else
+  (void)mkdir(zDir, 0755);
+#endif
+}
+
 /*
 ** Export the kv table to individual files in the filesystem
 */
@@ -474,32 +478,77 @@ static int exportMain(int argc, char **argv){
   char *zDb;
   char *zDir;
   sqlite3 *db;
-  char *zSql;
+  sqlite3_stmt *pStmt;
   int rc;
-  char *zErrMsg = 0;
+  int ePathType;
+  int nFN;
+  char *zFN;
+  char *zTail;
+  size_t nWrote;
+  int i;
 
   assert( strcmp(argv[1],"export")==0 );
   assert( argc>=3 );
+  if( argc<4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY [OPTIONS]");
   zDb = argv[2];
-  if( argc!=4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY");
   zDir = argv[3];
-  if( pathType(zDir)!=PATH_DIR ){
+  kvtest_mkdir(zDir);
+  for(i=4; i<argc; i++){
+    const char *z = argv[i];
+    if( z[0]=='-' && z[1]=='-' ) z++;
+    if( strcmp(z,"-tree")==0 ){
+      zFN = sqlite3_mprintf("%s/00", zDir);
+      kvtest_mkdir(zFN);
+      sqlite3_free(zFN);
+      continue;
+    }
+    fatalError("unknown argument: \"%s\"\n", argv[i]);
+  }
+  ePathType = pathType(zDir);
+  if( ePathType!=PATH_DIR && ePathType!=PATH_TREE ){
     fatalError("object \"%s\" is not a directory", zDir);
   }
   rc = sqlite3_open(zDb, &db);
   if( rc ){
     fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
   }
-  sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
-                          writefileFunc, 0, 0);
-  zSql = sqlite3_mprintf(
-    "SELECT writefile(printf('%s/%%06d',k),v) FROM kv;",
-    zDir
-  );
-  rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg);
-  if( rc ) fatalError("database create failed: %s", zErrMsg);
-  sqlite3_free(zSql);
+  rc = sqlite3_prepare_v2(db, "SELECT k, v FROM kv ORDER BY k", -1, &pStmt, 0);
+  if( rc ){
+    fatalError("prepare_v2 failed: %s\n", sqlite3_errmsg(db));
+  }
+  nFN = (int)strlen(zDir);
+  zFN = sqlite3_mprintf("%s/00/00/00.extra---------------------", zDir);
+  if( zFN==0 ){
+    fatalError("malloc failed\n");
+  }
+  zTail = zFN + nFN + 1;
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    int iKey = sqlite3_column_int(pStmt, 0);
+    int nData = sqlite3_column_bytes(pStmt, 1);
+    const void *pData = sqlite3_column_blob(pStmt, 1);
+    FILE *out;
+    if( ePathType==PATH_DIR ){
+      sqlite3_snprintf(20, zTail, "%06d", iKey);
+    }else{
+      sqlite3_snprintf(20, zTail, "%02d", iKey/10000);
+      kvtest_mkdir(zFN);
+      sqlite3_snprintf(20, zTail, "%02d/%02d", iKey/10000, (iKey/100)%100);
+      kvtest_mkdir(zFN);
+      sqlite3_snprintf(20, zTail, "%02d/%02d/%02d",
+                       iKey/10000, (iKey/100)%100, iKey%100);
+    }
+    out = fopen(zFN, "wb");      
+    nWrote = fwrite(pData, 1, nData, out);
+    fclose(out);
+    printf("\r%s   ", zTail); fflush(stdout);
+    if( nWrote!=nData ){
+      fatalError("Wrote only %d of %d bytes to %s\n",
+                  (int)nWrote, nData, zFN);
+    }
+  }
+  sqlite3_finalize(pStmt);
   sqlite3_close(db);
+  sqlite3_free(zFN);
   printf("\n");
   return 0;
 }
@@ -911,10 +960,15 @@ static int runMain(int argc, char **argv){
   }
   if( iMax<=0 ) iMax = 1000;
   for(i=0; i<nCount; i++){
-    if( eType==PATH_DIR ){
-      /* CASE 1: Reading blobs out of separate files */
+    if( eType==PATH_DIR || eType==PATH_TREE ){
+      /* CASE 1: Reading or writing blobs out of separate files */
       char *zKey;
-      zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);
+      if( eType==PATH_DIR ){
+        zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);
+      }else{
+        zKey = sqlite3_mprintf("%s/%02d/%02d/%02d", zDb, iKey/10000,
+                               (iKey/100)%100, iKey%100);
+      }
       nData = 0;
       if( isUpdateTest ){
         updateFile(zKey, &nData, doFsync);