From: drh Date: Thu, 29 Dec 2016 03:57:43 +0000 (+0000) Subject: Add the kvtest.c test program for measuring key/value read performance under X-Git-Tag: version-3.16.0~11^2~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=cac028ba19d383c2f49cde751b156cedb69f2aa6;p=thirdparty%2Fsqlite.git Add the kvtest.c test program for measuring key/value read performance under various scenarios. FossilOrigin-Name: 489e0787c1ea47963174387e8ade6295ceff568e --- diff --git a/manifest b/manifest index 7dd19b6bf3..d07e3a5fa2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\sthe\snockpt.test\smodule\sto\srun\sunder\sthe\sinmemory_journal\npermutation. -D 2016-12-27T15:59:15.338 +C Add\sthe\skvtest.c\stest\sprogram\sfor\smeasuring\skey/value\sread\sperformance\sunder\nvarious\sscenarios. +D 2016-12-29T03:57:43.130 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da @@ -894,6 +894,7 @@ F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff +F test/kvtest.c 1f8e6f0352ebb0ef4179cd7968af594eee130d57 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1539,7 +1540,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7854bee260996087bdb7f8729ba8cfb4f467f93f -R 0002370ca7e07ee4867da4a22905f821 +P a6af06f164b1f65779e2171ec4946119c66f9be8 +R a9bf163ca1249527027c884b8ea94c96 +T *branch * kvtest +T *sym-kvtest * +T -sym-trunk * U drh -Z 948aac4cd107424288db7dcd7f723dea +Z fb835c04c55072b05b31100f6eb7b616 diff --git a/manifest.uuid b/manifest.uuid index 5dc63da264..437dcdf57b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a6af06f164b1f65779e2171ec4946119c66f9be8 \ No newline at end of file +489e0787c1ea47963174387e8ade6295ceff568e \ No newline at end of file diff --git a/test/kvtest.c b/test/kvtest.c new file mode 100644 index 0000000000..9b9a0c3d40 --- /dev/null +++ b/test/kvtest.c @@ -0,0 +1,473 @@ +/* +** 2016-12-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements "key-value" performance test for SQLite. The +** purpose is to compare the speed of SQLite for accessing large BLOBs +** versus reading those same BLOB values out of individual files in the +** filesystem. +** +** Run "kvtest --help" for further information, or see comments below. +*/ +static const char zHelp[] = +"Usage: kvhelp COMMAND ARGS...\n" +"\n" +" kvhelp init DBFILE --count N --size M --pagesize X\n" +"\n" +" Generate a new test database file named DBFILE containing N\n" +" BLOBs each of size M bytes. The page size of the new database\n" +" file will be X\n" +"\n" +" kvhelp export DBFILE DIRECTORY\n" +"\n" +" Export all the blobs in the kv table of DBFILE into separate\n" +" files in DIRECTORY.\n" +"\n" +" kvhelp run DBFILE [options]\n" +"\n" +" Run a performance test. DBFILE can be either the name of a\n" +" database or a directory containing sample files. Options:\n" +"\n" +" --count N Read N blobs\n" +" --blob-api Use the BLOB API\n" +" --random Read blobs in a random order\n" +" --desc Read blobs in descending order\n" +" --max-id N Maximum blob key to use\n" +" --start N Start reading with this blob key\n" +; + +/* Reference resources used */ +#include +#include +#include +#include +#include +#include +#include +#include "sqlite3.h" + +/* +** Show thqe help text and quit. +*/ +static void showHelp(void){ + fprintf(stdout, "%s", zHelp); + exit(1); +} + +/* +** Show an error message an quit. +*/ +static void fatalError(const char *zFormat, ...){ + va_list ap; + fprintf(stdout, "ERROR: "); + va_start(ap, zFormat); + vfprintf(stdout, zFormat, ap); + va_end(ap); + fprintf(stdout, "\n"); + exit(1); +} + +/* +** Check the filesystem object zPath. Determine what it is: +** +** PATH_DIR A directory +** PATH_DB An SQLite database +** PATH_NEXIST Does not exist +** PATH_OTHER Something else +*/ +#define PATH_DIR 1 +#define PATH_DB 2 +#define PATH_NEXIST 0 +#define PATH_OTHER 99 +static int pathType(const char *zPath){ + struct stat x; + int rc; + if( access(zPath,R_OK) ) return PATH_NEXIST; + memset(&x, 0, sizeof(x)); + rc = stat(zPath, &x); + if( rc<0 ) return PATH_OTHER; + if( S_ISDIR(x.st_mode) ) return PATH_DIR; + if( (x.st_size%512)==0 ) return PATH_DB; + return PATH_OTHER; +} + +/* +** A Pseudo-random number generator with a fixed seed. Use this so +** that the same sequence of "random" numbers are generated on each +** run, for repeatability. +*/ +static unsigned int randInt(void){ + static unsigned int x = 0x333a13cd; + static unsigned int y = 0xecb2adea; + x = (x>>1) ^ ((1+~(x&1)) & 0xd0000001); + y = y*1103515245 + 12345; + return x^y; +} + +/* +** Do database initialization. +*/ +static int initMain(int argc, char **argv){ + char *zDb; + int i, rc; + int nCount = 1000; + int sz = 10000; + int pgsz = 4096; + sqlite3 *db; + char *zSql; + char *zErrMsg = 0; + + assert( strcmp(argv[1],"init")==0 ); + assert( argc>=3 ); + zDb = argv[2]; + for(i=3; i65536 || ((pgsz-1)&pgsz)!=0 ){ + fatalError("the --pagesize must be power of 2 between 512 and 65536"); + } + continue; + } + fatalError("unknown option: \"%s\"", argv[i]); + } + rc = sqlite3_open(zDb, &db); + if( rc ){ + fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db)); + } + zSql = sqlite3_mprintf( + "DROP TABLE IF EXISTS kv;\n" + "PRAGMA page_size=%d;\n" + "VACUUM;\n" + "BEGIN;\n" + "CREATE TABLE kv(k INTEGER PRIMARY KEY, v BLOB);\n" + "WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<%d)" + " INSERT INTO kv(k,v) SELECT x, randomblob(%d) FROM c;\n" + "COMMIT;\n", + pgsz, nCount, sz + ); + rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg); + if( rc ) fatalError("database create failed: %s", zErrMsg); + sqlite3_free(zSql); + sqlite3_close(db); + return 0; +} + +/* +** Implementation of the "writefile(X,Y)" SQL function. The argument Y +** is written into file X. The number of bytes written is returned. Or +** NULL is returned if something goes wrong, such as being unable to open +** file X for writing. +*/ +static void writefileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + FILE *out; + const char *z; + sqlite3_int64 rc; + const char *zFile; + + zFile = (const char*)sqlite3_value_text(argv[0]); + if( zFile==0 ) return; + out = fopen(zFile, "wb"); + if( out==0 ) return; + z = (const char*)sqlite3_value_blob(argv[1]); + if( z==0 ){ + rc = 0; + }else{ + rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out); + } + fclose(out); + sqlite3_result_int64(context, rc); +} + +/* +** Export the kv table to individual files in the filesystem +*/ +static int exportMain(int argc, char **argv){ + char *zDb; + char *zDir; + sqlite3 *db; + char *zSql; + int rc; + char *zErrMsg = 0; + + assert( strcmp(argv[1],"export")==0 ); + assert( argc>=3 ); + zDb = argv[2]; + if( argc!=4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY"); + zDir = argv[3]; + if( pathType(zDir)!=PATH_DIR ){ + fatalError("object \"%s\" is not a directory", zDir); + } + rc = sqlite3_open(zDb, &db); + if( rc ){ + fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db)); + } + sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0, + writefileFunc, 0, 0); + zSql = sqlite3_mprintf( + "SELECT writefile(printf('%s/%%06d',k),v) FROM kv;", + zDir + ); + rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg); + if( rc ) fatalError("database create failed: %s", zErrMsg); + sqlite3_free(zSql); + sqlite3_close(db); + return 0; +} + +/* +** Read the content of file zName into memory obtained from sqlite3_malloc64() +** and return a pointer to the buffer. The caller is responsible for freeing +** the memory. +** +** If parameter pnByte is not NULL, (*pnByte) is set to the number of bytes +** read. +** +** For convenience, a nul-terminator byte is always appended to the data read +** from the file before the buffer is returned. This byte is not included in +** the final value of (*pnByte), if applicable. +** +** NULL is returned if any error is encountered. The final value of *pnByte +** is undefined in this case. +*/ +static unsigned char *readFile(const char *zName, int *pnByte){ + FILE *in = fopen(zName, "rb"); + long nIn; + size_t nRead; + unsigned char *pBuf; + if( in==0 ) return 0; + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + pBuf = sqlite3_malloc64( nIn+1 ); + if( pBuf==0 ) return 0; + nRead = fread(pBuf, nIn, 1, in); + fclose(in); + if( nRead!=1 ){ + sqlite3_free(pBuf); + return 0; + } + pBuf[nIn] = 0; + if( pnByte ) *pnByte = nIn; + return pBuf; +} + +/* +** Return the current time in milliseconds since the beginning of +** the Julian epoch. +*/ +static sqlite3_int64 timeOfDay(void){ + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t; +} + +/* Blob access order */ +#define ORDER_ASC 1 +#define ORDER_DESC 2 +#define ORDER_RANDOM 3 + +/* +** Run a performance test +*/ +static int runMain(int argc, char **argv){ + int eType; /* Is zDb a database or a directory? */ + char *zDb; /* Database or directory name */ + int i; /* Loop counter */ + int rc; /* Return code from SQLite calls */ + int nCount = 1000; /* Number of blob fetch operations */ + int nExtra = 0; /* Extra cycles */ + int iKey = 1; /* Next blob key */ + int iMax = 1000; /* Largest allowed key */ + int bBlobApi = 0; /* Use the incremental blob I/O API */ + int eOrder = ORDER_ASC; /* Access order */ + sqlite3 *db = 0; /* Database connection */ + sqlite3_stmt *pStmt = 0; /* Prepared statement for SQL access */ + sqlite3_blob *pBlob = 0; /* Handle for incremental Blob I/O */ + sqlite3_int64 tmStart; /* Start time */ + sqlite3_int64 tmElapsed; /* Elapsed time */ + int nData = 0; /* Bytes of data */ + sqlite3_int64 nTotal = 0; /* Total data read */ + unsigned char *pData; /* Content of the blob */ + + + assert( strcmp(argv[1],"run")==0 ); + assert( argc>=3 ); + zDb = argv[2]; + eType = pathType(zDb); + if( eType==PATH_OTHER ) fatalError("unknown object type: \"%s\"", zDb); + if( eType==PATH_NEXIST ) fatalError("object does not exist: \"%s\"", zDb); + for(i=3; iiMax ) iKey = 1; + }else if( eOrder==ORDER_DESC ){ + iKey--; + if( iKey<=0 ) iKey = iMax; + }else{ + iKey = (randInt()%iMax)+1; + } + nTotal += nData; + if( nData==0 ){ nCount++; nExtra++; } + } + if( pStmt ) sqlite3_finalize(pStmt); + if( pBlob ) sqlite3_blob_close(pBlob); + if( db ) sqlite3_close(db); + tmElapsed = timeOfDay() - tmStart; + if( nExtra ){ + printf("%d cycles due to %d misses\n", nCount, nExtra); + } + printf("Total elapsed time: %.3f\n", tmElapsed/1000.0); + printf("Microseconds per BLOB read: %.3f\n", tmElapsed*1000.0/nCount); + printf("Content read rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed)); + return 0; +} + + +int main(int argc, char **argv){ + if( argc<3 ) showHelp(); + if( strcmp(argv[1],"init")==0 ){ + return initMain(argc, argv); + } + if( strcmp(argv[1],"export")==0 ){ + return exportMain(argc, argv); + } + if( strcmp(argv[1],"run")==0 ){ + return runMain(argc, argv); + } + showHelp(); + return 0; +}