From: drh Date: Fri, 2 Jun 2017 19:31:46 +0000 (+0000) Subject: Work toward enhancing kvtest to measure write performance. X-Git-Tag: version-3.20.0~222 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a2e71ced793cd465d7d218d1ed5649ebdc2c16a6;p=thirdparty%2Fsqlite.git Work toward enhancing kvtest to measure write performance. FossilOrigin-Name: fc73e7d2f16386f96c55c42f9830193f7c178521a7ad90c3117b85ef629b5ce4 --- diff --git a/manifest b/manifest index 3a6ab7f29f..6578836d33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\sOP_Seek\sopcode\sinto\sOP_DeferredSeek\sfor\sbetter\nclarity\sof\sfunction.\s\sNo\sfunctional\scode\schanges. -D 2017-06-02T15:44:22.919 +C Work\stoward\senhancing\skvtest\sto\smeasure\swrite\sperformance. +D 2017-06-02T19:31:46.986 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -925,7 +925,7 @@ F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb2848 F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c b9a9822dda05a1aa481215a52e2fc93cd8b22ee5 +F test/kvtest.c 63f9a1c31391f304e87d4854aea54c01afa9e4f6d05e11124598c7415b893a85 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 @@ -1582,7 +1582,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d637feb4e3032ee0067c204b56d2af5f7eab228e645f2e77b0b043c2b299724d -R 16a899dc6b5945a11289cea26923fb73 +P ab33d299c7dab52703d06f3441c8a98c6c809b2612ec65d71aab2919bd2b1540 +R b213b6f467b3c9502658fd4e2800b6d6 U drh -Z 08866295af3c1947bc08b06ca4b13f90 +Z 24f39ffaffd16b2467372fb4bd435c2b diff --git a/manifest.uuid b/manifest.uuid index c863243890..c0a9616b92 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ab33d299c7dab52703d06f3441c8a98c6c809b2612ec65d71aab2919bd2b1540 \ No newline at end of file +fc73e7d2f16386f96c55c42f9830193f7c178521a7ad90c3117b85ef629b5ce4 \ No newline at end of file diff --git a/test/kvtest.c b/test/kvtest.c index 4fa733f1a7..3c34d0a1fb 100644 --- a/test/kvtest.c +++ b/test/kvtest.c @@ -90,12 +90,14 @@ static const char zHelp[] = " --cache-size N Database cache size\n" " --count N Read N blobs\n" " --desc Read blobs in descending order\n" +" --integrity-check Run 'PRAGMA integrity_check' after test\n" " --max-id N Maximum blob key to use\n" " --mmap N Mmap as much as N bytes of DBFILE\n" " --jmode MODE Set MODE journal mode prior to starting\n" " --random Read blobs in a random order\n" " --start N Start reading with this blob key\n" " --stats Output operating stats before exiting\n" +" --update To an overwrite test\n" ; /* Reference resources used */ @@ -118,6 +120,38 @@ static const char zHelp[] = # define access _access #endif +#include +#include + +/* +** The following macros are used to cast pointers to integers and +** integers to pointers. The way you do this varies from one compiler +** to the next, so we have developed the following set of #if statements +** to generate appropriate macros for a wide range of compilers. +** +** The correct "ANSI" way to do this is to use the intptr_t type. +** Unfortunately, that typedef is not available on all compilers, or +** if it is available, it requires an #include of specific headers +** that vary from one machine to the next. +** +** Ticket #3860: The llvm-gcc-4.2 compiler from Apple chokes on +** the ((void*)&((char*)0)[X]) construct. But MSVC chokes on ((void*)(X)). +** So we have to define the macros in different ways depending on the +** compiler. +*/ +#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ +# define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) +# define SQLITE_PTR_TO_INT(X) ((sqlite3_int64)(__PTRDIFF_TYPE__)(X)) +#elif !defined(__GNUC__) /* Works for compilers other than LLVM */ +# define SQLITE_INT_TO_PTR(X) ((void*)&((char*)0)[X]) +# define SQLITE_PTR_TO_INT(X) ((sqlite3_int64)(((char*)X)-(char*)0)) +#elif defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ +# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) +# define SQLITE_PTR_TO_INT(X) ((sqlite3_int64)(intptr_t)(X)) +#else /* Generates a warning - but it always works */ +# define SQLITE_INT_TO_PTR(X) ((void*)(X)) +# define SQLITE_PTR_TO_INT(X) ((sqlite3_int64)(X)) +#endif /* ** Show thqe help text and quit. @@ -409,6 +443,26 @@ static void writefileFunc( sqlite3_result_int64(context, rc); } +/* +** remember(V,PTR) +** +** Return the integer value V. Also save the value of V in a +** C-language variable whose address is PTR. +*/ +static void rememberFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + sqlite3_int64 v; + sqlite3_int64 ptr; + assert( argc==2 ); + v = sqlite3_value_int64(argv[0]); + ptr = sqlite3_value_int64(argv[1]); + *(sqlite3_int64*)SQLITE_INT_TO_PTR(ptr) = v; + sqlite3_result_int64(pCtx, v); +} + /* ** Export the kv table to individual files in the filesystem */ @@ -483,6 +537,40 @@ static unsigned char *readFile(const char *zName, int *pnByte){ return pBuf; } +/* +** Overwrite a file with randomness. Do not change the size of the +** file. +*/ +static void updateFile(const char *zName, int *pnByte){ + FILE *out; /* FILE from which to read content of zName */ + sqlite3_int64 sz; /* Size of zName in bytes */ + size_t nWritten; /* Number of bytes actually read */ + unsigned char *pBuf; /* Content to store on disk */ + + sz = fileSize(zName); + if( sz<0 ){ + fatalError("No such file: \"%s\"", zName); + } + *pnByte = (int)sz; + if( sz==0 ) return; + pBuf = sqlite3_malloc64( sz ); + if( pBuf==0 ){ + fatalError("Cannot allocate %lld bytes\n", sz); + } + sqlite3_randomness((int)sz, pBuf); + out = fopen(zName, "wb"); + if( out==0 ){ + fatalError("Cannot open \"%s\" for writing\n", zName); + } + nWritten = fwrite(pBuf, 1, (size_t)sz, out); + fclose(out); + if( nWritten!=(size_t)sz ){ + fatalError("Wrote only %d of %d bytes to \"%s\"\n", + (int)nWritten, (int)sz, zName); + } + sqlite3_free(pBuf); +} + /* ** Return the current time in milliseconds since the beginning of ** the Julian epoch. @@ -637,6 +725,9 @@ static int runMain(int argc, char **argv){ int bBlobApi = 0; /* Use the incremental blob I/O API */ int bStats = 0; /* Print stats before exiting */ int eOrder = ORDER_ASC; /* Access order */ + int isUpdateTest = 0; /* Do in-place updates rather than reads */ + int doIntegrityCk = 0; /* Run PRAGMA integrity_check after the test */ + int noSync = 0; /* Disable synchronous mode */ sqlite3 *db = 0; /* Database connection */ sqlite3_stmt *pStmt = 0; /* Prepared statement for SQL access */ sqlite3_blob *pBlob = 0; /* Handle for incremental Blob I/O */ @@ -713,8 +804,26 @@ static int runMain(int argc, char **argv){ bStats = 1; continue; } + if( strcmp(z, "-update")==0 ){ + isUpdateTest = 1; + continue; + } + if( strcmp(z, "-integrity-check")==0 ){ + doIntegrityCk = 1; + continue; + } + if( strcmp(z, "-nosync")==0 ){ + noSync = 1; + continue; + } fatalError("unknown option: \"%s\"", argv[i]); } + if( eType==PATH_DB ){ + /* Recover any prior crashes prior to starting the timer */ + sqlite3_open(zDb, &db); + sqlite3_exec(db, "SELECT rowid FROM sqlite_master LIMIT 1", 0, 0, 0); + sqlite3_close(db); + } tmStart = timeOfDay(); if( eType==PATH_DB ){ char *zSql; @@ -724,9 +833,13 @@ static int runMain(int argc, char **argv){ } zSql = sqlite3_mprintf("PRAGMA mmap_size=%d", mmapSize); sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); zSql = sqlite3_mprintf("PRAGMA cache_size=%d", iCache); sqlite3_exec(db, zSql, 0, 0, 0); sqlite3_free(zSql); + if( noSync ){ + sqlite3_exec(db, "PRAGMA synchronous=OFF", 0, 0, 0); + } pStmt = 0; sqlite3_prepare_v2(db, "PRAGMA page_size", -1, &pStmt, 0); if( sqlite3_step(pStmt)==SQLITE_ROW ){ @@ -770,13 +883,18 @@ static int runMain(int argc, char **argv){ char *zKey; zKey = sqlite3_mprintf("%s/%06d", zDb, iKey); nData = 0; - pData = readFile(zKey, &nData); + if( isUpdateTest ){ + updateFile(zKey, &nData); + }else{ + pData = readFile(zKey, &nData); + sqlite3_free(pData); + } sqlite3_free(zKey); - sqlite3_free(pData); }else if( bBlobApi ){ /* CASE 2: Reading from database using the incremental BLOB I/O API */ if( pBlob==0 ){ - rc = sqlite3_blob_open(db, "main", "kv", "v", iKey, 0, &pBlob); + rc = sqlite3_blob_open(db, "main", "kv", "v", iKey, + isUpdateTest, &pBlob); if( rc ){ fatalError("could not open sqlite3_blob handle: %s", sqlite3_errmsg(db)); @@ -791,17 +909,36 @@ static int runMain(int argc, char **argv){ pData = sqlite3_realloc(pData, nAlloc); } if( pData==0 ) fatalError("cannot allocate %d bytes", nData+1); - rc = sqlite3_blob_read(pBlob, pData, nData, 0); - if( rc!=SQLITE_OK ){ - fatalError("could not read the blob at %d: %s", iKey, - sqlite3_errmsg(db)); + if( isUpdateTest ){ + sqlite3_randomness((int)nData, pData); + rc = sqlite3_blob_write(pBlob, pData, nData, 0); + if( rc!=SQLITE_OK ){ + fatalError("could not write the blob at %d: %s", iKey, + sqlite3_errmsg(db)); + } + }else{ + rc = sqlite3_blob_read(pBlob, pData, nData, 0); + if( rc!=SQLITE_OK ){ + fatalError("could not read the blob at %d: %s", iKey, + sqlite3_errmsg(db)); + } } } }else{ /* CASE 3: Reading from database using SQL */ if( pStmt==0 ){ - rc = sqlite3_prepare_v2(db, - "SELECT v FROM kv WHERE k=?1", -1, &pStmt, 0); + if( isUpdateTest ){ + sqlite3_create_function(db, "remember", 2, SQLITE_UTF8, 0, + rememberFunc, 0, 0); + + rc = sqlite3_prepare_v2(db, + "UPDATE kv SET v=randomblob(remember(length(v),?2))" + " WHERE k=?1", -1, &pStmt, 0); + sqlite3_bind_int64(pStmt, 2, SQLITE_PTR_TO_INT(&nData)); + }else{ + rc = sqlite3_prepare_v2(db, + "SELECT v FROM kv WHERE k=?1", -1, &pStmt, 0); + } if( rc ){ fatalError("cannot prepare query: %s", sqlite3_errmsg(db)); } @@ -809,12 +946,11 @@ static int runMain(int argc, char **argv){ sqlite3_reset(pStmt); } sqlite3_bind_int(pStmt, 1, iKey); + nData = 0; rc = sqlite3_step(pStmt); if( rc==SQLITE_ROW ){ nData = sqlite3_column_bytes(pStmt, 0); pData = (unsigned char*)sqlite3_column_blob(pStmt, 0); - }else{ - nData = 0; } } if( eOrder==ORDER_ASC ){ @@ -835,13 +971,25 @@ static int runMain(int argc, char **argv){ if( bStats ){ display_stats(db, 0); } - if( db ) sqlite3_close(db); + if( db ){ + sqlite3_exec(db, "COMMIT", 0, 0, 0); + sqlite3_close(db); + } tmElapsed = timeOfDay() - tmStart; if( nExtra ){ printf("%d cycles due to %d misses\n", nCount, nExtra); } if( eType==PATH_DB ){ printf("SQLite version: %s\n", sqlite3_libversion()); + if( doIntegrityCk ){ + sqlite3_open(zDb, &db); + sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &pStmt, 0); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("integrity-check: %s\n", sqlite3_column_text(pStmt, 0)); + } + sqlite3_finalize(pStmt); + sqlite3_close(db); + } } printf("--count %d --max-id %d", nCount-nExtra, iMax); switch( eOrder ){ @@ -852,11 +1000,17 @@ static int runMain(int argc, char **argv){ if( eType==PATH_DB ){ printf("--cache-size %d --jmode %s\n", iCache, zJMode); printf("--mmap %d%s\n", mmapSize, bBlobApi ? " --blob-api" : ""); + if( noSync ) printf("--nosync\n"); } if( iPagesize ) printf("Database page size: %d\n", iPagesize); printf("Total elapsed time: %.3f\n", tmElapsed/1000.0); - printf("Microseconds per BLOB read: %.3f\n", tmElapsed*1000.0/nCount); - printf("Content read rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed)); + if( isUpdateTest ){ + printf("Microseconds per BLOB write: %.3f\n", tmElapsed*1000.0/nCount); + printf("Content write rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed)); + }else{ + printf("Microseconds per BLOB read: %.3f\n", tmElapsed*1000.0/nCount); + printf("Content read rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed)); + } return 0; }