From: drh Date: Sat, 6 Apr 2013 13:09:11 +0000 (+0000) Subject: Many improvements to the mptest program. Added a simple test script. X-Git-Tag: version-3.7.17~105^2~9 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3f5bc38037eda131bd0505cd8961e4955d5ce0cf;p=thirdparty%2Fsqlite.git Many improvements to the mptest program. Added a simple test script. FossilOrigin-Name: 07b0401a9b61b1664fc6dcddac3b5969fc0f481a --- diff --git a/manifest b/manifest index 0c4454579c..53c2472e9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sprototype\sTH3-script-style\stest\sharness\sthat\sstarts\smultiple\sprocesses\noperating\son\sthe\ssame\sdatabase\sfile\sat\sthe\ssame\stime. -D 2013-04-06T00:19:37.887 +C Many\simprovements\sto\sthe\smptest\sprogram.\s\sAdded\sa\ssimple\stest\sscript. +D 2013-04-06T13:09:11.141 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,9 @@ F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 -F mptest/mptest.c 04110c35f415218e54e030b2e37215ad4adf01f3 +F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 +F mptest/mptest.c d73b294c354719f33086d8d3ebc273553b26073c +F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1045,10 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b095e2cdb61ca8487255687f58fb1024d40f3986 -R c143d7671ed86699bfdf0a0519618d31 -T *branch * mptest -T *sym-mptest * -T -sym-trunk * +P c318fafe686120d7fb8e487eb3bb4942d497665c +R e5c361902695f1e216fb1ed024ee2204 U drh -Z f0aabffa9579562b9fff6a9c66b5a272 +Z 1a794dfe7c097a8f7d97d3bcdba2a731 diff --git a/manifest.uuid b/manifest.uuid index fcf249224b..a5a84885b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c318fafe686120d7fb8e487eb3bb4942d497665c \ No newline at end of file +07b0401a9b61b1664fc6dcddac3b5969fc0f481a \ No newline at end of file diff --git a/mptest/config01.test b/mptest/config01.test new file mode 100644 index 0000000000..be6b570678 --- /dev/null +++ b/mptest/config01.test @@ -0,0 +1,19 @@ +/* +** Configure five tasks in different ways, then run tests. +*/ +PRAGMA page_size=8192; +--task 1 + PRAGMA journal_mode=PERSIST; + PRAGMA mmap_limit=0; +--end +--task 2 + PRAGMA journal_mode=TRUNCATE; + PRAGMA mmap_limit=28672; +--end +--task 3 + PRAGMA journal_mode=MEMORY; +--end +--task 4 + PRAGMA journal_mode=OFF; +--end +--source multiwrite01.test diff --git a/mptest/mptest.c b/mptest/mptest.c index 01660af690..e07e974de3 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -57,8 +57,13 @@ static struct Global { int iTrace; /* Tracing level */ int bSqlTrace; /* True to trace SQL commands */ int nError; /* Number of errors */ + int nTest; /* Number of --match operators */ + int iTimeout; /* Milliseconds until a busy timeout */ } g; +/* Default timeout */ +#define DEFAULT_TIMEOUT 10000 + /* ** Print a message adding zPrefix[] to the beginning of every line. */ @@ -226,7 +231,9 @@ static void fatalError(const char *zFormat, ...){ sqlite3_free(zMsg); if( g.db ){ int nTry = 0; - while( trySql("CREATE TABLE halt(x);")==SQLITE_BUSY && (nTry++)<100 ){ + g.iTimeout = 0; + while( trySql("UPDATE client SET wantHalt=1;")==SQLITE_BUSY + && (nTry++)<100 ){ sqlite3_sleep(10); } } @@ -262,6 +269,18 @@ static int clipLength(const char *z){ return n; } +/* +** Busy handler with a g.iTimeout-millisecond timeout +*/ +static int busyHandler(void *pCD, int count){ + if( count*10>g.iTimeout ){ + if( g.iTimeout>0 ) errorMessage("timeout after %dms", g.iTimeout); + return 0; + } + sqlite3_sleep(10); + return 1; +} + /* ** SQL Trace callback */ @@ -408,6 +427,7 @@ static int evalSql(String *p, const char *zFormat, ...){ va_start(ap, zFormat); zSql = sqlite3_vmprintf(zFormat, ap); va_end(ap); + assert( g.iTimeout>0 ); rc = sqlite3_exec(g.db, zSql, evalCallback, p, &zErrMsg); sqlite3_free(zSql); if( rc ){ @@ -435,21 +455,34 @@ static int startScript( sqlite3_stmt *pStmt = 0; int taskId; int rc; + int totalTime = 0; *pzScript = 0; + g.iTimeout = 0; while(1){ - if( trySql("SELECT * FROM halt")==SQLITE_OK ) return SQLITE_DONE; rc = trySql("BEGIN IMMEDIATE"); if( rc==SQLITE_BUSY ){ sqlite3_sleep(10); + totalTime += 10; continue; } if( rc!=SQLITE_OK ){ fatalError("%s\nBEGIN IMMEDIATE", sqlite3_errmsg(g.db)); } - if( g.nError ){ - runSql("UPDATE clienterror SET cnt=cnt+%d", g.nError); + if( g.nError || g.nTest ){ + runSql("UPDATE counters SET nError=nError+%d, nTest=nTest+%d", + g.nError, g.nTest); g.nError = 0; + g.nTest = 0; + } + pStmt = prepareSql("SELECT 1 FROM client WHERE id=%d AND wantHalt",iClient); + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + if( rc==SQLITE_ROW ){ + runSql("DELETE FROM client WHERE id=%d", iClient); + runSql("COMMIT"); + g.iTimeout = DEFAULT_TIMEOUT; + return SQLITE_DONE; } pStmt = prepareSql( "SELECT script, id FROM task" @@ -466,37 +499,58 @@ static int startScript( " SET starttime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" " WHERE id=%d;", taskId); runSql("COMMIT;"); + g.iTimeout = DEFAULT_TIMEOUT; return SQLITE_OK; } sqlite3_finalize(pStmt); if( rc==SQLITE_DONE ){ + if( totalTime>30000 ){ + errorMessage("Waited over 30 seconds with no work. Giving up."); + runSql("DELETE FROM client WHERE id=%d; COMMIT;", iClient); + sqlite3_close(g.db); + exit(1); + } runSql("COMMIT;"); sqlite3_sleep(100); + totalTime += 100; continue; } fatalError("%s", sqlite3_errmsg(g.db)); } + g.iTimeout = DEFAULT_TIMEOUT; } /* -** Mark a script as having finished. +** Mark a script as having finished. Remove the CLIENT table entry +** if bShutdown is true. */ -static int finishScript(int taskId){ - sqlite3_stmt *pStmt; - int rc; - pStmt = prepareSql( - "UPDATE task" - " SET endtime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" - " WHERE id=%d;", taskId); - do{ - rc = sqlite3_step(pStmt); - sqlite3_reset(pStmt); - }while( rc==SQLITE_BUSY ); - sqlite3_finalize(pStmt); - if( rc!=SQLITE_DONE ){ - fatalError("%s\n", sqlite3_errmsg(g.db)); +static int finishScript(int iClient, int taskId, int bShutdown){ + runSql("UPDATE task" + " SET endtime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" + " WHERE id=%d;", taskId); + if( bShutdown ){ + runSql("DELETE FROM client WHERE id=%d", iClient); + } + return SQLITE_OK; +} + +/* +** Start up a client process for iClient, if it is not already +** running. If the client is already running, then this routine +** is a no-op. +*/ +static void startClient(int iClient){ + runSql("INSERT OR IGNORE INTO client VALUES(%d,0)", iClient); + if( sqlite3_changes(g.db) ){ + char *zSys; + zSys = sqlite3_mprintf( + "%s \"%s\" --client %d --trace %d %s&", + g.argv0, g.zDbFile, iClient, g.iTrace, + g.bSqlTrace ? "--sqltrace " : ""); + + system(zSys); + sqlite3_free(zSys); } - return rc; } /* @@ -597,13 +651,18 @@ static void waitForClient(int iClient, int iTimeout, char *zErrPrefix){ int rc; if( iClient>0 ){ pStmt = prepareSql( - "SELECT 1 FROM task WHERE client=%d AND endtime IS NULL", + "SELECT 1 FROM task" + " WHERE client=%d" + " AND client IN (SELECT id FROM client)" + " AND endtime IS NULL", iClient); }else{ pStmt = prepareSql( - "SELECT 1 FROM task WHERE client=%d AND endtime IS NULL", - iClient); + "SELECT 1 FROM task" + " WHERE client IN (SELECT id FROM client)" + " AND endtime IS NULL"); } + g.iTimeout = 0; while( ((rc = sqlite3_step(pStmt))==SQLITE_BUSY || rc==SQLITE_ROW) && iTimeout>0 ){ @@ -612,6 +671,7 @@ static void waitForClient(int iClient, int iTimeout, char *zErrPrefix){ iTimeout -= 50; } sqlite3_finalize(pStmt); + g.iTimeout = DEFAULT_TIMEOUT; if( rc!=SQLITE_DONE ){ if( zErrPrefix==0 ) zErrPrefix = ""; if( iClient>0 ){ @@ -639,16 +699,13 @@ static void runScript( int ii = 0; int iBegin = 0; int n, c, j; - int rc; int len; int nArg; String sResult; char zCmd[30]; char zError[1000]; char azArg[MX_ARG][100]; - unsigned char isRunning[100]; - memset(isRunning, 0, sizeof(isRunning)); memset(&sResult, 0, sizeof(sResult)); stringReset(&sResult); while( (c = zScript[ii])!=0 ){ @@ -699,7 +756,7 @@ static void runScript( */ if( strcmp(zCmd, "exit")==0 ){ int rc = atoi(azArg[0]); - finishScript(taskId); + finishScript(iClient, taskId, 1); if( rc==0 ) sqlite3_close(g.db); exit(rc); }else @@ -727,6 +784,7 @@ static void runScript( errorMessage("line %d of %s:\nExpected [%.*s]\n Got [%s]", prevLine, zFilename, len-jj-1, zAns, sResult.z); } + g.nTest++; stringReset(&sResult); }else @@ -736,7 +794,7 @@ static void runScript( ** Run a subscript from a separate file. */ if( strcmp(zCmd, "source")==0 ){ - char *zNewFile = azArg[1]; + char *zNewFile = azArg[0]; char *zNewScript = readFile(zNewFile); if( g.iTrace ) logMessage("begin script [%s]\n", zNewFile); runScript(0, 0, zNewScript, zNewFile); @@ -762,26 +820,9 @@ static void runScript( */ if( strcmp(zCmd, "start")==0 ){ int iNewClient = atoi(azArg[0]); - char *zSys; - if( iNewClient<1 || iNewClient>=sizeof(isRunning) ){ - errorMessage("line %d of %s: bad client number: %d", - prevLine, zFilename, iNewClient); - goto start_error; + if( iNewClient>0 ){ + startClient(iNewClient); } - if( isRunning[iNewClient] ){ - errorMessage("line %d of %s: client already running: %d", - prevLine, zFilename, iNewClient); - goto start_error; - } - zSys = sqlite3_mprintf( - "%s \"%s\" --client %d --trace %d %s&", - g.argv0, g.zDbFile, iNewClient, g.iTrace, - g.bSqlTrace ? "--sqltrace " : ""); - - system(zSys); - sqlite3_free(zSys); - isRunning[iNewClient] = 1; - start_error: {/* no-op */} }else /* @@ -803,30 +844,24 @@ static void runScript( ** ** --end ** - ** Assign work to a client. + ** Assign work to a client. Start the client if it is not running + ** already. */ if( strcmp(zCmd, "task")==0 ){ int iTarget = atoi(azArg[0]); int iEnd; char *zTask; - sqlite3_stmt *pStmt; iEnd = findEnd(zScript+ii+len, &lineno); - if( iTarget<0 || iTarget>=sizeof(isRunning) - || !isRunning[iTarget] ){ - errorMessage("line %d of %s: client %d is not running", + if( iTarget<0 ){ + errorMessage("line %d of %s: bad client number: %d", prevLine, zFilename, iTarget); - goto task_error; - } - zTask = sqlite3_mprintf("%.*s", iEnd, zScript+ii+len); - pStmt = prepareSql("INSERT INTO task(client,script)" - " VALUES(%d,'%q')", iTarget, zTask); - sqlite3_free(zTask); - while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY ){ - sqlite3_reset(pStmt); - sqlite3_sleep(10); + }else{ + zTask = sqlite3_mprintf("%.*s", iEnd, zScript+ii+len); + startClient(iTarget); + runSql("INSERT INTO task(client,script)" + " VALUES(%d,'%q')", iTarget, zTask); + sqlite3_free(zTask); } - sqlite3_finalize(pStmt); - task_error: iEnd += tokenLength(zScript+ii+len+iEnd, &lineno); len += iEnd; iBegin = ii+len; @@ -964,6 +999,8 @@ int main(int argc, char **argv){ } rc = sqlite3_open_v2(g.zDbFile, &g.db, openFlags, g.zVfs); if( rc ) fatalError("cannot open [%s]", g.zDbFile); + sqlite3_busy_handler(g.db, busyHandler, 0); + g.iTimeout = DEFAULT_TIMEOUT; if( g.bSqlTrace ) sqlite3_trace(g.db, sqlTraceCallback, 0); if( iClient>0 ){ if( n>0 ) unrecognizedArguments(argv[0], n, argv+2); @@ -977,12 +1014,13 @@ int main(int argc, char **argv){ iClient, taskId); runScript(iClient, taskId, zScript, zTaskName); if( g.iTrace ) logMessage("end task %d", taskId); - finishScript(taskId); + finishScript(iClient, taskId, 0); sqlite3_sleep(10); } if( g.iTrace ) logMessage("end-client"); }else{ sqlite3_stmt *pStmt; + int iTimeout; if( n==0 ){ fatalError("missing script filename"); } @@ -995,8 +1033,11 @@ int main(int argc, char **argv){ " endtime DATE,\n" " script TEXT\n" ");" - "CREATE TABLE clienterror(cnt);\n" - "INSERT INTO clienterror VALUES(0);\n" + "CREATE INDEX task_i1 ON task(client, starttime);\n" + "CREATE INDEX task_i2 ON task(client, endtime);\n" + "CREATE TABLE counters(nError,nTest);\n" + "INSERT INTO counters VALUES(0,0);\n" + "CREATE TABLE client(id INTEGER PRIMARY KEY, wantHalt);\n" ); zScript = readFile(argv[2]); if( g.iTrace ) logMessage("begin script [%s]\n", argv[2]); @@ -1004,16 +1045,25 @@ int main(int argc, char **argv){ sqlite3_free(zScript); if( g.iTrace ) logMessage("end script [%s]\n", argv[2]); waitForClient(0, 2000, "during shutdown...\n"); - while( trySql("CREATE TABLE halt(x);")==SQLITE_BUSY ){ + trySql("UPDATE client SET wantHalt=1"); + sqlite3_sleep(10); + g.iTimeout = 0; + iTimeout = 1000; + while( ((rc = trySql("SELECT 1 FROM client"))==SQLITE_BUSY + || rc==SQLITE_ROW) && iTimeout>0 ){ sqlite3_sleep(10); + iTimeout -= 10; } sqlite3_sleep(100); - pStmt = prepareSql("SELECT cnt FROM clienterror"); - while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY ){ + pStmt = prepareSql("SELECT nError, nTest FROM counters"); + iTimeout = 1000; + while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY && iTimeout>0 ){ sqlite3_sleep(10); + iTimeout -= 10; } if( rc==SQLITE_ROW ){ g.nError += sqlite3_column_int(pStmt, 0); + g.nTest += sqlite3_column_int(pStmt, 1); } sqlite3_finalize(pStmt); } @@ -1021,11 +1071,7 @@ int main(int argc, char **argv){ maybeClose(g.pLog); maybeClose(g.pErrLog); if( iClient==0 ){ - if( g.nError ){ - printf("ERRORS: %d\n", g.nError); - }else if( g.iTrace ){ - printf("All OK\n"); - } + printf("Summary: %d errors in %d tests\n", g.nError, g.nTest); } return g.nError>0; } diff --git a/mptest/multiwrite01.test b/mptest/multiwrite01.test new file mode 100644 index 0000000000..dc71884047 --- /dev/null +++ b/mptest/multiwrite01.test @@ -0,0 +1,141 @@ +/* +** This script sets up five different tasks all writing and updating +** the database at the same time, but each in its own table. +*/ +--task 1 + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t1 VALUES(1, randomblob(2000)); + INSERT INTO t1 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t1 SELECT a+2, randomblob(1500) FROM t1; + INSERT INTO t1 SELECT a+4, randomblob(1500) FROM t1; + INSERT INTO t1 SELECT a+8, randomblob(1500) FROM t1; + --sleep 1 + INSERT INTO t1 SELECT a+16, randomblob(1500) FROM t1; + --sleep 1 + INSERT INTO t1 SELECT a+32, randomblob(1500) FROM t1; + SELECT count(*) FROM t1; + --match 64 + SELECT avg(length(b)) FROM t1; + --match 1500.0 + --sleep 2 + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t1; + --match 247 +--end + + +--task 2 + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t2 VALUES(1, randomblob(2000)); + INSERT INTO t2 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t2 SELECT a+2, randomblob(1500) FROM t2; + INSERT INTO t2 SELECT a+4, randomblob(1500) FROM t2; + INSERT INTO t2 SELECT a+8, randomblob(1500) FROM t2; + --sleep 1 + INSERT INTO t2 SELECT a+16, randomblob(1500) FROM t2; + --sleep 1 + INSERT INTO t2 SELECT a+32, randomblob(1500) FROM t2; + SELECT count(*) FROM t2; + --match 64 + SELECT avg(length(b)) FROM t2; + --match 1500.0 + --sleep 2 + UPDATE t2 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t2; + --match 247 +--end + +--task 3 + DROP TABLE IF EXISTS t3; + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t3 VALUES(1, randomblob(2000)); + INSERT INTO t3 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t3 SELECT a+2, randomblob(1500) FROM t3; + INSERT INTO t3 SELECT a+4, randomblob(1500) FROM t3; + INSERT INTO t3 SELECT a+8, randomblob(1500) FROM t3; + --sleep 1 + INSERT INTO t3 SELECT a+16, randomblob(1500) FROM t3; + --sleep 1 + INSERT INTO t3 SELECT a+32, randomblob(1500) FROM t3; + SELECT count(*) FROM t3; + --match 64 + SELECT avg(length(b)) FROM t3; + --match 1500.0 + --sleep 2 + UPDATE t3 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t3; + --match 247 +--end + +--task 4 + DROP TABLE IF EXISTS t4; + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t4 VALUES(1, randomblob(2000)); + INSERT INTO t4 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t4 SELECT a+2, randomblob(1500) FROM t4; + INSERT INTO t4 SELECT a+4, randomblob(1500) FROM t4; + INSERT INTO t4 SELECT a+8, randomblob(1500) FROM t4; + --sleep 1 + INSERT INTO t4 SELECT a+16, randomblob(1500) FROM t4; + --sleep 1 + INSERT INTO t4 SELECT a+32, randomblob(1500) FROM t4; + SELECT count(*) FROM t4; + --match 64 + SELECT avg(length(b)) FROM t4; + --match 1500.0 + --sleep 2 + UPDATE t4 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t4; + --match 247 +--end + +--task 5 + DROP TABLE IF EXISTS t5; + CREATE TABLE t5(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t5 VALUES(1, randomblob(2000)); + INSERT INTO t5 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t5 SELECT a+2, randomblob(1500) FROM t5; + INSERT INTO t5 SELECT a+4, randomblob(1500) FROM t5; + INSERT INTO t5 SELECT a+8, randomblob(1500) FROM t5; + --sleep 1 + INSERT INTO t5 SELECT a+16, randomblob(1500) FROM t5; + --sleep 1 + INSERT INTO t5 SELECT a+32, randomblob(1500) FROM t5; + SELECT count(*) FROM t5; + --match 64 + SELECT avg(length(b)) FROM t5; + --match 1500.0 + --sleep 2 + UPDATE t5 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t5; + --match 247 +--end + +--wait all +SELECT count(*), total(length(b)) FROM t1; +--match 64 247 +SELECT count(*), total(length(b)) FROM t2; +--match 64 247 +SELECT count(*), total(length(b)) FROM t3; +--match 64 247 +SELECT count(*), total(length(b)) FROM t4; +--match 64 247 +SELECT count(*), total(length(b)) FROM t5; +--match 64 247 +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; +DROP TABLE t5;