From: dan Date: Thu, 17 Mar 2016 21:06:42 +0000 (+0000) Subject: Add an API to indicate the percentage progress of an rbu update. X-Git-Tag: version-3.12.0~34^2~6 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=e42195b0440c03403d6f928c9cf7865a49c9a4b4;p=thirdparty%2Fsqlite.git Add an API to indicate the percentage progress of an rbu update. FossilOrigin-Name: ffc58d2c2576a5b6e1c2c7112612c5760e711afd --- diff --git a/ext/rbu/rbuprogress.test b/ext/rbu/rbuprogress.test new file mode 100644 index 0000000000..9fcd014065 --- /dev/null +++ b/ext/rbu/rbuprogress.test @@ -0,0 +1,179 @@ +# 2016 March 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbuprogress + + +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +# +proc create_rbu1 {filename} { + forcedelete $filename + sqlite3 rbu1 $filename + rbu1 eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(2, 'two', 'three', 0); + INSERT INTO data_t1 VALUES(3, NULL, 8.2, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 3); + } + rbu1 close + return $filename +} + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +} + +do_test 1.1 { + create_rbu1 rbu.db + sqlite3rbu rbu test.db rbu.db + rbu stage_progress +} {0 0} +do_test 1.2 { rbu step ; rbu stage_progress } {3333 0} +do_test 1.3 { rbu step ; rbu stage_progress } {6666 0} +do_test 1.4 { rbu step ; rbu stage_progress } {10000 0} +do_test 1.5 { rbu step ; rbu stage_progress } {10000 0} +do_test 1.6 { rbu step ; rbu stage_progress } {10000 0} +do_test 1.7 { rbu step ; rbu stage_progress } {10000 5000} +do_test 1.8 { rbu step ; rbu stage_progress } {10000 10000} +do_test 1.9 { rbu step ; rbu stage_progress } {10000 10000} + +do_test 1.10 { + rbu close +} {SQLITE_DONE} + +#------------------------------------------------------------------------- + +proc do_sp_test {tn target rbu reslist} { + uplevel [list do_test $tn [subst -nocommands { + sqlite3rbu rbu $target $rbu + set res [list] + while 1 { + set rc [rbu step] + if {[set rc] != "SQLITE_OK"} { error "error 1" } + lappend res [lindex [rbu stage_progress] 0] + if {[lindex [set res] end]==10000} break + } + if {[set res] != [list $reslist]} { + error "reslist is incorrect (expect=$reslist got=[set res])" + } + + # One step to clean up the temporary tables used to update the only + # target table in the rbu database. And one more to move the *-oal + # file to *-wal. + rbu step + rbu step + + # Do the checkpoint. + while {[rbu step]=="SQLITE_OK"} { } + + rbu close + }] {SQLITE_DONE}] +} + +proc create_db_file {filename sql} { + forcedelete $filename + sqlite3 tmpdb $filename + tmpdb eval $sql + tmpdb close +} + +reset_db +do_test 2.1.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(5, 5, 5, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 2); + } +} {} +do_sp_test 2.1.1 test.db rbu.db {5000 10000} + +reset_db +do_test 2.2.0 { + execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + create_rbu1 rbu.db +} {rbu.db} +do_sp_test 2.2.1 test.db rbu.db {3333 6666 10000} + +reset_db +do_test 2.3.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(5, NULL, NULL, 1); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 3); + } +} {} +do_sp_test 2.3.1 test.db rbu.db {1666 3333 6000 8000 10000} + +reset_db +do_test 2.4.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(2, 4, 4, '.xx'); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + } +} {} +do_sp_test 2.4.1 test.db rbu.db {3333 6666 10000} + +reset_db +do_test 2.5.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, NULL, 4, '.xx'); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + } +} {} +do_sp_test 2.5.1 test.db rbu.db {10000} + +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 474e39fe8d..31d462b2b7 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -147,14 +147,15 @@ ** RBU_STATE_OALSZ: ** Valid if STAGE==1. The size in bytes of the *-oal file. */ -#define RBU_STATE_STAGE 1 -#define RBU_STATE_TBL 2 -#define RBU_STATE_IDX 3 -#define RBU_STATE_ROW 4 -#define RBU_STATE_PROGRESS 5 -#define RBU_STATE_CKPT 6 -#define RBU_STATE_COOKIE 7 -#define RBU_STATE_OALSZ 8 +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 +#define RBU_STATE_PHASEONESTEP 9 #define RBU_STAGE_OAL 1 #define RBU_STAGE_MOVE 2 @@ -200,6 +201,7 @@ struct RbuState { i64 nProgress; u32 iCookie; i64 iOalSz; + i64 nPhaseOneStep; }; struct RbuUpdateStmt { @@ -244,6 +246,7 @@ struct RbuObjIter { int iTnum; /* Root page of current object */ int iPkTnum; /* If eType==EXTERNAL, root of PK index */ int bUnique; /* Current index is unique */ + int nIndex; /* Number of aux. indexes on table zTbl */ /* Statements created by rbuObjIterPrepareAll() */ int nCol; /* Number of columns in current object */ @@ -314,6 +317,7 @@ struct sqlite3rbu { const char *zVfsName; /* Name of automatically created rbu vfs */ rbu_file *pTargetFd; /* File handle open on target db */ i64 iOalSz; + i64 nPhaseOneStep; /* The following state variables are used as part of the incremental ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding @@ -1144,6 +1148,7 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ ); } + pIter->nIndex = 0; while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ const char *zIdx = (const char*)sqlite3_column_text(pList, 1); sqlite3_stmt *pXInfo = 0; @@ -1157,6 +1162,7 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ } rbuFinalize(p, pXInfo); bIndex = 1; + pIter->nIndex++; } rbuFinalize(p, pList); @@ -1823,6 +1829,14 @@ static void rbuTmpInsertFunc( int rc = SQLITE_OK; int i; + assert( sqlite3_value_int(apVal[0])!=0 + || p->objiter.eType==RBU_PK_EXTERNAL + || p->objiter.eType==RBU_PK_NONE + ); + if( sqlite3_value_int(apVal[0])!=0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + } + for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); } @@ -2567,6 +2581,17 @@ static void rbuStepOneOp(sqlite3rbu *p, int eType){ assert( p->rc==SQLITE_OK ); assert( eType!=RBU_DELETE || pIter->zIdx==0 ); + assert( eType==RBU_DELETE || eType==RBU_IDX_DELETE + || eType==RBU_INSERT || eType==RBU_IDX_INSERT + ); + + /* If this is a delete, decrement nPhaseOneStep by nIndex. If the DELETE + ** statement below does actually delete a row, nPhaseOneStep will be + ** incremented by the same amount when SQL function rbu_tmp_insert() + ** is invoked by the trigger. */ + if( eType==RBU_DELETE ){ + p->nPhaseOneStep -= p->objiter.nIndex; + } if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){ pWriter = pIter->pDelete; @@ -2642,7 +2667,10 @@ static int rbuStep(sqlite3rbu *p){ rbuBadControlError(p); } else if( eType==RBU_REPLACE ){ - if( pIter->zIdx==0 ) rbuStepOneOp(p, RBU_DELETE); + if( pIter->zIdx==0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + rbuStepOneOp(p, RBU_DELETE); + } if( p->rc==SQLITE_OK ) rbuStepOneOp(p, RBU_INSERT); } else if( eType!=RBU_UPDATE ){ @@ -2652,6 +2680,7 @@ static int rbuStep(sqlite3rbu *p){ sqlite3_value *pVal; sqlite3_stmt *pUpdate = 0; assert( eType==RBU_UPDATE ); + p->nPhaseOneStep -= p->objiter.nIndex; rbuGetUpdateStmt(p, pIter, zMask, &pUpdate); if( pUpdate ){ int i; @@ -2729,6 +2758,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ "(%d, %d), " "(%d, %lld), " "(%d, %lld), " + "(%d, %lld), " "(%d, %lld) ", p->zStateDb, RBU_STATE_STAGE, eStage, @@ -2738,7 +2768,8 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ RBU_STATE_PROGRESS, p->nProgress, RBU_STATE_CKPT, p->iWalCksum, RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie, - RBU_STATE_OALSZ, p->iOalSz + RBU_STATE_OALSZ, p->iOalSz, + RBU_STATE_PHASEONESTEP, p->nPhaseOneStep ) ); assert( pInsert==0 || rc==SQLITE_OK ); @@ -2925,6 +2956,10 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); break; + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = (u32)sqlite3_column_int64(pStmt, 1); + break; + default: rc = SQLITE_CORRUPT; break; @@ -3032,6 +3067,97 @@ static void rbuDeleteVfs(sqlite3rbu *p){ } } +/* +** +*/ +static void rbuIndexCntFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3rbu *p = (sqlite3rbu*)sqlite3_user_data(pCtx); + sqlite3_stmt *pStmt = 0; + char *zErrmsg = 0; + int rc; + + assert( nVal==1 ); + + rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &zErrmsg, + sqlite3_mprintf("PRAGMA index_list = %Q", sqlite3_value_text(apVal[0])) + ); + if( rc!=SQLITE_OK ){ + sqlite3_result_error(pCtx, zErrmsg, -1); + }else{ + int nIndex = 0; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + nIndex++; + } + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + sqlite3_result_int(pCtx, nIndex); + }else{ + sqlite3_result_error(pCtx, sqlite3_errmsg(p->dbMain), -1); + } + } + + sqlite3_free(zErrmsg); +} + +/* +** If the RBU database contains the rbu_count table, use it to initialize +** the sqlite3rbu.nPhaseOneStep variable. The schema of the rbu_count table +** is assumed to contain the same columns as: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There should be one row in the table for each data_xxx table in the +** database. The 'tbl' column should contain the name of a data_xxx table, +** and the cnt column the number of rows it contains. +** +** sqlite3rbu.nPhaseOneStep is initialized to the sum of (1 + nIndex) * cnt +** for all rows in the rbu_count table, where nIndex is the number of +** indexes on the corresponding target database table. +*/ +static void rbuInitPhaseOneSteps(sqlite3rbu *p){ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + int bExists = 0; /* True if rbu_count exists */ + + p->nPhaseOneStep = -1; + + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_index_cnt", 1, SQLITE_UTF8, (void*)p, rbuIndexCntFunc, 0, 0 + ); + + /* Check for the rbu_count table. If it does not exist, or if an error + ** occurs, nPhaseOneStep will be left set to -1. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + "SELECT 1 FROM sqlite_master WHERE tbl_name = 'rbu_count'" + ); + } + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + bExists = 1; + } + p->rc = sqlite3_finalize(pStmt); + } + + if( p->rc==SQLITE_OK && bExists ){ + p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + "SELECT sum(cnt * (1 + rbu_index_cnt(rbu_target_name(tbl))))" + "FROM rbu_count" + ); + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->nPhaseOneStep = sqlite3_column_int64(pStmt, 0); + } + p->rc = sqlite3_finalize(pStmt); + } + } + } +} + /* ** Open and return a new RBU handle. */ @@ -3077,6 +3203,7 @@ sqlite3rbu *sqlite3rbu_open( if( pState->eStage==0 ){ rbuDeleteOalFile(p); + rbuInitPhaseOneSteps(p); p->eStage = RBU_STAGE_OAL; }else{ p->eStage = pState->eStage; @@ -3243,6 +3370,38 @@ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){ return pRbu->nProgress; } +void sqlite3rbu_stage_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){ + const int MAX_PROGRESS = 10000; + switch( p->eStage ){ + case RBU_STAGE_OAL: + if( p->nPhaseOneStep>0 ){ + *pnOne = (int)(MAX_PROGRESS * (i64)p->nProgress/(i64)p->nPhaseOneStep); + }else{ + *pnOne = -1; + } + *pnTwo = 0; + break; + + case RBU_STAGE_MOVE: + *pnOne = MAX_PROGRESS; + *pnTwo = 0; + break; + + case RBU_STAGE_CKPT: + *pnOne = MAX_PROGRESS; + *pnTwo = (int)(MAX_PROGRESS * (i64)p->nStep / (i64)p->nFrame); + break; + + case RBU_STAGE_DONE: + *pnOne = MAX_PROGRESS; + *pnTwo = MAX_PROGRESS; + break; + + default: + assert( 0 ); + } +} + int sqlite3rbu_savestate(sqlite3rbu *p){ int rc = p->rc; diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index f1a0f3cd84..fb81c85a0a 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -400,6 +400,8 @@ int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); */ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); +void sqlite3rbu_stage_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); + /* ** Create an RBU VFS named zName that accesses the underlying file-system ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index 3fa85b7569..e35d76e745 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -66,6 +66,7 @@ static int test_sqlite3rbu_cmd( {"create_rbu_delta", 2, ""}, /* 2 */ {"savestate", 2, ""}, /* 3 */ {"dbMain_eval", 3, "SQL"}, /* 4 */ + {"stage_progress", 2, ""}, /* 5 */ {0,0,0} }; int iCmd; @@ -136,6 +137,18 @@ static int test_sqlite3rbu_cmd( break; } + case 5: /* stage_progress */ { + int one, two; + Tcl_Obj *pObj; + sqlite3rbu_stage_progress(pRbu, &one, &two); + + pObj = Tcl_NewObj(); + Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(one)); + Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(two)); + Tcl_SetObjResult(interp, pObj); + break; + } + default: /* seems unlikely */ assert( !"cannot happen" ); break; diff --git a/manifest b/manifest index 3d02d28e9b..e0a85e57cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sprepared\sstatements\sfor\ssome\spragmas\scan\snow\sbe\sreused\swithout\sinvoking\nan\sautomatic\sreprepare. -D 2016-03-16T21:29:54.761 +C Add\san\sAPI\sto\sindicate\sthe\spercentage\sprogress\sof\san\srbu\supdate. +D 2016-03-17T21:06:42.412 F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66 @@ -242,10 +242,11 @@ F ext/rbu/rbudiff.test 6cc806dc36389292f2a8f5842d0103721df4a07d F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda +F ext/rbu/rbuprogress.test d63b70f838a20422dce6445bcaed10890e506d02 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/sqlite3rbu.c 5956f8bee63b5ab2b04e65c1801ea0f5920dac92 -F ext/rbu/sqlite3rbu.h 0bdeb3be211aaba7d85445fa36f4701a25a3dbde -F ext/rbu/test_rbu.c 4a4cdcef4ef9379fc2a21f008805c80b27bcf573 +F ext/rbu/sqlite3rbu.c 9bcf35b2f1d8eaf1c82b47bead114b2289ea06c6 +F ext/rbu/sqlite3rbu.h f8ee94f95fc80a35b7cb7ef3f2f5ea740b662477 +F ext/rbu/test_rbu.c 5b6d31af188193d929234d1cd08ee967df092f66 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1456,8 +1457,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 10a3e2a01db9f80452a2a3369fd25b6fd9798274 db1ce7e13e656fcd2766f1b1f225cbfefe8f73ad -R 2716c51dfa2bfe8ed5e18127e51ae3c8 -T +closed db1ce7e13e656fcd2766f1b1f225cbfefe8f73ad -U drh -Z e3b23194e47816995dbce6c752704ef0 +P 97b0e88cc7c3d677217d0bfab4cb4a34a4abb238 +R 9de86de419de6ede9128347e06885dfc +T *branch * rbu-percent-progress +T *sym-rbu-percent-progress * +T -sym-trunk * +U dan +Z f4146c58d33c341586c7f6e824f90dbe diff --git a/manifest.uuid b/manifest.uuid index 06db10b8a2..ca52cccd3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -97b0e88cc7c3d677217d0bfab4cb4a34a4abb238 \ No newline at end of file +ffc58d2c2576a5b6e1c2c7112612c5760e711afd \ No newline at end of file