From e336b8914c44d341801746e3fd66afadc603b4dc Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 1 Jul 2016 12:39:58 +0000 Subject: [PATCH] Add the sqlite3rbu_state() API. Used to determine the current state (creating OAL, ready to move OAL, incremental-checkpoint, finished or error) of an RBU operation. FossilOrigin-Name: 92e7df0ff5c4c118c63d92a767dc82700438a310 --- ext/rbu/rbu1.test | 44 +++++++++------------ ext/rbu/rbu5.test | 27 +------------ ext/rbu/rbu_common.tcl | 37 ++++++++++++++++++ ext/rbu/sqlite3rbu.c | 87 ++++++++++++++++++++++++++++-------------- ext/rbu/sqlite3rbu.h | 38 ++++++++++++++++++ ext/rbu/test_rbu.c | 14 +++++++ manifest | 27 +++++++------ manifest.uuid | 2 +- 8 files changed, 182 insertions(+), 94 deletions(-) diff --git a/ext/rbu/rbu1.test b/ext/rbu/rbu1.test index 51d2ce8171..cd41728886 100644 --- a/ext/rbu/rbu1.test +++ b/ext/rbu/rbu1.test @@ -10,10 +10,7 @@ #*********************************************************************** # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} -source $testdir/tester.tcl +source [file join [file dirname [info script]] rbu_common.tcl] set ::testprefix rbu1 db close @@ -96,26 +93,6 @@ proc create_rbu5 {filename} { return $filename } -# Run the RBU in file $rbu on target database $target until completion. -# -proc run_rbu {target rbu} { - sqlite3rbu rbu $target $rbu - while 1 { - set rc [rbu step] - if {$rc!="SQLITE_OK"} break - } - rbu close -} - -proc step_rbu {target rbu} { - while 1 { - sqlite3rbu rbu $target $rbu - set rc [rbu step] - rbu close - if {$rc != "SQLITE_OK"} break - } - set rc -} # Same as [step_rbu], except using a URI to open the target db. # @@ -641,10 +618,25 @@ foreach {tn3 create_vfs destroy_vfs} { # correctly. reset_db forcedelete rbu.db - do_test $tn3.8 { + do_test $tn3.8.1 { list [catch { run_rbu test.db rbu.db } msg] $msg } {0 SQLITE_DONE} - + + # Test that an RBU database containing only empty data_xxx tables is + # also handled correctly. + reset_db + forcedelete rbu.db + do_execsql_test $tn3.8.2.1 { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 2); + ATTACH 'rbu.db' AS rbu; + CREATE TABLE data_t1(a, b, rbu_control); + DETACH rbu; + } + do_test $tn3.8.2.1 { + list [catch { run_rbu test.db rbu.db } msg] $msg + } {0 SQLITE_DONE} + # Test that RBU can update indexes containing NULL values. # reset_db diff --git a/ext/rbu/rbu5.test b/ext/rbu/rbu5.test index 9a0f17bc5d..8e4ae8cb69 100644 --- a/ext/rbu/rbu5.test +++ b/ext/rbu/rbu5.test @@ -12,35 +12,10 @@ # Test some properties of the pager_rbu_mode and rbu_mode pragmas. # -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} -source $testdir/tester.tcl +source [file join [file dirname [info script]] rbu_common.tcl] set ::testprefix rbu5 -# Run the RBU in file $rbu on target database $target until completion. -# -proc run_rbu {target rbu} { - sqlite3rbu rbu $target $rbu - while { [rbu step]=="SQLITE_OK" } {} - rbu close -} - - -# Run the RBU in file $rbu on target database $target one step at a -# time until completion. -# -proc step_rbu {target rbu} { - while 1 { - sqlite3rbu rbu $target $rbu - set rc [rbu step] - rbu close - if {$rc != "SQLITE_OK"} break - } - set rc -} - # Return a list of the primary key columns for table $tbl in the database # opened by database handle $db. # diff --git a/ext/rbu/rbu_common.tcl b/ext/rbu/rbu_common.tcl index 4a8c5cba9f..a58c3aa0a7 100644 --- a/ext/rbu/rbu_common.tcl +++ b/ext/rbu/rbu_common.tcl @@ -15,12 +15,43 @@ if {![info exists testdir]} { } source $testdir/tester.tcl +proc check_prestep_state {target state} { + set oal_exists [file exists $target-oal] + set wal_exists [file exists $target-wal] + set progress [rbu progress] + + if {($progress==0 && $state!="oal" && $state!="done") + || ($oal_exists && $wal_exists) + || ($progress>0 && $state=="oal" && (!$oal_exists || $wal_exists)) + || ($state=="move" && (!$oal_exists || $wal_exists)) + || ($state=="checkpoint" && ($oal_exists || !$wal_exists)) + || ($state=="done" && ($oal_exists && $progress!=0)) + } { + error "B: state=$state progress=$progress oal=$oal_exists wal=$wal_exists" + } +} + +proc check_poststep_state {rc target state} { + if {$rc=="SQLITE_OK" || $rc=="SQLITE_DONE"} { + set oal_exists [file exists $target-oal] + set wal_exists [file exists $target-wal] + if {$state=="move" && ($oal_exists || !$wal_exists)} { + error "A: state=$state progress=$progress oal=$oal_exists wal=$wal_exists" + } + } +} + # Run the RBU in file $rbu on target database $target until completion. # proc run_rbu {target rbu} { sqlite3rbu rbu $target $rbu while 1 { + set state [rbu state] + + check_prestep_state $target $state set rc [rbu step] + check_poststep_state $rc $target $state + if {$rc!="SQLITE_OK"} break } rbu close @@ -29,7 +60,10 @@ proc run_rbu {target rbu} { proc step_rbu {target rbu} { while 1 { sqlite3rbu rbu $target $rbu + set state [rbu state] + check_prestep_state $target $state set rc [rbu step] + check_poststep_state $rc $target $state rbu close if {$rc != "SQLITE_OK"} break } @@ -41,7 +75,10 @@ proc do_rbu_vacuum_test {tn step} { if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db } while 1 { if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db } + set state [rbu state] + check_prestep_state test.db $state set rc [rbu step] + check_poststep_state $rc test.db $state if {$rc!="SQLITE_OK"} break if {$step==1} { rbu close } } diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 1cfb71384f..73c6647a2c 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -3555,30 +3555,7 @@ static sqlite3rbu *openRbuHandle( if( p->rc==SQLITE_OK ){ if( p->eStage==RBU_STAGE_OAL ){ sqlite3 *db = p->dbMain; - - if( pState->eStage==0 && rbuIsVacuum(p) ){ - rbuCopyPragma(p, "page_size"); - rbuCopyPragma(p, "auto_vacuum"); - } - - /* Open transactions both databases. The *-oal file is opened or - ** created at this point. */ - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); - } - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg); - } - - /* Check if the main database is a zipvfs db. If it is, set the upper - ** level pager to use "journal_mode=off". This prevents it from - ** generating a large journal using a temp file. */ - if( p->rc==SQLITE_OK ){ - int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0); - if( frc==SQLITE_OK ){ - p->rc = sqlite3_exec(db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg); - } - } + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg); /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ @@ -3589,12 +3566,34 @@ static sqlite3rbu *openRbuHandle( ** update finished. */ if( p->rc==SQLITE_OK && p->objiter.zTbl==0 ){ p->rc = SQLITE_DONE; - } + p->eStage = RBU_STAGE_DONE; + }else{ + if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){ + rbuCopyPragma(p, "page_size"); + rbuCopyPragma(p, "auto_vacuum"); + } - if( p->rc==SQLITE_OK ){ - rbuSetupOal(p, pState); - } + /* Open transactions both databases. The *-oal file is opened or + ** created at this point. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + } + /* Check if the main database is a zipvfs db. If it is, set the upper + ** level pager to use "journal_mode=off". This prevents it from + ** generating a large journal using a temp file. */ + if( p->rc==SQLITE_OK ){ + int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0); + if( frc==SQLITE_OK ){ + p->rc = sqlite3_exec( + db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg); + } + } + + if( p->rc==SQLITE_OK ){ + rbuSetupOal(p, pState); + } + } }else if( p->eStage==RBU_STAGE_MOVE ){ /* no-op */ }else if( p->eStage==RBU_STAGE_CKPT ){ @@ -3761,9 +3760,39 @@ void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){ } } +/* +** Return the current state of the RBU vacuum or update operation. +*/ +int sqlite3rbu_state(sqlite3rbu *p){ + int aRes[] = { + 0, SQLITE_RBU_STATE_OAL, SQLITE_RBU_STATE_MOVE, + 0, SQLITE_RBU_STATE_CHECKPOINT, SQLITE_RBU_STATE_DONE + }; + + assert( RBU_STAGE_OAL==1 ); + assert( RBU_STAGE_MOVE==2 ); + assert( RBU_STAGE_CKPT==4 ); + assert( RBU_STAGE_DONE==5 ); + assert( aRes[RBU_STAGE_OAL]==SQLITE_RBU_STATE_OAL ); + assert( aRes[RBU_STAGE_MOVE]==SQLITE_RBU_STATE_MOVE ); + assert( aRes[RBU_STAGE_CKPT]==SQLITE_RBU_STATE_CHECKPOINT ); + assert( aRes[RBU_STAGE_DONE]==SQLITE_RBU_STATE_DONE ); + + if( p->rc!=SQLITE_OK && p->rc!=SQLITE_DONE ){ + return SQLITE_RBU_STATE_ERROR; + }else{ + assert( p->rc!=SQLITE_DONE || p->eStage==RBU_STAGE_DONE ); + assert( p->eStage==RBU_STAGE_OAL + || p->eStage==RBU_STAGE_MOVE + || p->eStage==RBU_STAGE_CKPT + || p->eStage==RBU_STAGE_DONE + ); + return aRes[p->eStage]; + } +} + int sqlite3rbu_savestate(sqlite3rbu *p){ int rc = p->rc; - if( rc==SQLITE_DONE ) return SQLITE_OK; assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE ); diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index fce9e493db..9ce39f543c 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -474,6 +474,44 @@ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); */ void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); +/* +** Obtain an indication as to the current stage of an RBU update or vacuum. +** This function always returns one of the SQLITE_RBU_STATE_XXX constants +** defined in this file. Return values should be interpreted as follows: +** +** SQLITE_RBU_STATE_OAL: +** RBU is currently building a *-oal file. The next call to sqlite3rbu_step() +** may either add further data to the *-oal file, or compute data that will +** be added by a subsequent call. +** +** SQLITE_RBU_STATE_MOVE: +** RBU has finished building the *-oal file. The next call to sqlite3rbu_step() +** will move the *-oal file to the equivalent *-wal path. If the current +** operation is an RBU update, then the updated version of the database +** file will become visible to ordinary SQLite clients following the next +** call to sqlite3rbu_step(). +** +** SQLITE_RBU_STATE_CHECKPOINT: +** RBU is currently performing an incremental checkpoint. The next call to +** sqlite3rbu_step() will copy a page of data from the *-wal file into +** the target database file. +** +** SQLITE_RBU_STATE_DONE: +** The RBU operation has finished. Any subsequent calls to sqlite3rbu_step() +** will immediately return SQLITE_DONE. +** +** SQLITE_RBU_STATE_ERROR: +** An error has occurred. Any subsequent calls to sqlite3rbu_step() will +** immediately return the SQLite error code associated with the error. +*/ +#define SQLITE_RBU_STATE_OAL 1 +#define SQLITE_RBU_STATE_MOVE 2 +#define SQLITE_RBU_STATE_CHECKPOINT 3 +#define SQLITE_RBU_STATE_DONE 4 +#define SQLITE_RBU_STATE_ERROR 5 + +int sqlite3rbu_state(sqlite3rbu *pRbu); + /* ** Create an RBU VFS named zName that accesses the underlying file-system ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index 2d52262c71..6418d30d59 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -69,6 +69,8 @@ static int test_sqlite3rbu_cmd( {"dbMain_eval", 3, "SQL"}, /* 4 */ {"bp_progress", 2, ""}, /* 5 */ {"db", 3, "RBU"}, /* 6 */ + {"state", 2, ""}, /* 7 */ + {"progress", 2, ""}, /* 8 */ {0,0,0} }; int iCmd; @@ -166,6 +168,18 @@ static int test_sqlite3rbu_cmd( } break; } + case 7: /* state */ { + const char *aRes[] = { 0, "oal", "move", "checkpoint", "done", "error" }; + int eState = sqlite3rbu_state(pRbu); + assert( eState>0 && eState<=5 ); + Tcl_SetResult(interp, (char*)aRes[eState], TCL_STATIC); + break; + } + case 8: /* progress */ { + sqlite3_int64 nStep = sqlite3rbu_progress(pRbu); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nStep)); + break; + } default: /* seems unlikely */ assert( !"cannot happen" ); diff --git a/manifest b/manifest index 2dd253cf30..20d61b1e4e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Prevent\sthe\sWhereLoop.rSetup\scost\sestimate\sfrom\sgoing\snegative\son\scomplex\nqueries. -D 2016-06-26T04:06:28.081 +C Add\sthe\ssqlite3rbu_state()\sAPI.\sUsed\sto\sdetermine\sthe\scurrent\sstate\s(creating\sOAL,\sready\sto\smove\sOAL,\sincremental-checkpoint,\sfinished\sor\serror)\sof\san\sRBU\soperation. +D 2016-07-01T12:39:58.096 F Makefile.in bc2b4864a23a4a21c3e26d7b4350f51bab324d45 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 50149765ef72f4e652b9a0f1f6462c4784bb9423 @@ -226,14 +226,14 @@ F ext/misc/vfsstat.c 6110aeeaab2f1df17a923c8a8acef3c74f6dc515 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rbu/rbu.c b2c0b5e6ae1a89affc0edfc127ebfa5f637a0ce4 -F ext/rbu/rbu1.test 42bd835e019eff789ec241017965277baeb658b1 +F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 046b0980041d30700464a800bbf6733ed2df515d F ext/rbu/rbu11.test 9bc68c2d3dbeb1720153626e3bd0466dcc017702 F ext/rbu/rbu12.test bde22ed0004dd5d1888c72a84ae407e574aeae16 F ext/rbu/rbu13.test 462ff799c4afedc3ef8a47ff818c0ffbf14ae4f2 F ext/rbu/rbu14.test 01f5dcba904aecadbaea69d4ccdc2ea43dd30560 F ext/rbu/rbu3.test 8bd4c6b87367c358981b6a47dc3d654fa60bff90 -F ext/rbu/rbu5.test 2e24fee3e615aecd99bbdd46967935a641e866f7 +F ext/rbu/rbu5.test cf1fa84d6107efd73e292a7be8e1fb31953ca45c F ext/rbu/rbu6.test 32e8ed60631b6facdb6366bd2b5f5f25245e7edb F ext/rbu/rbu7.test fd025d5ba440fcfe151fbb0e3835e1e7fe964fa1 F ext/rbu/rbu8.test 3bbf2c35d71a843c463efe93946f14ad10c3ede0 @@ -241,7 +241,7 @@ F ext/rbu/rbu9.test 0806d1772c9f4981774ff028de6656e4183082af F ext/rbu/rbuA.test c1a7b3e2d926b8f8448bb3b4ae787e314ee4b2b3 F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831 -F ext/rbu/rbu_common.tcl 3a4b916b6f5dca9c9da9a30863e272fe5ea4414f +F ext/rbu/rbu_common.tcl a38e8e2d4a50fd6aaf151633714c1b1d2fae3ead F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 F ext/rbu/rbudiff.test 4c9f8df6f723f553781d3d117501b7e9d170a145 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 @@ -252,9 +252,9 @@ F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0 F ext/rbu/rbuvacuum2.test 45009e127c3fb385e5c0fd5a8a63fb922a79d0ab -F ext/rbu/sqlite3rbu.c 3d35b1dc46f5ca0f0f59db8e6163fe7627ea107c -F ext/rbu/sqlite3rbu.h 2acd0a6344a6079de15c8dc9d84d3df83a665930 -F ext/rbu/test_rbu.c 9bbdf6bd8efd58fbc4f192698df50569598fbb9e +F ext/rbu/sqlite3rbu.c 948677ee0ec57da51148e6c5f64ac68afcf36ab2 +F ext/rbu/sqlite3rbu.h db8858120c9be14b60c9225f9da28221f5f6b945 +F ext/rbu/test_rbu.c 9f043b74c46c45b231f4313aed1fccb379a76fe6 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c d26a815b0df1c412a6881dae8d7fd3c9c08cce68 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1502,7 +1502,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 507014e4c7a70cd09410c89c8ed466c8edab39d2 -R ae55d782914cbf2db58c75f82f862316 -U drh -Z e21bca843071316b56d3c29d5bf9cc08 +P f81050859170c8708a1b296da8dd3ef0dd314a11 +R 2c74d9aac55f1cbb7c7d3232cf9071e1 +T *branch * rbu-state-api +T *sym-rbu-state-api * +T -sym-trunk * +U dan +Z b7514d1a78fc46b36ed5446f76a12e50 diff --git a/manifest.uuid b/manifest.uuid index ee49eb40ac..85f8706971 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f81050859170c8708a1b296da8dd3ef0dd314a11 \ No newline at end of file +92e7df0ff5c4c118c63d92a767dc82700438a310 \ No newline at end of file -- 2.39.5