From: dan Date: Tue, 5 Sep 2017 16:24:38 +0000 (+0000) Subject: Add experimental API sqlite3rbu_temp_size_limit(). For limiting the amount of X-Git-Tag: version-3.21.0~105 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f21124f2517e5f3b97751773246e121378621f24;p=thirdparty%2Fsqlite.git Add experimental API sqlite3rbu_temp_size_limit(). For limiting the amount of temporary disk space RBU uses. FossilOrigin-Name: 7fdd629830679db620d477df3c206bf84598cc935ccb51547c0d8444a186b63e --- diff --git a/ext/rbu/rbutemplimit.test b/ext/rbu/rbutemplimit.test new file mode 100644 index 0000000000..274f870b73 --- /dev/null +++ b/ext/rbu/rbutemplimit.test @@ -0,0 +1,129 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbutemplimit + +db close +sqlite3_shutdown +sqlite3_config_uri 1 + +proc setup_databases {} { + forcedelete test.db2 + forcedelete test.db + sqlite3 db test.db + execsql { + -- Create target database schema. + -- + CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB(100), c BLOB(100)); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b BLOB(100), c BLOB(100)); + CREATE INDEX i1b ON t1(b); + CREATE INDEX i1c ON t1(c); + CREATE INDEX i2b ON t2(b); + CREATE INDEX i2c ON t2(c); + + -- Create a large RBU database. + -- + ATTACH 'test.db2' AS rbu; + CREATE TABLE rbu.data_t1(a, b, c, rbu_control); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10000 + ) + INSERT INTO data_t1 SELECT i, randomblob(100), randomblob(100), 0 FROM s; + CREATE TABLE rbu.data_t2(a, b, c, rbu_control); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<15000 + ) + INSERT INTO data_t2 SELECT i, randomblob(100), randomblob(100), 0 FROM s; + } + db close +} + +proc run_rbu_cachesize {target rbu cachesize temp_limit} { + sqlite3rbu rbu $target $rbu + rbu temp_size_limit $temp_limit + sqlite3_exec_nr [rbu db 1] "PRAGMA cache_size = $cachesize" + while 1 { + set rc [rbu step] + set ::A([rbu temp_size]) 1 + if {$rc!="SQLITE_OK"} break + } + list [catch {rbu close} msg] $msg +} + +proc step_rbu_cachesize {target rbu stepsize cachesize temp_limit} { + set res "" + while 1 { + sqlite3rbu rbu $target $rbu + rbu temp_size_limit $temp_limit + sqlite3_exec_nr [rbu db 1] "PRAGMA cache_size = $cachesize" + for {set i 0} {$i < $stepsize} {incr i} { + set rc [rbu step] + set ::A([rbu temp_size]) 1 + if {$rc!="SQLITE_OK"} break + } + set res [list [catch {rbu close} msg] $msg] + if {$res != "0 SQLITE_OK"} break + } + set res +} + +do_test 1.1.0 { setup_databases } {} + +do_test 1.1.1 { + unset -nocomplain ::A + run_rbu_cachesize test.db test.db2 10 0 +} {0 SQLITE_DONE} + +do_test 1.1.2 { llength [array names ::A] } 3 + +do_test 1.1.3 { + foreach {a0 a1 a2} [lsort -integer [array names ::A]] {} + list [expr $a0==0] \ + [expr $a1>1048576] [expr $a1<1200000] \ + [expr $a2>1500000] [expr $a2<1700000] +} {1 1 1 1 1} + +do_test 1.2.1 { + setup_databases + run_rbu_cachesize test.db test.db2 10 1000000 +} {1 SQLITE_FULL} +do_test 1.2.2 { info commands rbu } {} + +do_test 1.3.1 { + setup_databases + run_rbu_cachesize test.db test.db2 10 1300000 +} {1 SQLITE_FULL} +do_test 1.3.2 { info commands rbu } {} + +do_test 1.4.1 { + setup_databases + run_rbu_cachesize test.db test.db2 10 1800000 +} {0 SQLITE_DONE} +do_test 1.4.2 { info commands rbu } {} + +do_test 1.5.1 { + setup_databases + unset -nocomplain ::A + step_rbu_cachesize test.db test.db2 1000 10 2400000 +} {0 SQLITE_DONE} +do_test 1.5.2 { info commands rbu } {} + +do_test 1.6.1 { + setup_databases + unset -nocomplain ::A + step_rbu_cachesize test.db test.db2 1000 10 1400000 +} {1 SQLITE_FULL} +do_test 1.6.2 { info commands rbu } {} + +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 033127853b..fbcfab9104 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -371,6 +371,8 @@ struct sqlite3rbu { int pgsz; u8 *aBuf; i64 iWalCksum; + i64 szTemp; /* Current size of all temp files in use */ + i64 szTempLimit; /* Total size limit for temp files */ /* Used in RBU vacuum mode only */ int nRbu; /* Number of RBU VFS in the stack */ @@ -379,23 +381,33 @@ struct sqlite3rbu { /* ** An rbu VFS is implemented using an instance of this structure. +** +** Variable pRbu is only non-NULL for automatically created RBU VFS objects. +** It is NULL for RBU VFS objects created explicitly using +** sqlite3rbu_create_vfs(). It is used to track the total amount of temp +** space used by the RBU handle. */ struct rbu_vfs { sqlite3_vfs base; /* rbu VFS shim methods */ sqlite3_vfs *pRealVfs; /* Underlying VFS */ sqlite3_mutex *mutex; /* Mutex to protect pMain */ + sqlite3rbu *pRbu; /* Owner RBU object */ rbu_file *pMain; /* Linked list of main db files */ }; /* ** Each file opened by an rbu VFS is represented by an instance of ** the following structure. +** +** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable +** "sz" is set to the current size of the database file. */ struct rbu_file { sqlite3_file base; /* sqlite3_file methods */ sqlite3_file *pReal; /* Underlying file handle */ rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ + i64 sz; /* Size of file in bytes (temp only) */ int openFlags; /* Flags this file was opened with */ u32 iCookie; /* Cookie value for main db files */ @@ -3409,6 +3421,7 @@ static void rbuCreateVfs(sqlite3rbu *p){ sqlite3_vfs *pVfs = sqlite3_vfs_find(zRnd); assert( pVfs ); p->zVfsName = pVfs->zName; + ((rbu_vfs*)pVfs)->pRbu = p; } } @@ -3781,6 +3794,7 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ /* Close the open database handle and VFS object. */ sqlite3_close(p->dbRbu); sqlite3_close(p->dbMain); + assert( p->szTemp==0 ); rbuDeleteVfs(p); sqlite3_free(p->aBuf); sqlite3_free(p->aFrame); @@ -3968,6 +3982,7 @@ int sqlite3rbu_savestate(sqlite3rbu *p){ */ static void rbuUnlockShm(rbu_file *p){ + assert( p->openFlags & SQLITE_OPEN_MAIN_DB ); if( p->pRbu ){ int (*xShmLock)(sqlite3_file*,int,int,int) = p->pReal->pMethods->xShmLock; int i; @@ -3980,6 +3995,18 @@ static void rbuUnlockShm(rbu_file *p){ } } +/* +*/ +static int rbuUpdateTempSize(rbu_file *pFd, sqlite3_int64 nNew){ + sqlite3rbu *pRbu = pFd->pRbu; + i64 nDiff = nNew - pFd->sz; + pRbu->szTemp += nDiff; + pFd->sz = nNew; + assert( pRbu->szTemp>=0 ); + if( pRbu->szTempLimit && pRbu->szTemp>pRbu->szTempLimit ) return SQLITE_FULL; + return SQLITE_OK; +} + /* ** Close an rbu file. */ @@ -4005,6 +4032,9 @@ static int rbuVfsClose(sqlite3_file *pFile){ rbuUnlockShm(p); p->pReal->pMethods->xShmUnmap(p->pReal, 0); } + else if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){ + rbuUpdateTempSize(p, 0); + } /* Close the underlying file handle */ rc = p->pReal->pMethods->xClose(p->pReal); @@ -4122,11 +4152,19 @@ static int rbuVfsWrite( assert( p->openFlags & SQLITE_OPEN_MAIN_DB ); rc = rbuCaptureDbWrite(p->pRbu, iOfst); }else{ - if( pRbu && pRbu->eStage==RBU_STAGE_OAL - && (p->openFlags & SQLITE_OPEN_WAL) - && iOfst>=pRbu->iOalSz - ){ - pRbu->iOalSz = iAmt + iOfst; + if( pRbu ){ + if( pRbu->eStage==RBU_STAGE_OAL + && (p->openFlags & SQLITE_OPEN_WAL) + && iOfst>=pRbu->iOalSz + ){ + pRbu->iOalSz = iAmt + iOfst; + }else if( p->openFlags & SQLITE_OPEN_DELETEONCLOSE ){ + i64 szNew = iAmt+iOfst; + if( szNew>p->sz ){ + rc = rbuUpdateTempSize(p, szNew); + if( rc!=SQLITE_OK ) return rc; + } + } } rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst); if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ @@ -4145,6 +4183,10 @@ static int rbuVfsWrite( */ static int rbuVfsTruncate(sqlite3_file *pFile, sqlite_int64 size){ rbu_file *p = (rbu_file*)pFile; + if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){ + int rc = rbuUpdateTempSize(p, size); + if( rc!=SQLITE_OK ) return rc; + } return p->pReal->pMethods->xTruncate(p->pReal, size); } @@ -4534,6 +4576,8 @@ static int rbuVfsOpen( pDb->pWalFd = pFd; } } + }else{ + pFd->pRbu = pRbuVfs->pRbu; } if( oflags & SQLITE_OPEN_MAIN_DB @@ -4801,6 +4845,20 @@ int sqlite3rbu_create_vfs(const char *zName, const char *zParent){ return rc; } +/* +** Configure the aggregate temp file size limit for this RBU handle. +*/ +sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu *pRbu, sqlite3_int64 n){ + if( n>=0 ){ + pRbu->szTempLimit = n; + } + return pRbu->szTempLimit; +} + +sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu *pRbu){ + return pRbu->szTemp; +} + /**************************************************************************/ diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index 2f038fd8fd..1acbcca469 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -352,6 +352,28 @@ SQLITE_API sqlite3rbu *sqlite3rbu_vacuum( const char *zState ); +/* +** Configure a limit for the amount of temp space that may be used by +** the RBU handle passed as the first argument. The new limit is specified +** in bytes by the second parameter. If it is positive, the limit is updated. +** If the second parameter to this function is passed zero, then the limit +** is removed entirely. If the second parameter is negative, the limit is +** not modified (this is useful for querying the current limit). +** +** In all cases the returned value is the current limit in bytes (zero +** indicates unlimited). +** +** If the temp space limit is exceeded during operation, an SQLITE_FULL +** error is returned. +*/ +SQLITE_API sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu*, sqlite3_int64); + +/* +** Return the current amount of temp file space, in bytes, currently used by +** the RBU handle passed as the only argument. +*/ +SQLITE_API sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu*); + /* ** Internally, each RBU connection uses a separate SQLite database ** connection to access the target and rbu update databases. This diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index fba90dcdc4..631bff2a76 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -69,16 +69,18 @@ static int SQLITE_TCLAPI test_sqlite3rbu_cmd( int nArg; const char *zUsage; } aCmd[] = { - {"step", 2, ""}, /* 0 */ - {"close", 2, ""}, /* 1 */ - {"create_rbu_delta", 2, ""}, /* 2 */ - {"savestate", 2, ""}, /* 3 */ - {"dbMain_eval", 3, "SQL"}, /* 4 */ - {"bp_progress", 2, ""}, /* 5 */ - {"db", 3, "RBU"}, /* 6 */ - {"state", 2, ""}, /* 7 */ - {"progress", 2, ""}, /* 8 */ - {"close_no_error", 2, ""}, /* 9 */ + {"step", 2, ""}, /* 0 */ + {"close", 2, ""}, /* 1 */ + {"create_rbu_delta", 2, ""}, /* 2 */ + {"savestate", 2, ""}, /* 3 */ + {"dbMain_eval", 3, "SQL"}, /* 4 */ + {"bp_progress", 2, ""}, /* 5 */ + {"db", 3, "RBU"}, /* 6 */ + {"state", 2, ""}, /* 7 */ + {"progress", 2, ""}, /* 8 */ + {"close_no_error", 2, ""}, /* 9 */ + {"temp_size_limit", 3, "LIMIT"}, /* 10 */ + {"temp_size", 2, ""}, /* 11 */ {0,0,0} }; int iCmd; @@ -193,6 +195,22 @@ static int SQLITE_TCLAPI test_sqlite3rbu_cmd( Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nStep)); break; } + + case 10: /* temp_size_limit */ { + sqlite3_int64 nLimit; + if( Tcl_GetWideIntFromObj(interp, objv[2], &nLimit) ){ + ret = TCL_ERROR; + }else{ + nLimit = sqlite3rbu_temp_size_limit(pRbu, nLimit); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nLimit)); + } + break; + } + case 11: /* temp_size */ { + sqlite3_int64 sz = sqlite3rbu_temp_size(pRbu); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(sz)); + break; + } default: /* seems unlikely */ assert( !"cannot happen" ); diff --git a/manifest b/manifest index 606b100619..592f8659d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sthe\sSQLITE_CORRUPT_BKPT\sreturn\scode\sin\sa\scouple\smore\splaces. -D 2017-09-04T19:31:54.200 +C Add\sexperimental\sAPI\ssqlite3rbu_temp_size_limit().\sFor\slimiting\sthe\samount\sof\ntemporary\sdisk\sspace\sRBU\suses. +D 2017-09-05T16:24:38.115 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a7a74bf60ad395098c0bd175ab054cd65ef85d7f034198d52bcc4d9e5fb4c6b @@ -318,11 +318,12 @@ F ext/rbu/rbufts.test a2bbd202c9321fba15fb4a62a90add7d70e07bd8404e1e598135adbfff F ext/rbu/rbuprogress.test 1849d4e0e50616edf5ce75ce7db86622e656b5cf F ext/rbu/rburesume.test 8acb77f4a422ff55acfcfc9cc15a5cb210b1de83 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 +F ext/rbu/rbutemplimit.test cd553a9288d515d0b5f87d277e76fd18c4aa740b761e7880fab11ce986ea18d1 F ext/rbu/rbuvacuum.test ff357e9b556ca7ad4673da0ff7f244def919ff858e0f9f350d3e30fdd83a62a8 F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa -F ext/rbu/sqlite3rbu.c 920941a6ff7dbbea0970717c43662878fda5c37e43752de329f0fdd76680ab75 -F ext/rbu/sqlite3rbu.h 82c102e5ae41025e3b245d3d5944315f82811da85e2cd363a75caa97cbd0cd3e -F ext/rbu/test_rbu.c ec18cfc69a104309df23c359e3c80306c9a6bdd1d2c53c8b70ae158e9832dcd6 +F ext/rbu/sqlite3rbu.c a1a303de8b90f987ef63bf9cef57f5d7dd7983a9e8aed3775a759d87ad57075d +F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d007e79b2 +F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cf84d52958a7ec6a506f1711e119db847ed6bb5dedde78a58e97503287afcda1 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1651,7 +1652,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 77854694b2da989aa4dbbdbd3ccf61756c46cc368de9731f5fd1c3aa38d7cad5 -R 4f0c4978d96d81c3edc4766e1765023c -U mistachkin -Z 249910ae9570afe838571fb260f0586c +P 72d22c226bf4311345e8844fd9801ebddf77aceb80a038dce46608bf4ccae636 +R 6f77c70118414e1c190062cb8a829104 +U dan +Z c201de8958e66e7927017757d360b434 diff --git a/manifest.uuid b/manifest.uuid index b52e0e026f..5ee97ca57f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72d22c226bf4311345e8844fd9801ebddf77aceb80a038dce46608bf4ccae636 \ No newline at end of file +7fdd629830679db620d477df3c206bf84598cc935ccb51547c0d8444a186b63e \ No newline at end of file