From: danielk1977 Date: Mon, 19 Dec 2005 14:18:11 +0000 (+0000) Subject: Add some very simple test cases (and resulting bug fixes) for release_memory(). ... X-Git-Tag: version-3.6.10~3334 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0190d1da466f4ba92e2e84ad038476861c9600b4;p=thirdparty%2Fsqlite.git Add some very simple test cases (and resulting bug fixes) for release_memory(). (CVS 2826) FossilOrigin-Name: 154282fca54bf03d310d6931660f99805bb5477f --- diff --git a/manifest b/manifest index 8f54e5cae3..28739124d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s(untested)\ssqlite3_release_memory()\sfunction.\s(CVS\s2825) -D 2005-12-18T08:51:23 +C Add\ssome\svery\ssimple\stest\scases\s(and\sresulting\sbug\sfixes)\sfor\srelease_memory().\s(CVS\s2826) +D 2005-12-19T14:18:11 F Makefile.in e3c6b3a38d734d41574c04f2fc90d18de2b87102 F Makefile.linux-gcc aee18d8a05546dcf1888bd4547e442008a49a092 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -59,8 +59,8 @@ F src/os_unix.c 6394d2fa3a8bfbceb227579b44b4b343b5b54a8f F src/os_unix.h 5768d56d28240d3fe4537fac08cc85e4fb52279e F src/os_win.c 9feb97f49b93d451f8ef7c5dd388e05a44647dc6 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b -F src/pager.c d501a79293f575065e3ab9169e2dea7b945ce155 -F src/pager.h 5c14873902723bf735582d55d7042abfbb3810bc +F src/pager.c 154825d3b0cd290bf1eeb2cfa6715fd08941fdec +F src/pager.h e0acb095b3ad0bca48f2ab00c87346665643f64f F src/parse.y 142a4b347c82217332e2d3dfa317ff2b7ac32f9c F src/pragma.c 8883b4d34796efa315bdd0ec1b03f580ef1575b9 F src/prepare.c 1417a396efe55e2767f9f97f694d21b8cac2f4d6 @@ -68,10 +68,10 @@ F src/printf.c f47a2f4b5387cd2ebb12e9117a1a5d6bd9a2b812 F src/random.c ff5e9a8cad790e2a51cd4d2e7737dc8540e09d1d F src/select.c 2292b065bc6be61e01aad39a2e1b93e332fb7e57 F src/shell.c 4872acee1d2a826c73c914961e469e563204b7f9 -F src/sqlite.h.in addab2c24bc84b4fe74cda7bb102abd34983b14f -F src/sqliteInt.h 5581e01a1ca1a9a4d70babb96021740bc17eea2e +F src/sqlite.h.in 5b16b66db128f5aac5e82721b9012bfa88958bd7 +F src/sqliteInt.h 4e822139dc9322795a9644a3d3b281d53f6d99ed F src/table.c 486dcfce532685b53b5a2b5da8bba0ded6fb2316 -F src/tclsqlite.c b21dd96ffab1625c986513e003a1945a6ae112ae +F src/tclsqlite.c ce481c0a21a20641cdfe87c9cbbb328cfb3a58b9 F src/test1.c d6924b182773b2ad3b22e435e4d3bfd5a846da9e F src/test2.c 36390cdfc70c08e5ee0b466d0654a117f398bbff F src/test3.c 7c97833e33496c2b69f4fe6b9882ac60a481da97 @@ -82,7 +82,7 @@ F src/tokenize.c 7a3a3d3cc734f684a77c4dfd09eb46fcee25394c F src/trigger.c 2925ba96d964d9b717e74006bf7e64b8a6b70d97 F src/update.c ec8e540617b116725b5a55c8d6b4db8bc67fdd7d F src/utf.c b7bffac4260177ae7f83c01d025fe0f5ed70ce71 -F src/util.c 31edbc07f9fb44b23d0c33709ecd50b89ecd96a0 +F src/util.c 0ed80dea0cd12dc09d4c576f6a132fd5cf963b51 F src/vacuum.c fbfdd3967fd34e2f260fafed88dcbf3c10856b94 F src/vdbe.c 09aaed71f076bfd4286607ee4845100b910a492f F src/vdbe.h 8729a4ee16ff9aeab2af9667df3cf300ff978e13 @@ -182,6 +182,7 @@ F test/malloc.test a5ed721cf7d1b12602ede4f98c11b65ab1582cc0 F test/malloc2.test e6e321db96d6c94cb18bf82ad7215070c41e624e F test/malloc3.test 9797d39eca7087a1022bcc0eb0b22215475c9698 F test/malloc4.test 2e29d155eb4b4808019ef47eeedfcbe9e09e0f05 +F test/malloc5.test 2418a9d667a50175744ba4c147ee1c75de6cfe5c F test/manydb.test d81debbf5871242e3b5df1d3bb5e14c50431b6f8 F test/memdb.test 1860e060be810bf0775bc57408a5b7c4954bcaea F test/memleak.test df2b2b96e77f8ba159a332299535b1e5f18e49ac @@ -327,7 +328,7 @@ F www/tclsqlite.tcl ddcf912ea48695603c8ed7efb29f0812ef8d1b49 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl a99cf5f6d8bd4d5537584a2b342f0fb9fa601d8b F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513 -P 83c8ae5bee3b6bdb556d2e85fa260ba855742601 -R 7b8c8cafe73ba9b7fdaabb41edaaf0e4 +P 345addaa03d3bfa3429a59597fbd3addcff62e30 +R 4eebf0e3d04c46eed73662dfdeebac6b U danielk1977 -Z e796870a052bdfafa8d3fb52aa5cdd1a +Z 06c0c23d27172e085b09b2f497c7c9da diff --git a/manifest.uuid b/manifest.uuid index a017fcd3ca..d96b9bdf2a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -345addaa03d3bfa3429a59597fbd3addcff62e30 \ No newline at end of file +154282fca54bf03d310d6931660f99805bb5477f \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 4ea0c46397..377ee017b3 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,7 +18,7 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.225 2005/12/18 08:51:23 danielk1977 Exp $ +** @(#) $Id: pager.c,v 1.226 2005/12/19 14:18:11 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" @@ -2323,7 +2323,7 @@ static int pager_recycle(Pager *pPager, int syncOk, PgHdr **ppPg){ ** very slow operation, so we work hard to avoid it. But sometimes ** it can't be helped. */ - if( pPg==0 && syncOk ){ + if( pPg==0 && pPager->pFirst && syncOk ){ int rc = syncJournal(pPager); if( rc!=0 ){ sqlite3pager_rollback(pPager); @@ -2393,12 +2393,14 @@ static int pager_recycle(Pager *pPager, int syncOk, PgHdr **ppPg){ ** by the current thread may be sqliteFree()ed. ** ** nReq is the number of bytes of memory required. Once this much has -** been released, the function returns. The return value is the total number +** been released, the function returns. A negative value for nReq means +** free as much memory as possible. The return value is the total number ** of bytes of memory released. */ +#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT int sqlite3pager_release_memory(int nReq){ SqliteTsd *pTsd = sqlite3Tsd(); - Pager *pPager; + Pager *p; int nReleased = 0; int i; @@ -2411,7 +2413,7 @@ int sqlite3pager_release_memory(int nReq){ for(i=0; i==0 || i==1; i++){ /* Loop through all the SQLite pagers opened by the current thread. */ - for(pPager=pTsd->pPager; pPager && nReleasedpNext){ + for(p=pTsd->pPager; p && (nReq<0 || nReleasedpNext){ PgHdr *pPg; int rc; @@ -2419,7 +2421,7 @@ int sqlite3pager_release_memory(int nReq){ ** calling fsync() if this is the first iteration of the outermost ** loop). */ - while( SQLITE_OK==(rc = pager_recycle(pPager, i, &pPg)) && pPg) { + while( SQLITE_OK==(rc = pager_recycle(p, i, &pPg)) && pPg) { /* We've found a page to free. At this point the page has been ** removed from the page hash-table, free-list and synced-list ** (pFirstSynced). It is still in the all pages (pAll) list. @@ -2429,10 +2431,11 @@ int sqlite3pager_release_memory(int nReq){ ** probably is though. */ PgHdr *pTmp; - if( pPg==pPager->pAll ){ - pPager->pAll = pPg->pNextAll; + assert( pPg ); + if( pPg==p->pAll ){ + p->pAll = pPg->pNextAll; }else{ - for( pTmp=pPager->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll ); + for( pTmp=p->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll ); pTmp->pNextAll = pPg->pNextAll; } nReleased += sqliteAllocSize(pPg); @@ -2456,6 +2459,7 @@ int sqlite3pager_release_memory(int nReq){ return nReleased; } +#endif /* ** Acquire a page. diff --git a/src/pager.h b/src/pager.h index 64d6dbd5f0..1d7069826f 100644 --- a/src/pager.h +++ b/src/pager.h @@ -13,7 +13,7 @@ ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** -** @(#) $Id: pager.h,v 1.47 2005/12/18 08:51:24 danielk1977 Exp $ +** @(#) $Id: pager.h,v 1.48 2005/12/19 14:18:11 danielk1977 Exp $ */ #ifndef _PAGER_H_ @@ -108,6 +108,7 @@ int sqlite3pager_rename(Pager*, const char *zNewName); void sqlite3pager_set_codec(Pager*,void(*)(void*,void*,Pgno,int),void*); int sqlite3pager_movepage(Pager*,void*,Pgno); int sqlite3pager_reset(Pager*); +int sqlite3pager_release_memory(int); #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) int sqlite3pager_lockstate(Pager*); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 7fb156077e..845ff1e22d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -12,7 +12,7 @@ ** This header file defines the interface that the SQLite library ** presents to client programs. ** -** @(#) $Id: sqlite.h.in,v 1.149 2005/12/18 08:51:24 danielk1977 Exp $ +** @(#) $Id: sqlite.h.in,v 1.150 2005/12/19 14:18:11 danielk1977 Exp $ */ #ifndef _SQLITE3_H_ #define _SQLITE3_H_ @@ -1336,6 +1336,8 @@ void *sqlite3_update_hook( */ void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); +int sqlite3_release_memory(int); + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index aa2a186f59..822b338b3a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -11,7 +11,7 @@ ************************************************************************* ** Internal interface definitions for SQLite. ** -** @(#) $Id: sqliteInt.h,v 1.440 2005/12/18 08:51:24 danielk1977 Exp $ +** @(#) $Id: sqliteInt.h,v 1.441 2005/12/19 14:18:11 danielk1977 Exp $ */ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ @@ -272,7 +272,6 @@ typedef struct SqliteTsd SqliteTsd; struct SqliteTsd { int isInit; /* True if structure has been initialised */ int mallocFailed; /* True after a malloc() has failed */ - #ifndef SQLITE_OMIT_MEMORY_MANAGEMENT unsigned int nSoftHeapLimit; /* (uint)-1 for unlimited */ unsigned int nAlloc; /* Number of bytes currently allocated */ diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 337cdefb47..7d1dc03488 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -11,7 +11,7 @@ ************************************************************************* ** A TCL Interface to SQLite ** -** $Id: tclsqlite.c,v 1.140 2005/12/16 06:54:03 danielk1977 Exp $ +** $Id: tclsqlite.c,v 1.141 2005/12/19 14:18:11 danielk1977 Exp $ */ #ifndef NO_TCL /* Omit this whole file if TCL is unavailable */ @@ -666,9 +666,9 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ "exists", "function", "last_insert_rowid", "nullvalue", "onecolumn", "profile", "progress", "rekey", "rollback_hook", - "soft_heap_limit", "timeout", "total_changes", - "trace", "transaction", "update_hook", - "version", 0 + "release_memory", "soft_heap_limit", "timeout", + "total_changes", "trace", "transaction", + "update_hook", "version", 0 }; enum DB_enum { DB_AUTHORIZER, DB_BUSY, DB_CACHE, @@ -678,9 +678,9 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ DB_EXISTS, DB_FUNCTION, DB_LAST_INSERT_ROWID, DB_NULLVALUE, DB_ONECOLUMN, DB_PROFILE, DB_PROGRESS, DB_REKEY, DB_ROLLBACK_HOOK, - DB_SOFT_HEAP_LIMIT, DB_TIMEOUT, DB_TOTAL_CHANGES, - DB_TRACE, DB_TRANSACTION, DB_UPDATE_HOOK, - DB_VERSION + DB_RELEASE_MEMORY, DB_SOFT_HEAP_LIMIT, DB_TIMEOUT, + DB_TOTAL_CHANGES, DB_TRACE, DB_TRANSACTION, + DB_UPDATE_HOOK, DB_VERSION }; /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */ @@ -1754,9 +1754,10 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ ** $db soft_heap_limit N ** ** Set the soft-heap-limit for this thread. Note that the limit is - ** per-thread, not per-database. + ** per-thread, not per-database. An empty string is returned. */ case DB_SOFT_HEAP_LIMIT: { +#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT int n; if( objc!=3 ){ Tcl_WrongNumArgs(interp, 2, objv, "BYTES"); @@ -1767,6 +1768,34 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ } sqlite3_soft_heap_limit(n); Tcl_ResetResult(interp); +#endif + break; + } + + /* + ** $db release_memory ?N? + ** + ** Try to release memory currently held (but not really required) by + ** SQLite database connections opened by the current thread. If an + ** integer argument is supplied, then SQLite stops trying to free memory + ** after N bytes have been freed. + ** + ** The value returned is the number of bytes actually freed. + **/ + case DB_RELEASE_MEMORY: { + int nRelease = 0; + int N = -1; + if( objc!=2 && objc!=3 ){ + Tcl_WrongNumArgs(interp, 2, objv, "?N?"); + return TCL_ERROR; + } +#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT + if( objc==3 && TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &N) ){ + return TCL_ERROR; + } + nRelease = sqlite3_release_memory(N); +#endif + Tcl_SetObjResult(interp, Tcl_NewIntObj(nRelease)); break; } diff --git a/src/util.c b/src/util.c index 109f12fe5e..360f27a335 100644 --- a/src/util.c +++ b/src/util.c @@ -14,7 +14,7 @@ ** This file contains functions for allocating memory, comparing ** strings, and stuff like that. ** -** $Id: util.c,v 1.155 2005/12/18 08:51:24 danielk1977 Exp $ +** $Id: util.c,v 1.156 2005/12/19 14:18:11 danielk1977 Exp $ */ #include "sqliteInt.h" #include "os.h" @@ -35,7 +35,7 @@ ** sqlite3AllocSize() ** ** The function sqlite3FreeX performs the same task as sqlite3Free and is -** guaranteed to be a real function. +** guaranteed to be a real function. The same holds for sqlite3MallocX ** ** The above APIs are implemented in terms of the functions provided at the Os ** level (not in this file). The Os level interface is never accessed directly @@ -48,7 +48,8 @@ ** ** Functions sqlite3MallocRaw() and sqlite3Realloc() may invoke ** sqlite3_release_memory() if a call to sqlite3Os.xMalloc() or -** sqlite3Os.xRealloc() fails. Function sqlite3Malloc() usually invokes +** sqlite3Os.xRealloc() fails (or if the soft-heap-limit for the thread is +** exceeded). Function sqlite3Malloc() usually invokes ** sqlite3MallocRaw(). ** ** MALLOC TEST WRAPPER ARCHITECTURE @@ -63,11 +64,6 @@ ** * Audit outstanding memory allocations (i.e check for leaks). */ -/* -** TODO! -*/ -#define sqlite3_release_memory(x) 0 - #ifdef SQLITE_MEMDEBUG /*-------------------------------------------------------------------------- ** Begin code for memory allocation system test layer. @@ -156,6 +152,37 @@ const char *sqlite3_malloc_id = 0; TESTALLOC_STACKSIZE /* backtrace() stack */ \ ) + +#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT +/* +** Set the soft heap-size limit for the current thread. +*/ +void sqlite3_soft_heap_limit(int n){ + unsigned int N; + if( n<0 ){ + /* No limit */ + N = 0xFFFFFFFF; + }else{ + N = n; + } + sqlite3Tsd()->nSoftHeapLimit = N; +} + +/* +** Release memory held by SQLite instances created by the current thread. +*/ +int sqlite3_release_memory(int n){ + return sqlite3pager_release_memory(n); +} +#else +/* If SQLITE_OMIT_MEMORY_MANAGEMENT is defined, then define a version +** of sqlite3_release_memory() to be used by other code in this file. +** This is done for no better reason than to reduce the number of +** pre-processor #ifndef statements. +*/ +#define sqlite3_release_memory(x) 0 /* 0 == no memory freed */ +#endif + /* ** For keeping track of the number of mallocs and frees. This ** is used to check for memory leaks. The iMallocFail and iMallocReset @@ -452,7 +479,8 @@ void OSMALLOC_FAILED(){ int OSSIZEOF(void *p){ if( p ){ - return sqlite3Os.xAllocationSize(p) - TESTALLOC_OVERHEAD; + u32 *pOs = (u32 *)getOsPointer(p); + return sqlite3Os.xAllocationSize(pOs) - TESTALLOC_OVERHEAD; } return 0; } @@ -1264,21 +1292,6 @@ void sqlite3MallocClearFailed(){ sqlite3Tsd()->mallocFailed = 0; } -#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT -/* -** Set the soft heap-size limit for the current thread. -*/ -void sqlite3_soft_heap_limit(int n){ - unsigned int N; - if( n<0 ){ - /* No limit */ - N = 0xFFFFFFFF; - }else{ - N = n; - } - sqlite3Tsd()->nSoftHeapLimit = N; -} -#endif #ifndef NDEBUG /* diff --git a/test/malloc5.test b/test/malloc5.test new file mode 100644 index 0000000000..917f515c94 --- /dev/null +++ b/test/malloc5.test @@ -0,0 +1,128 @@ +# 2005 November 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# $Id: malloc5.test,v 1.1 2005/12/19 14:18:12 danielk1977 Exp $ + +#--------------------------------------------------------------------------- +# NOTES ON EXPECTED BEHAVIOUR +# +#--------------------------------------------------------------------------- + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_test malloc5-1.1 { + # Simplest possible test. Call [db release_memory] when there is exactly + # one unused page in a single pager cache. This test case set's the + # value of the ::pgalloc variable, which is used in subsequent tests. + # + # Note: Even though executing this statement on an empty database + # modifies 2 pages (the root of sqlite_master and the new root page), + # the sqlite_master root (page 1) is never freed because the btree layer + # retains a reference to it for the entire transaction. + execsql { + BEGIN; + CREATE TABLE abc(a, b, c); + } + set ::pgalloc [db release_memory] + expr $::pgalloc > 0 +} {1} +do_test malloc5-1.2 { + # Test that the transaction started in the above test is still active. + # Because the page freed had been written to, freeing it required a + # journal sync and exclusive lock on the database file. Test the file + # appears to be locked. + sqlite3 db2 test.db + catchsql { + SELECT * FROM abc; + } db2 +} {1 {database is locked}} +do_test malloc5-1.3 { + # Again call [db release_memory] when there is exactly one unused page + # in the cache. The same amount of memory is required, but no journal-sync + # or exclusive lock should be established. + execsql { + COMMIT; + BEGIN; + SELECT * FROM abc; + } + db release_memory +} $::pgalloc +do_test malloc5-1.4 { + # Database should not be locked this time. + catchsql { + SELECT * FROM abc; + } db2 +} {0 {}} +do_test malloc5-1.5 { + # Manipulate the cache so that it contains two unused pages. One requires + # a journal-sync to free, the other does not. + execsql { + SELECT * FROM abc; + CREATE TABLE def(d, e, f); + } + db release_memory 500 +} $::pgalloc +do_test malloc5-1.6 { + # Database should not be locked this time. The above test case only + # requested 500 bytes of memory, which can be obtained by freeing the page + # that does not require an fsync(). + catchsql { + SELECT * FROM abc; + } db2 +} {0 {}} +do_test malloc5-1.7 { + # Release another 500 bytes of memory. This time we require a sync(), + # so the database file will be locked afterwards. + db release_memory 500 +} $::pgalloc +do_test malloc5-1.8 { + catchsql { + SELECT * FROM abc; + } db2 +} {1 {database is locked}} +do_test malloc5-1.9 { + execsql { + COMMIT; + } +} {} + + +do_test malloc5-2.1 { + # Put some data in tables abc and def. Both tables are still wholly + # contained within their root pages. + execsql { + INSERT INTO abc VALUES(1, 2, 3); + INSERT INTO abc VALUES(4, 5, 6); + INSERT INTO def VALUES(7, 8, 9); + INSERT INTO def VALUES(10,11,12); + } +} {} +do_test malloc5-2.2 { + # Load the root-page for table def into the cache. Then query table abc. + # Halfway through the query call sqlite3_release_memory(). The goal of this + # test is to make sure we don't free pages that are in use (specifically, + # the root of table abc). + set nRelease 0 + execsql { + BEGIN; + SELECT * FROM def; + } + db eval {SELECT * FROM abc} { + incr nRelease [db release_memory] + lappend data $a $b $c + } + list $nRelease $data +} [list $pgalloc [list 1 2 3 4 5 6]] + +finish_test + +