From: danielk1977 Date: Sun, 18 Dec 2005 08:51:22 +0000 (+0000) Subject: Add the (untested) sqlite3_release_memory() function. (CVS 2825) X-Git-Tag: version-3.6.10~3335 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=13f7299bbe4490460dc4915279ecb167c1d9931b;p=thirdparty%2Fsqlite.git Add the (untested) sqlite3_release_memory() function. (CVS 2825) FossilOrigin-Name: 345addaa03d3bfa3429a59597fbd3addcff62e30 --- diff --git a/manifest b/manifest index a846a709fe..8f54e5cae3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Verify\sthat\sthe\srollback-hook\sis\sinvoked\scorrectly\swhen\sa\smalloc()\sfailure\soccurs.\s(CVS\s2824) -D 2005-12-16T15:24:29 +C Add\sthe\s(untested)\ssqlite3_release_memory()\sfunction.\s(CVS\s2825) +D 2005-12-18T08:51:23 F Makefile.in e3c6b3a38d734d41574c04f2fc90d18de2b87102 F Makefile.linux-gcc aee18d8a05546dcf1888bd4547e442008a49a092 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -59,8 +59,8 @@ F src/os_unix.c 6394d2fa3a8bfbceb227579b44b4b343b5b54a8f F src/os_unix.h 5768d56d28240d3fe4537fac08cc85e4fb52279e F src/os_win.c 9feb97f49b93d451f8ef7c5dd388e05a44647dc6 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b -F src/pager.c 49f63a54b57164a70df0b1539141003fd27856c6 -F src/pager.h e7b41ce8e7b5f629d456708b7ad9a8c8ede37140 +F src/pager.c d501a79293f575065e3ab9169e2dea7b945ce155 +F src/pager.h 5c14873902723bf735582d55d7042abfbb3810bc F src/parse.y 142a4b347c82217332e2d3dfa317ff2b7ac32f9c F src/pragma.c 8883b4d34796efa315bdd0ec1b03f580ef1575b9 F src/prepare.c 1417a396efe55e2767f9f97f694d21b8cac2f4d6 @@ -68,8 +68,8 @@ F src/printf.c f47a2f4b5387cd2ebb12e9117a1a5d6bd9a2b812 F src/random.c ff5e9a8cad790e2a51cd4d2e7737dc8540e09d1d F src/select.c 2292b065bc6be61e01aad39a2e1b93e332fb7e57 F src/shell.c 4872acee1d2a826c73c914961e469e563204b7f9 -F src/sqlite.h.in 184143f88471cdd4690ae9304ba134a48d10dae1 -F src/sqliteInt.h e6063bc65e02682ee923f795b74afa85d9823f9a +F src/sqlite.h.in addab2c24bc84b4fe74cda7bb102abd34983b14f +F src/sqliteInt.h 5581e01a1ca1a9a4d70babb96021740bc17eea2e F src/table.c 486dcfce532685b53b5a2b5da8bba0ded6fb2316 F src/tclsqlite.c b21dd96ffab1625c986513e003a1945a6ae112ae F src/test1.c d6924b182773b2ad3b22e435e4d3bfd5a846da9e @@ -82,7 +82,7 @@ F src/tokenize.c 7a3a3d3cc734f684a77c4dfd09eb46fcee25394c F src/trigger.c 2925ba96d964d9b717e74006bf7e64b8a6b70d97 F src/update.c ec8e540617b116725b5a55c8d6b4db8bc67fdd7d F src/utf.c b7bffac4260177ae7f83c01d025fe0f5ed70ce71 -F src/util.c 8bb5e0553692d36c769d6cd033eb7f68a7586648 +F src/util.c 31edbc07f9fb44b23d0c33709ecd50b89ecd96a0 F src/vacuum.c fbfdd3967fd34e2f260fafed88dcbf3c10856b94 F src/vdbe.c 09aaed71f076bfd4286607ee4845100b910a492f F src/vdbe.h 8729a4ee16ff9aeab2af9667df3cf300ff978e13 @@ -327,7 +327,7 @@ F www/tclsqlite.tcl ddcf912ea48695603c8ed7efb29f0812ef8d1b49 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl a99cf5f6d8bd4d5537584a2b342f0fb9fa601d8b F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513 -P 3baa3ff32435b64e7ae7646b17a98fef9296aaa0 -R 7c3f6fd17e2d44aaf5ad837b01c26c23 +P 83c8ae5bee3b6bdb556d2e85fa260ba855742601 +R 7b8c8cafe73ba9b7fdaabb41edaaf0e4 U danielk1977 -Z 06f497bf2cbb4256c1ee4b87f97222aa +Z e796870a052bdfafa8d3fb52aa5cdd1a diff --git a/manifest.uuid b/manifest.uuid index 8c8443dd54..a017fcd3ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83c8ae5bee3b6bdb556d2e85fa260ba855742601 \ No newline at end of file +345addaa03d3bfa3429a59597fbd3addcff62e30 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d52334e722..4ea0c46397 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,7 +18,7 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.224 2005/12/09 20:02:05 drh Exp $ +** @(#) $Id: pager.c,v 1.225 2005/12/18 08:51:23 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" @@ -284,6 +284,9 @@ struct Pager { void (*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */ void *pCodecArg; /* First argument to xCodec() */ PgHdr *aHash[N_PG_HASH]; /* Hash table to map page number to PgHdr */ +#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT + Pager *pNext; /* Linked list of pagers in this thread */ +#endif }; /* @@ -1595,6 +1598,7 @@ int sqlite3pager_open( int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; int noReadlock = (flags & PAGER_NO_READLOCK)!=0; char zTemp[SQLITE_TEMPNAME_SIZE]; + SqliteTsd *pTsd = sqlite3Tsd(); *ppPager = 0; memset(&fd, 0, sizeof(fd)); @@ -1680,6 +1684,8 @@ int sqlite3pager_open( pPager->pBusyHandler = 0; memset(pPager->aHash, 0, sizeof(pPager->aHash)); *ppPager = pPager; + pPager->pNext = pTsd->pPager; + pTsd->pPager = pPager; return SQLITE_OK; } @@ -1970,6 +1976,10 @@ int sqlite3pager_truncate(Pager *pPager, Pgno nPage){ */ int sqlite3pager_close(Pager *pPager){ PgHdr *pPg, *pNext; +#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT + SqliteTsd *pTsd = sqlite3Tsd(); +#endif + switch( pPager->state ){ case PAGER_RESERVED: case PAGER_SYNCED: @@ -2026,6 +2036,16 @@ int sqlite3pager_close(Pager *pPager){ ** } */ +#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT + if( pPager==pTsd->pPager ){ + pTsd->pPager = pPager->pNext; + }else{ + Pager *pTmp; + for(pTmp = pTsd->pPager; pTmp->pNext!=pPager; pTmp=pTmp->pNext); + pTmp->pNext = pPager->pNext; + } +#endif + sqliteFree(pPager); return SQLITE_OK; } @@ -2289,6 +2309,154 @@ static int hasHotJournal(Pager *pPager){ } } +static int pager_recycle(Pager *pPager, int syncOk, PgHdr **ppPg){ + PgHdr *pPg; + *ppPg = 0; + + /* Find a page to recycle. Try to locate a page that does not + ** require us to do an fsync() on the journal. + */ + pPg = pPager->pFirstSynced; + + /* If we could not find a page that does not require an fsync() + ** on the journal file then fsync the journal file. This is a + ** very slow operation, so we work hard to avoid it. But sometimes + ** it can't be helped. + */ + if( pPg==0 && syncOk ){ + int rc = syncJournal(pPager); + if( rc!=0 ){ + sqlite3pager_rollback(pPager); + return SQLITE_IOERR; + } + if( pPager->fullSync ){ + /* If in full-sync mode, write a new journal header into the + ** journal file. This is done to avoid ever modifying a journal + ** header that is involved in the rollback of pages that have + ** already been written to the database (in case the header is + ** trashed when the nRec field is updated). + */ + pPager->nRec = 0; + assert( pPager->journalOff > 0 ); + rc = writeJournalHdr(pPager); + if( rc!=0 ){ + sqlite3pager_rollback(pPager); + return SQLITE_IOERR; + } + } + pPg = pPager->pFirst; + } + if( pPg==0 ){ + return SQLITE_OK; + } + + assert( pPg->nRef==0 ); + + /* Write the page to the database file if it is dirty. + */ + if( pPg->dirty ){ + int rc; + assert( pPg->needSync==0 ); + pPg->pDirty = 0; + rc = pager_write_pagelist( pPg ); + if( rc!=SQLITE_OK ){ + sqlite3pager_rollback(pPager); + return SQLITE_IOERR; + } + } + assert( pPg->dirty==0 ); + + /* If the page we are recycling is marked as alwaysRollback, then + ** set the global alwaysRollback flag, thus disabling the + ** sqlite_dont_rollback() optimization for the rest of this transaction. + ** It is necessary to do this because the page marked alwaysRollback + ** might be reloaded at a later time but at that point we won't remember + ** that is was marked alwaysRollback. This means that all pages must + ** be marked as alwaysRollback from here on out. + */ + if( pPg->alwaysRollback ){ + pPager->alwaysRollback = 1; + } + + /* Unlink the old page from the free list and the hash table + */ + unlinkPage(pPg); + TEST_INCR(pPager->nOvfl); + + *ppPg = pPg; + return SQLITE_OK; +} + +/* +** This function is called to free superfluous dynamically allocated memory +** held by the pager system. Memory in use by any SQLite pager allocated +** by the current thread may be sqliteFree()ed. +** +** nReq is the number of bytes of memory required. Once this much has +** been released, the function returns. The return value is the total number +** of bytes of memory released. +*/ +int sqlite3pager_release_memory(int nReq){ + SqliteTsd *pTsd = sqlite3Tsd(); + Pager *pPager; + int nReleased = 0; + int i; + + /* Outermost loop runs for at most two iterations. First iteration we + ** try to find memory that can be released without calling fsync(). Second + ** iteration (which only runs if the first failed to free nReq bytes of + ** memory) is permitted to call fsync(). This is of course much more + ** expensive. + */ + for(i=0; i==0 || i==1; i++){ + + /* Loop through all the SQLite pagers opened by the current thread. */ + for(pPager=pTsd->pPager; pPager && nReleasedpNext){ + PgHdr *pPg; + int rc; + + /* For each pager, try to free as many pages as possible (without + ** calling fsync() if this is the first iteration of the outermost + ** loop). + */ + while( SQLITE_OK==(rc = pager_recycle(pPager, i, &pPg)) && pPg) { + /* We've found a page to free. At this point the page has been + ** removed from the page hash-table, free-list and synced-list + ** (pFirstSynced). It is still in the all pages (pAll) list. + ** Remove it from this list before freeing. + ** + ** Todo: Check the Pager.pStmt list to make sure this is Ok. It + ** probably is though. + */ + PgHdr *pTmp; + if( pPg==pPager->pAll ){ + pPager->pAll = pPg->pNextAll; + }else{ + for( pTmp=pPager->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll ); + pTmp->pNextAll = pPg->pNextAll; + } + nReleased += sqliteAllocSize(pPg); + sqliteFree(pPg); + } + + if( rc!=SQLITE_OK ){ + /* Assert that fsync() was enabled and the error was an io-error + ** or a full database. Nothing else should be able to wrong in + ** pager_recycle. + */ + assert( i && (rc==SQLITE_IOERR || rc==SQLITE_FULL) ); + + /* TODO: Figure out what to do about this. The IO-error + ** belongs to the connection that is executing a transaction. + */ + assert(0); + } + } + } + + return nReleased; +} + /* ** Acquire a page. ** @@ -2429,70 +2597,11 @@ int sqlite3pager_get(Pager *pPager, Pgno pgno, void **ppPage){ pPager->nMaxPage++; } }else{ - /* Find a page to recycle. Try to locate a page that does not - ** require us to do an fsync() on the journal. - */ - pPg = pPager->pFirstSynced; - - /* If we could not find a page that does not require an fsync() - ** on the journal file then fsync the journal file. This is a - ** very slow operation, so we work hard to avoid it. But sometimes - ** it can't be helped. - */ - if( pPg==0 ){ - int rc = syncJournal(pPager); - if( rc!=0 ){ - sqlite3pager_rollback(pPager); - return SQLITE_IOERR; - } - if( pPager->fullSync ){ - /* If in full-sync mode, write a new journal header into the - ** journal file. This is done to avoid ever modifying a journal - ** header that is involved in the rollback of pages that have - ** already been written to the database (in case the header is - ** trashed when the nRec field is updated). - */ - pPager->nRec = 0; - assert( pPager->journalOff > 0 ); - rc = writeJournalHdr(pPager); - if( rc!=0 ){ - sqlite3pager_rollback(pPager); - return SQLITE_IOERR; - } - } - pPg = pPager->pFirst; - } - assert( pPg->nRef==0 ); - - /* Write the page to the database file if it is dirty. - */ - if( pPg->dirty ){ - assert( pPg->needSync==0 ); - pPg->pDirty = 0; - rc = pager_write_pagelist( pPg ); - if( rc!=SQLITE_OK ){ - sqlite3pager_rollback(pPager); - return SQLITE_IOERR; - } - } - assert( pPg->dirty==0 ); - - /* If the page we are recycling is marked as alwaysRollback, then - ** set the global alwaysRollback flag, thus disabling the - ** sqlite_dont_rollback() optimization for the rest of this transaction. - ** It is necessary to do this because the page marked alwaysRollback - ** might be reloaded at a later time but at that point we won't remember - ** that is was marked alwaysRollback. This means that all pages must - ** be marked as alwaysRollback from here on out. - */ - if( pPg->alwaysRollback ){ - pPager->alwaysRollback = 1; + rc = pager_recycle(pPager, 1, &pPg); + if( rc!=SQLITE_OK ){ + return rc; } - - /* Unlink the old page from the free list and the hash table - */ - unlinkPage(pPg); - TEST_INCR(pPager->nOvfl); + assert(pPg) ; } pPg->pgno = pgno; if( pPager->aInJournal && (int)pgno<=pPager->origDbSize ){ diff --git a/src/pager.h b/src/pager.h index f29ecac1f5..64d6dbd5f0 100644 --- a/src/pager.h +++ b/src/pager.h @@ -13,9 +13,12 @@ ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** -** @(#) $Id: pager.h,v 1.46 2005/09/19 19:05:21 drh Exp $ +** @(#) $Id: pager.h,v 1.47 2005/12/18 08:51:24 danielk1977 Exp $ */ +#ifndef _PAGER_H_ +#define _PAGER_H_ + /* ** The default size of a database page. */ @@ -114,3 +117,5 @@ int sqlite3pager_lockstate(Pager*); void sqlite3pager_refdump(Pager*); int pager3_refinfo_enable; #endif + +#endif /* _PAGER_H_ */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 551fddc23b..7fb156077e 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -12,7 +12,7 @@ ** This header file defines the interface that the SQLite library ** presents to client programs. ** -** @(#) $Id: sqlite.h.in,v 1.148 2005/12/16 15:24:29 danielk1977 Exp $ +** @(#) $Id: sqlite.h.in,v 1.149 2005/12/18 08:51:24 danielk1977 Exp $ */ #ifndef _SQLITE3_H_ #define _SQLITE3_H_ @@ -1287,7 +1287,7 @@ sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** allocated anyway and the current operation proceeds. ** ** This function is only available if the library was compiled without the -** SQLITE_OMIT_SOFTHEAPLIMIT option set. +** SQLITE_OMIT_MEMORY_MANAGEMENT option set. */ void sqlite3_soft_heap_limit(int); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 11624cab9d..aa2a186f59 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -11,7 +11,7 @@ ************************************************************************* ** Internal interface definitions for SQLite. ** -** @(#) $Id: sqliteInt.h,v 1.439 2005/12/16 06:54:02 danielk1977 Exp $ +** @(#) $Id: sqliteInt.h,v 1.440 2005/12/18 08:51:24 danielk1977 Exp $ */ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ @@ -226,6 +226,7 @@ struct BusyHandler { */ #include "vdbe.h" #include "btree.h" +#include "pager.h" /* ** This macro casts a pointer to an integer. Useful for doing @@ -261,7 +262,8 @@ extern int sqlite3_iMallocReset; /* Set iMallocFail to this when it reaches 0 */ #endif -#define sqliteFree(x) sqlite3FreeX(x) +#define sqliteFree(x) sqlite3FreeX(x) +#define sqliteAllocSize(x) sqlite3AllocSize(x) /* ** An instance of this structure is allocated for each thread that uses SQLite. @@ -270,9 +272,11 @@ typedef struct SqliteTsd SqliteTsd; struct SqliteTsd { int isInit; /* True if structure has been initialised */ int mallocFailed; /* True after a malloc() has failed */ -#ifndef SQLITE_OMIT_SOFTHEAPLIMIT + +#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT unsigned int nSoftHeapLimit; /* (uint)-1 for unlimited */ unsigned int nAlloc; /* Number of bytes currently allocated */ + Pager *pPager; /* Linked list of all pagers in this thread */ #endif #ifndef NDEBUG @@ -1437,6 +1441,7 @@ char *sqlite3StrNDup(const char*, int); void sqlite3ReallocOrFree(void**,int); void sqlite3FreeX(void*); void *sqlite3MallocX(int); +int sqlite3AllocSize(void *); char *sqlite3MPrintf(const char*, ...); char *sqlite3VMPrintf(const char*, va_list); diff --git a/src/util.c b/src/util.c index af25da9227..109f12fe5e 100644 --- a/src/util.c +++ b/src/util.c @@ -14,7 +14,7 @@ ** This file contains functions for allocating memory, comparing ** strings, and stuff like that. ** -** $Id: util.c,v 1.154 2005/12/15 10:50:54 danielk1977 Exp $ +** $Id: util.c,v 1.155 2005/12/18 08:51:24 danielk1977 Exp $ */ #include "sqliteInt.h" #include "os.h" @@ -32,6 +32,7 @@ ** sqlite3Realloc() ** sqlite3ReallocOrFree() ** sqlite3Free() +** sqlite3AllocSize() ** ** The function sqlite3FreeX performs the same task as sqlite3Free and is ** guaranteed to be a real function. @@ -457,11 +458,15 @@ int OSSIZEOF(void *p){ } #else -#define OSMALLOC(x) sqlite3Os.xMalloc(x) -#define OSREALLOC(x,y) sqlite3Os.xRealloc(x,y) -#define OSFREE(x) sqlite3Os.xFree(x) -#define OSSIZEOF(x) sqlite3Os.xAllocationSize(x) +/* Define macros to call the sqlite3Os.xXXX interface directly if +** the SQLITE_MEMDEBUG macro is not defined. +*/ +#define OSMALLOC(x) sqlite3Os.xMalloc(x) +#define OSREALLOC(x,y) sqlite3Os.xRealloc(x,y) +#define OSFREE(x) sqlite3Os.xFree(x) +#define OSSIZEOF(x) sqlite3Os.xAllocationSize(x) #define OSMALLOC_FAILED() + #endif /* ** End code for memory allocation system test layer. @@ -477,9 +482,9 @@ int OSSIZEOF(void *p){ ** called to try to avoid this. No indication of whether or not this is ** successful is returned to the caller. ** -** If SQLITE_OMIT_SOFTHEAPLIMIT is defined, this function is a no-op. +** If SQLITE_OMIT_MEMORY_MANAGEMENT is defined, this function is a no-op. */ -#ifndef SQLITE_OMIT_SOFTHEAPLIMIT +#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT static void handleSoftLimit(int n){ SqliteTsd *pTsd = sqlite3Tsd(); pTsd->nAlloc += n; @@ -573,6 +578,20 @@ void sqlite3ReallocOrFree(void **pp, int n){ *pp = p; } +/* +** Return the number of bytes allocated at location p. p must be either +** a NULL pointer (in which case 0 is returned) or a pointer returned by +** sqlite3Malloc(), sqlite3Realloc() or sqlite3ReallocOrFree(). +** +** The number of bytes allocated does not include any overhead inserted by +** any malloc() wrapper functions that may be called. So the value returned +** is the number of bytes that were available to SQLite using pointer p, +** regardless of how much memory was actually allocated. +*/ +int sqlite3AllocSize(void *p){ + return OSSIZEOF(p); +} + /* ** Make a copy of a string in memory obtained from sqliteMalloc(). These ** functions call sqlite3MallocRaw() directly instead of sqliteMalloc(). This @@ -1245,7 +1264,7 @@ void sqlite3MallocClearFailed(){ sqlite3Tsd()->mallocFailed = 0; } -#ifndef SQLITE_OMIT_SOFTHEAPLIMIT +#ifndef SQLITE_OMIT_MEMORY_MANAGEMENT /* ** Set the soft heap-size limit for the current thread. */