From: dan Date: Thu, 22 Feb 2018 16:46:42 +0000 (+0000) Subject: Add an LRU cache of uncompressed frame content to the zonefile virtual table X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=2707b5633470044961bf180a1adf6cccc51ac012;p=thirdparty%2Fsqlite.git Add an LRU cache of uncompressed frame content to the zonefile virtual table implementation. FossilOrigin-Name: 883e7e75d65622e8d06c46e48b7cc756cc0e3345b8124a8038cab0a8d51d0458 --- diff --git a/ext/zonefile/zonefile.c b/ext/zonefile/zonefile.c index fbf4ca846b..b2540e2484 100644 --- a/ext/zonefile/zonefile.c +++ b/ext/zonefile/zonefile.c @@ -1911,6 +1911,19 @@ static int zffUpdate( return rc; } +/* Each entry in the frame-cache is represented by an instance of the +** following structure. */ +typedef struct ZonefileFrame ZonefileFrame; +struct ZonefileFrame { + i64 iFileid; /* Fileid for this frame */ + i64 iFrameOff; /* Offset of frame in file */ + int nBuf; /* Size of aBuf[] in bytes */ + u8 *aBuf; /* Buffer containing uncompressed frame data */ + ZonefileFrame *pLruNext; /* Next element in LRU list */ + ZonefileFrame *pLruPrev; /* Previous element in LRU list */ + ZonefileFrame *pHashNext; /* Next element in same hash bucket */ +}; + typedef struct ZonefileTab ZonefileTab; struct ZonefileTab { sqlite3_vtab base; /* Base class - must be first */ @@ -1919,7 +1932,14 @@ struct ZonefileTab { ZonefileGlobal *pGlobal; char *zName; /* Name of this table */ char *zDb; /* Name of db containing this table */ - int nCacheSize; + + /* The following variables are used to implement the frame-cache */ + int nCacheSize; /* Configured cache size */ + int nCacheEntry; /* Current number of entries in cache */ + int nCacheBucket; /* Number of buckets in frame cache hash table */ + ZonefileFrame **aCache; /* Array of hash buckets */ + ZonefileFrame *pCacheFirst;/* Most recently used frame */ + ZonefileFrame *pCacheLast; /* Least recently used frame (first to discard) */ }; typedef struct ZonefileCsr ZonefileCsr; @@ -1928,14 +1948,6 @@ struct ZonefileCsr { sqlite3_stmt *pSelect; /* SELECT on %_shadow_idx table */ }; -typedef struct ZonefileFrame ZonefileFrame; -struct ZonefileFrame { - i64 iFileid; /* Fileid for this frame */ - i64 iFrameOff; /* Offset of frame in file */ - int nBuf; /* Size of aBuf[] in bytes */ - u8 *aBuf; /* Buffer containing uncompressed frame data */ -}; - /* ** Attempt to interpret the contents of string z as an integer. If ** successful, set (*piVal) to the integer value and return SQLITE_OK. @@ -2072,6 +2084,7 @@ static int zonefileCreateConnect( for(i=3; inCacheSize<1 ) p->nCacheSize = 1; } if( rc!=SQLITE_OK ){ @@ -2108,16 +2121,6 @@ static int zonefileConnect( return zonefileCreateConnect(0, pAux, db, argc, argv, ppVtab, pzErr); } -/* -** zonefile virtual table module xDisconnect method. -*/ -static int zonefileDisconnect(sqlite3_vtab *pVtab){ - ZonefileTab *pTab = (ZonefileTab*)pVtab; - sqlite3_finalize(pTab->pIdToName); - sqlite3_free(pTab); - return SQLITE_OK; -} - /* ** Zonefile virtual table module xBestIndex method. ** @@ -2191,32 +2194,6 @@ static int zonefileBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pInfo){ return SQLITE_OK; } -/* -** zonefile virtual table module xDestroy method. -*/ -static int zonefileDestroy(sqlite3_vtab *pVtab){ - ZonefileTab *pTab = (ZonefileTab*)pVtab; - int rc = SQLITE_OK; - char *zSql = sqlite3_mprintf( - "DROP TABLE IF EXISTS %Q.'%q_shadow_idx';" - "DROP TABLE IF EXISTS %Q.'%q_shadow_file';" - "DROP TABLE IF EXISTS %Q.'%q_shadow_frame';" - "DROP TABLE IF EXISTS %Q.'%q_files';", - pTab->zDb, pTab->zName, pTab->zDb, pTab->zName, pTab->zDb, pTab->zName - ); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0); - sqlite3_free(zSql); - } - - if( rc==SQLITE_OK ){ - zonefileDisconnect(pVtab); - } - return rc; -} - /* ** zonefile virtual table module xOpen method. */ @@ -2320,28 +2297,6 @@ static void zonefileFree(void *p){ sqlite3_free(p); } -static int zonefileCtxUncompress( - sqlite3_context *pCtx, - ZonefileCompress *pMethod, - void *pCmp, - u8 *aBuf, int nBuf, - int iKeyOff, int nKey -){ - int rc; - u8 *aUn = 0; - int nUn = 0; - - rc = zonefileUncompress(pMethod, pCmp, aBuf, nBuf, &aUn, &nUn); - if( rc==SQLITE_OK ){ - sqlite3_result_blob(pCtx, &aUn[iKeyOff], nKey, SQLITE_TRANSIENT); - }else if( rc==SQLITE_ERROR ){ - zonefileCtxError(pCtx, "failed to uncompress frame"); - } - - sqlite3_free(aUn); - return rc; -} - static int zonefileGetFile( sqlite3_context *pCtx, /* Leave error message here */ ZonefileCsr *pCsr, /* Cursor object */ @@ -2423,18 +2378,146 @@ static int zonefileValueReadDirect(sqlite3_context *pCtx, ZonefileCsr *pCsr){ return rc; } +#ifndef NDEBUG +static void zonefileCacheCheck(ZonefileTab *pTab){ + ZonefileFrame *p; + int n = 0; + for(p=pTab->pCacheFirst; p; p=p->pLruNext){ + assert( p!=pTab->pCacheFirst || p->pLruPrev==0 ); + assert( p!=pTab->pCacheLast || p->pLruNext==0 ); + assert( p==pTab->pCacheFirst || p->pLruPrev->pLruNext==p ); + assert( p==pTab->pCacheLast || p->pLruNext->pLruPrev==p ); + n++; + } + assert( n==pTab->nCacheEntry ); +} +#else +# define zonefileCacheCheck(x) +#endif + +/* +** Search the frame-cache belonging to virtual table pTab for an entry +** corresponding to the frame at byte offset iFrameOff of the file with +** file id iFile. If such an entry is found, return a pointer to it. +** Otherwise, if no such entry exists, return NULL. +*/ static ZonefileFrame *zonefileCacheFind( ZonefileTab *pTab, i64 iFile, i64 iFrameOff ){ + ZonefileFrame *pFrame; + for(pFrame=pTab->pCacheFirst; pFrame; pFrame=pFrame->pLruNext){ + if( pFrame->iFileid==iFile && pFrame->iFrameOff==iFrameOff ){ + /* Found a match. Move it to the front of the LRU list and return + ** a pointer to it. */ + assert( (pFrame->pLruPrev==0)==(pFrame==pTab->pCacheFirst) ); + assert( (pFrame->pLruNext==0)==(pFrame==pTab->pCacheLast) ); + if( pFrame->pLruPrev ){ + assert( pFrame==pFrame->pLruPrev->pLruNext ); + pFrame->pLruPrev->pLruNext = pFrame->pLruNext; + + if( pFrame->pLruNext ){ + assert( pFrame==pFrame->pLruNext->pLruPrev ); + pFrame->pLruNext->pLruPrev = pFrame->pLruPrev; + }else{ + assert( pFrame==pTab->pCacheLast ); + pTab->pCacheLast = pFrame->pLruPrev; + pTab->pCacheLast->pLruNext = 0; + } + + pFrame->pLruPrev = 0; + pFrame->pLruNext = pTab->pCacheFirst; + pTab->pCacheFirst->pLruPrev = pFrame; + pTab->pCacheFirst = pFrame; + } + + zonefileCacheCheck(pTab); + return pFrame; + } + } return 0; } -static void zonefileCacheStore( - ZonefileTab *pTab, - ZonefileFrame *pFrame -){ +/* +** Return the index of the hash bucket that iFileid/iFrameOff belongs to. +*/ +int zonefileCacheHash(ZonefileTab *pTab, i64 iFileid, i64 iFrameOff){ + u32 h; + h = (iFileid & 0xFFFFFFFF) ^ (iFrameOff & 0xFFFFFFFF); + return (h % pTab->nCacheBucket); +} + +/* +** Store the frame object passed as the second argument in the frame +** cache belonging to table pTab. +*/ +static int zonefileCacheStore(ZonefileTab *pTab, ZonefileFrame *pFrame){ + int h; + + /* Allocate the hash table if it has not already been allocated. */ + if( pTab->aCache==0 ){ + int nByte = pTab->nCacheSize * 2 * sizeof(ZonefileFrame*); + pTab->aCache = (ZonefileFrame**)sqlite3_malloc(nByte); + if( pTab->aCache==0 ){ + sqlite3_free(pFrame); + return SQLITE_NOMEM; + } + memset(pTab->aCache, 0, nByte); + pTab->nCacheBucket = pTab->nCacheSize * 2; + } + + /* Add the new entry to the hash table */ + h = zonefileCacheHash(pTab, pFrame->iFileid, pFrame->iFrameOff); + assert( h>=0 && hnCacheBucket ); + pFrame->pHashNext = pTab->aCache[h]; + pTab->aCache[h] = pFrame; + + /* Add the new entry to the LRU list */ + pFrame->pLruPrev = 0; + pFrame->pLruNext = pTab->pCacheFirst; + pTab->pCacheFirst = pFrame; + if( pFrame->pLruNext==0 ){ + pTab->pCacheLast = pFrame; + }else{ + pFrame->pLruNext->pLruPrev = pFrame; + } + pTab->nCacheEntry++; + + if( pTab->nCacheEntry>pTab->nCacheSize ){ + ZonefileFrame **pp; + + /* Remove the oldest entry from the LRU list. */ + ZonefileFrame *pLast = pTab->pCacheLast; + assert( pTab->pCacheLast ); + assert( pTab->nCacheEntry>1 ); + pTab->pCacheLast = pLast->pLruPrev; + pTab->pCacheLast->pLruNext = 0; + pTab->nCacheEntry--; + + /* Remove the same entry from the hash table. */ + h = zonefileCacheHash(pTab, pLast->iFileid, pLast->iFrameOff); + assert( h>=0 && hnCacheBucket ); + for(pp=&pTab->aCache[h]; *pp!=pLast; pp=&((*pp)->pHashNext)); + *pp = pLast->pHashNext; + sqlite3_free(pLast); + } + zonefileCacheCheck(pTab); + + return SQLITE_OK; +} + +/* +** Delete all resources associated with the frame-cache for table pTab. +*/ +static void zonefileCacheDelete(ZonefileTab *pTab){ + ZonefileFrame *p; + ZonefileFrame *pNext; + for(p=pTab->pCacheFirst; p; p=pNext){ + pNext = p->pLruNext; + sqlite3_free(p); + } + sqlite3_free(pTab->aCache); } static int zonefileValueReadCache(sqlite3_context *pCtx, ZonefileCsr *pCsr){ @@ -2537,6 +2620,7 @@ static int zonefileValueReadCache(sqlite3_context *pCtx, ZonefileCsr *pCsr){ if( p==0 ){ rc = SQLITE_NOMEM; }else{ + memset(p, 0, sizeof(ZonefileFrame)); p->aBuf = (u8*)&p[1]; p->nBuf = nOut; p->iFrameOff = iFrameOff; @@ -2552,6 +2636,9 @@ static int zonefileValueReadCache(sqlite3_context *pCtx, ZonefileCsr *pCsr){ if( rc!=SQLITE_OK ){ sqlite3_free(pFrame); pFrame = 0; + }else{ + rc = zonefileCacheStore(pTab, pFrame); + if( rc!=SQLITE_OK ) pFrame = 0; } zonefileReleaseFile(pCsr); zonefileFileClose(pFd); @@ -2568,7 +2655,6 @@ static int zonefileValueReadCache(sqlite3_context *pCtx, ZonefileCsr *pCsr){ if( pFrame ){ assert( rc==SQLITE_OK ); sqlite3_result_blob(pCtx, &pFrame->aBuf[iKeyOff], nKeySz, SQLITE_TRANSIENT); - sqlite3_free(pFrame); } return rc; @@ -2612,6 +2698,43 @@ static int zonefileColumn( return rc; } +/* +** zonefile virtual table module xDisconnect method. +*/ +static int zonefileDisconnect(sqlite3_vtab *pVtab){ + ZonefileTab *pTab = (ZonefileTab*)pVtab; + zonefileCacheDelete(pTab); + sqlite3_finalize(pTab->pIdToName); + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** zonefile virtual table module xDestroy method. +*/ +static int zonefileDestroy(sqlite3_vtab *pVtab){ + ZonefileTab *pTab = (ZonefileTab*)pVtab; + int rc = SQLITE_OK; + char *zSql = sqlite3_mprintf( + "DROP TABLE IF EXISTS %Q.'%q_shadow_idx';" + "DROP TABLE IF EXISTS %Q.'%q_shadow_file';" + "DROP TABLE IF EXISTS %Q.'%q_shadow_frame';" + "DROP TABLE IF EXISTS %Q.'%q_files';", + pTab->zDb, pTab->zName, pTab->zDb, pTab->zName, pTab->zDb, pTab->zName + ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + zonefileDisconnect(pVtab); + } + return rc; +} + /* ** zonefile virtual table module xRowid method. */ diff --git a/ext/zonefile/zonefile1.test b/ext/zonefile/zonefile1.test index 33e226c014..24a5205ef7 100644 --- a/ext/zonefile/zonefile1.test +++ b/ext/zonefile/zonefile1.test @@ -108,6 +108,7 @@ if {[lsearch $COMPRESSION_METHODS zstd_global_dict]>=0} { # puts $COMPRESSION_METHODS set extra_header 0 +set cachesize 0 foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS { if {$cmpidx == "zstd_global_dict"} continue reset_db @@ -115,8 +116,9 @@ foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS { set tn "$cmp/$cmpidx" set extra_header [expr {$extra_header ? 0 : 100}] + set cachesize [expr {$cachesize ? 0 : 10}] - do_execsql_test 2.$tn.0 { + do_execsql_test 2.$tn.0.1 { CREATE TABLE zz( k INTEGER PRIMARY KEY, frame INTEGER DEFAULT -1, @@ -124,8 +126,11 @@ foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS { v BLOB ); CREATE TABLE rt(k INTEGER PRIMARY KEY, v BLOB); - CREATE VIRTUAL TABLE zone USING zonefile; } + + do_execsql_test 2.$tn.0.2 " + CREATE VIRTUAL TABLE zone USING zonefile(cachesize = $cachesize) + " {} set nMinByte 0 set nMaxByte 444 diff --git a/manifest b/manifest index 46437ab12a..c3e848b156 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modifications\sto\sthe\szonefile\smodule\sto\smake\sit\seasier\sto\sadd\sa\scache\sof\nuncompressed\sframe\scontent. -D 2018-02-21T21:15:45.427 +C Add\san\sLRU\scache\sof\suncompressed\sframe\scontent\sto\sthe\szonefile\svirtual\stable\nimplementation. +D 2018-02-22T16:46:42.652 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -409,8 +409,8 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F ext/zonefile/README.md df86ef5b4f9aa8b07e1c8124b3f2dcea616927385aad59d525b784f0a06d446c -F ext/zonefile/zonefile.c 55bd8e0ff63f899903ee07936e32f54cb28045916ffba58d97a8bf2cb8f55e53 -F ext/zonefile/zonefile1.test 4cd9fa8d333c195f59792c4d5c1e8387e778e46354580fbef0b84efc932c6a47 +F ext/zonefile/zonefile.c ce2963a584f6e98f9238e84a5f7336d52750f669b4dc2d308842f78204189872 +F ext/zonefile/zonefile1.test 51ba36bd254026b86ceecb358a3f33837c7d086b75d3b45db85d0aee3fd0a9a0 F ext/zonefile/zonefileenc.test 5cc89a1e716b127a5220f03162ced3bd9d7df8819fe04a87f2d962315e8ebdc1 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 @@ -1709,7 +1709,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a63ea652546a4c63eccd72665becff38a97a0e39d2f11703cb6899451570fd4 -R 436b9d8cf180e93dddc41915bda96996 +P d9d5cc62f11058f9ba560381367ff4765dbbde08184e55abdb50ae1b6bf4a016 +R fb3194c43333fefa4c70a2354320310d U dan -Z 5dfc0a85444f2182f7082352bc1286fe +Z 7246c4cd1ede6422af161378c0967b5a diff --git a/manifest.uuid b/manifest.uuid index 111cf71533..b35f018b44 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9d5cc62f11058f9ba560381367ff4765dbbde08184e55abdb50ae1b6bf4a016 \ No newline at end of file +883e7e75d65622e8d06c46e48b7cc756cc0e3345b8124a8038cab0a8d51d0458 \ No newline at end of file