From: dan Date: Wed, 21 Feb 2018 16:36:08 +0000 (+0000) Subject: Have zonefile store encryption keys in a hash-table instead of a linked list. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=d98b4b24f2d6c620c9fd2268b6d1de4028aefa4f;p=thirdparty%2Fsqlite.git Have zonefile store encryption keys in a hash-table instead of a linked list. Add extra tests for key management. FossilOrigin-Name: 3a63ea652546a4c63eccd72665becff38a97a0e39d2f11703cb6899451570fd4 --- diff --git a/ext/zonefile/zonefile.c b/ext/zonefile/zonefile.c index d06b8fb1c6..2bd73f5c3d 100644 --- a/ext/zonefile/zonefile.c +++ b/ext/zonefile/zonefile.c @@ -15,6 +15,10 @@ SQLITE_EXTENSION_INIT1 #ifndef SQLITE_OMIT_VIRTUALTABLE +#include +#include +#include + #ifndef SQLITE_AMALGAMATION typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; @@ -130,20 +134,174 @@ static void zonefileCodecDestroy(ZonefileCodec *pCodec){ } #endif /* SQLITE_HAVE_ZONEFILE_CODEC */ +/* +** All zonefile and zonefile_files virtual table instances that belong +** to the same database handle (sqlite3*) share a single instance of the +** ZonefileGlobal object. This global object contains a table of +** configured encryption keys for the various zonefiles in the system. +*/ typedef struct ZonefileGlobal ZonefileGlobal; typedef struct ZonefileKey ZonefileKey; struct ZonefileGlobal { - ZonefileKey *pList; + int nEntry; /* Number of entries in the hash table */ + int nHash; /* Size of aHash[] array */ + ZonefileKey **aHash; /* Hash buckets */ }; struct ZonefileKey { - const char *zName; /* Table name */ - const char *zDb; /* Database name */ + const char *zName; /* Zonefile table name */ + const char *zDb; /* Database name ("main", "temp" etc.) */ i64 iFileid; /* File id */ const char *zKey; /* Key buffer */ int nKey; /* Size of zKey in bytes */ - ZonefileKey *pNext; /* Next key on same db connection */ + u32 iHash; /* zonefileKeyHash() value */ + ZonefileKey *pHashNext; /* Next colliding key in hash table */ }; +static u32 zonefileKeyHash( + const char *zDb, + const char *zTab, + i64 iFileid +){ + u32 iHash = 0; + int i; + for(i=0; zDb[i]; i++) iHash += (iHash<<3) + (u8)zDb[i]; + for(i=0; zTab[i]; i++) iHash += (iHash<<3) + (u8)zTab[i]; + return (iHash ^ (iFileid & 0xFFFFFFFF)); +} + +/* +** Store encryption key zKey in the key-store passed as the first argument. +** Return SQLITE_OK if successful, or an SQLite error code (SQLITE_NOMEM) +** otherwise. +*/ +static int zonefileKeyStore( + ZonefileGlobal *pGlobal, + const char *zDb, /* Database containing zonefile table */ + const char *zTab, /* Name of zonefile table */ + i64 iFileid, /* File-id to configure key for */ + const char *zKey /* Key to store */ +){ + ZonefileKey **pp; + u32 iHash = zonefileKeyHash(zDb, zTab, iFileid); + + /* Remove any old entry */ + if( pGlobal->nHash ){ + for(pp=&pGlobal->aHash[iHash%pGlobal->nHash]; *pp; pp=&((*pp)->pHashNext)){ + ZonefileKey *pThis = *pp; + if( pThis->iFileid==iFileid + && 0==sqlite3_stricmp(zTab, pThis->zName) + && 0==sqlite3_stricmp(zDb, pThis->zDb) + ){ + pGlobal->nEntry--; + *pp = pThis->pHashNext; + sqlite3_free(pThis); + break; + } + } + } + + if( zKey ){ + int nKey = strlen(zKey); + int nDb = strlen(zDb); + int nTab = strlen(zTab); + ZonefileKey *pNew; + + /* Resize the hash-table, if necessary */ + if( pGlobal->nEntry>=pGlobal->nHash ){ + int i; + int n = pGlobal->nHash ? pGlobal->nHash*2 : 16; + ZonefileKey **a = (ZonefileKey**)sqlite3_malloc(n*sizeof(ZonefileKey*)); + if( a==0 ) return SQLITE_NOMEM; + memset(a, 0, n*sizeof(ZonefileKey*)); + for(i=0; inHash; i++){ + ZonefileKey *p; + ZonefileKey *pNext; + for(p=pGlobal->aHash[i]; p; p=pNext){ + pNext = p->pHashNext; + p->pHashNext = a[p->iHash % n]; + a[p->iHash % n] = p; + } + } + sqlite3_free(pGlobal->aHash); + pGlobal->aHash = a; + pGlobal->nHash = n; + } + + pNew = (ZonefileKey*)sqlite3_malloc( + sizeof(ZonefileKey) + nKey+1 + nDb+1 + nTab+1 + ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(ZonefileKey)); + pNew->iFileid = iFileid; + pNew->iHash = iHash; + pNew->zKey = (const char*)&pNew[1]; + pNew->nKey = nKey; + pNew->zDb = &pNew->zKey[nKey+1]; + pNew->zName = &pNew->zDb[nDb+1]; + memcpy((char*)pNew->zKey, zKey, nKey+1); + memcpy((char*)pNew->zDb, zDb, nDb+1); + memcpy((char*)pNew->zName, zTab, nTab+1); + + pNew->pHashNext = pGlobal->aHash[iHash % pGlobal->nHash]; + pGlobal->aHash[iHash % pGlobal->nHash] = pNew; + } + + return SQLITE_OK; +} + +/* +** Search the key-store passed as the first argument for an encryption +** key to use with the file with file-id iFileid in zonefile table zTab +** in database zDb. If successful, set (*pzKey) to point to the key +** buffer and return the size of the key in bytes. +** +** If no key is found, return 0. The final value of (*pzKey) is undefined +** in this case. +*/ +static int zonefileKeyFind( + ZonefileGlobal *pGlobal, + const char *zDb, /* Database containing zonefile table */ + const char *zTab, /* Name of zonefile table */ + i64 iFileid, /* File-id to configure key for */ + const char **pzKey /* OUT: Pointer to key buffer */ +){ + if( pGlobal->nHash ){ + ZonefileKey *pKey; + u32 iHash = zonefileKeyHash(zDb, zTab, iFileid); + for(pKey=pGlobal->aHash[iHash%pGlobal->nHash]; pKey; pKey=pKey->pHashNext){ + if( pKey->iFileid==iFileid + && 0==sqlite3_stricmp(zDb, pKey->zDb) + && 0==sqlite3_stricmp(zTab, pKey->zName) + ){ + *pzKey = pKey->zKey; + return pKey->nKey; + } + } + } + + return 0; +} + +/* +** The pointer passed as the only argument must actually point to a +** ZonefileGlobal structure. This function frees the structure and all +** of its components. +*/ +static void zonefileKeyDestroy(void *p){ + ZonefileGlobal *pGlobal = (ZonefileGlobal*)p; + int i; + for(i=0; inHash; i++){ + ZonefileKey *pKey; + ZonefileKey *pNext; + for(pKey=pGlobal->aHash[i]; pKey; pKey=pNext){ + pNext = pKey->pHashNext; + sqlite3_free(pKey); + } + } + sqlite3_free(pGlobal->aHash); + sqlite3_free(pGlobal); +} + #define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024) #define ZONEFILE_DEFAULT_ENCRYPTION 1 @@ -178,10 +336,6 @@ static u32 zonefileGet32(const u8 *aBuf){ + (((u32)aBuf[3]) << 0); } -#include -#include -#include - static int zfGenericOpen(void **pp, u8 *aDict, int nDict){ *pp = 0; return SQLITE_OK; @@ -1644,50 +1798,6 @@ static int zonefilePopulateIndex( return rc; } -static int zonefileStoreKey( - ZonefileFilesTab *pTab, - i64 iFileid, - const char *zKey -){ - ZonefileKey **pp; - - for(pp=&pTab->pGlobal->pList; *pp; pp=&((*pp)->pNext)){ - ZonefileKey *pThis = *pp; - if( pThis->iFileid==iFileid - && 0==sqlite3_stricmp(pTab->zBase, pThis->zName) - && 0==sqlite3_stricmp(pTab->zDb, pThis->zDb) - ){ - *pp = pThis->pNext; - sqlite3_free(pThis); - break; - } - } - - if( zKey ){ - int nKey = strlen(zKey); - int nDb = strlen(pTab->zDb); - int nName = strlen(pTab->zBase); - ZonefileKey *pNew; - pNew = (ZonefileKey*)sqlite3_malloc( - sizeof(ZonefileKey) + nKey+1 + nDb+1 + nName+1 - ); - if( pNew==0 ) return SQLITE_NOMEM; - memset(pNew, 0, sizeof(ZonefileKey)); - pNew->iFileid = iFileid; - pNew->zKey = (const char*)&pNew[1]; - pNew->nKey = nKey; - pNew->zDb = &pNew->zKey[nKey+1]; - pNew->zName = &pNew->zDb[nDb+1]; - memcpy((char*)pNew->zKey, zKey, nKey+1); - memcpy((char*)pNew->zDb, pTab->zDb, nDb+1); - memcpy((char*)pNew->zName, pTab->zBase, nName+1); - - pNew->pNext = pTab->pGlobal->pList; - pTab->pGlobal->pList = pNew; - } - return SQLITE_OK; -} - /* ** zonefile_files virtual table module xUpdate method. ** @@ -1712,7 +1822,9 @@ static int zffUpdate( if( nVal>1 && sqlite3_value_nochange(apVal[2]) ){ const char *zKey = (const char*)sqlite3_value_text(apVal[3]); i64 iFileid = sqlite3_value_int64(apVal[0]); - return zonefileStoreKey(pTab, iFileid, zKey); + return zonefileKeyStore( + pTab->pGlobal, pTab->zDb, pTab->zBase, iFileid, zKey + ); }else{ if( pTab->pDelete==0 ){ rc = zonefilePrepare(pTab->db, &pTab->pDelete, &pVtab->zErrMsg, @@ -1766,7 +1878,7 @@ static int zffUpdate( if( rc==SQLITE_OK ){ const char *zKey = (const char*)sqlite3_value_text(apVal[3]); - rc = zonefileStoreKey(pTab, iFileid, zKey); + rc = zonefileKeyStore(pTab->pGlobal, pTab->zDb, pTab->zBase,iFileid,zKey); } } @@ -2123,22 +2235,6 @@ static int zonefileCtxUncompress( return rc; } -static int zonefileFindKey(ZonefileTab *pTab, i64 iFileid, const char **pzKey){ - ZonefileKey *pKey; - - for(pKey=pTab->pGlobal->pList; pKey; pKey=pKey->pNext){ - if( pKey->iFileid==iFileid - && 0==sqlite3_stricmp(pTab->zName, pKey->zName) - && 0==sqlite3_stricmp(pTab->zDb, pKey->zDb) - ){ - *pzKey = pKey->zKey; - return pKey->nKey; - } - } - - return 0; -} - static int zonefileGetFile( sqlite3_context *pCtx, /* Leave error message here */ ZonefileCsr *pCsr, /* Cursor object */ @@ -2228,13 +2324,14 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ /* Find the encryption method and key. */ if( hdr.encryptionType ){ - const char *z= 0; - int nKey = zonefileFindKey(pTab, sqlite3_column_int64(pCsr->pSelect,1), &z); - if( nKey==0 ){ + i64 iFileid = sqlite3_column_int64(pCsr->pSelect, 1); + const char *z = 0; + int n = zonefileKeyFind(pTab->pGlobal, pTab->zDb, pTab->zName, iFileid, &z); + if( n==0 ){ zErr = sqlite3_mprintf("missing encryption key for file \"%s\"", zFile); rc = SQLITE_ERROR; }else{ - rc = zonefileCodecCreate(hdr.encryptionType,(u8*)z, nKey, &pCodec, &zErr); + rc = zonefileCodecCreate(hdr.encryptionType, (u8*)z, n, &pCodec, &zErr); } } @@ -2338,17 +2435,6 @@ static int zonefileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ return SQLITE_OK; } -static void zonefileGlobalFree(void *p){ - ZonefileGlobal *pGlobal = (ZonefileGlobal*)p; - ZonefileKey *pKey; - ZonefileKey *pNext; - for(pKey=pGlobal->pList; pKey; pKey=pNext){ - pNext = pKey->pNext; - sqlite3_free(pKey); - } - sqlite3_free(pGlobal); -} - /* ** Register the "zonefile" extensions. */ @@ -2438,7 +2524,7 @@ static int zonefileRegister(sqlite3 *db){ } if( rc==SQLITE_OK ){ rc = sqlite3_create_module_v2(db, "zonefile", &zonefileModule, - (void*)pGlobal, zonefileGlobalFree + (void*)pGlobal, zonefileKeyDestroy ); pGlobal = 0; } diff --git a/ext/zonefile/zonefileenc.test b/ext/zonefile/zonefileenc.test new file mode 100644 index 0000000000..b8d74a8c22 --- /dev/null +++ b/ext/zonefile/zonefileenc.test @@ -0,0 +1,92 @@ +# 2018 Feb 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the zonefile extension. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join $testdir tester.tcl] +set testprefix zonefileenc +load_static_extension db zonefile + +set K { + braking bramble brambles brambly + bran branch branched branches + branching branchings brand branded +} + +set nFile 100 + +do_execsql_test 1.0 { + CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB); + CREATE TABLE rr(k INTEGER PRIMARY KEY, v); +} +do_test 1.1 { + for {set i 0} {$i < $nFile} {incr i} { + set k [lindex $K [expr $i % [llength $K]]] + execsql { + DELETE FROM zz; + INSERT INTO zz VALUES($i*10+1, 1, -1, randomblob(100)); + INSERT INTO zz VALUES($i*10+2, 2, -1, randomblob(100)); + INSERT INTO zz VALUES($i*10+3, 1, -1, randomblob(100)); + INSERT INTO rr SELECT k,v FROM zz; + + WITH p(n,v) AS ( + VALUES('encryptionType', 'xor') UNION ALL + VALUES('encryptionKey', $k) + ) + SELECT zonefile_write('test' || $i || '.zonefile', 'zz', + json_group_object(n, v) + ) FROM p; + } + } +} {} + +do_test 1.2 { + execsql { + CREATE VIRTUAL TABLE gg USING zonefile; + } + for {set i 0} {$i < $nFile} {incr i} { + execsql { + INSERT INTO gg_files(filename) VALUES('test' || $i || '.zonefile') + } + } +} {} + +do_catchsql_test 1.3 { + SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v; +} {1 {missing encryption key for file "test0.zonefile"}} +do_execsql_test 1.4 { + UPDATE gg_files SET ekey = 'braking' WHERE filename='test0.zonefile'; +} +do_catchsql_test 1.5 { + SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v; +} {1 {missing encryption key for file "test1.zonefile"}} + +proc k {i} { + lindex $::K [expr $i % [llength $::K]] +} +db func k k +do_execsql_test 1.6 { + UPDATE gg_files SET ekey = k(rowid-1); +} +do_execsql_test 1.7 { + SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v; +} {0} +do_execsql_test 1.8 { + SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v==gg.v; +} {300} + + +finish_test + diff --git a/manifest b/manifest index 7b59cd6d17..d667fb70b0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\szonefile,\schange\sthe\s"file\sTEXT"\scolumn\sback\sto\s"fileid\sINTEGER".\sThe\nfileid\scan\sbe\sused\sas\sa\skey\swith\sthe\sassociated\szonefile_files\stable,\swhich\ncontains\smore\sinformation\sthan\sjust\sthe\sfilename. -D 2018-02-21T10:43:19.310 +C Have\szonefile\sstore\sencryption\skeys\sin\sa\shash-table\sinstead\sof\sa\slinked\slist.\nAdd\sextra\stests\sfor\skey\smanagement. +D 2018-02-21T16:36:08.835 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -409,8 +409,9 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F ext/zonefile/README.md df86ef5b4f9aa8b07e1c8124b3f2dcea616927385aad59d525b784f0a06d446c -F ext/zonefile/zonefile.c 36ee984120fcfecab65d1b0f54cc6e7a440d71b6847e5d042c9c3acebaec1362 +F ext/zonefile/zonefile.c cd19e711853a13c16c0926b61f670b041c3a19926493e2b872c8ef1fa97991f4 F ext/zonefile/zonefile1.test 4cd9fa8d333c195f59792c4d5c1e8387e778e46354580fbef0b84efc932c6a47 +F ext/zonefile/zonefileenc.test 5cc89a1e716b127a5220f03162ced3bd9d7df8819fe04a87f2d962315e8ebdc1 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1708,7 +1709,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a13b2b38137025d04bbfc1b12f0d0563dcf7e0ab1d4b06ffc7cacf994f00e188 -R 6d843fda370cef6d5f337c17aede902e +P 38d23888cf5a7117c51bd9211bd93ec52a30360f7eb2bc83a13910c5d85fe739 +R 60f734b7b910cc2f794eaa1390ac35ea U dan -Z bc3200ce14e90f9591a4b7905d759f8a +Z 89722e52aeff8e71e402a249ff14e24d diff --git a/manifest.uuid b/manifest.uuid index fc4fe856fe..4d8d6ebee5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -38d23888cf5a7117c51bd9211bd93ec52a30360f7eb2bc83a13910c5d85fe739 \ No newline at end of file +3a63ea652546a4c63eccd72665becff38a97a0e39d2f11703cb6899451570fd4 \ No newline at end of file