From: dan Date: Tue, 20 Feb 2018 18:47:24 +0000 (+0000) Subject: Instead of just the frame number, store frame sizes and offsets in zonefile X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b0e7fff4929e057f1c17b5f9ead036433f1d4ec8;p=thirdparty%2Fsqlite.git Instead of just the frame number, store frame sizes and offsets in zonefile shadow table %_shadow_idx. FossilOrigin-Name: 56801c461c5d19cf96146fe0fa7f725c81da5cd7495c9608cd044125d00fecfe --- diff --git a/ext/zonefile/zonefile.c b/ext/zonefile/zonefile.c index 615f658672..e31abe0cc6 100644 --- a/ext/zonefile/zonefile.c +++ b/ext/zonefile/zonefile.c @@ -503,9 +503,7 @@ static ZonefileCompress *zonefileCompressByValue(int eType){ "CREATE TABLE z1(" \ " k INTEGER PRIMARY KEY," \ " v BLOB," \ - " fileid INTEGER," \ - " frame INTEGER," \ - " ofst INTEGER," \ + " file TEXT," \ " sz INTEGER" \ ")" @@ -1598,20 +1596,40 @@ static int zonefilePopulateIndex( if( rc==SQLITE_OK && pTab->pInsertIdx==0 ){ rc = zonefilePrepare(pTab->db, &pTab->pInsertIdx, &pTab->base.zErrMsg, - "INSERT INTO %Q.'%q_shadow_idx'(k, fileid, frame, ofst, sz)" - "VALUES(?,?,?,?,?)", + "INSERT INTO %Q.'%q_shadow_idx'(k, fileid, fofst, fsz, ofst, sz)" + "VALUES(?,?,?,?,?,?)", pTab->zDb, pTab->zBase ); } for(i=0; i0 ){ + iFrameOff = zonefileGet32(&aKey[(iFrame-1)*4]); + szFrame -= iFrameOff; + } + iFrameOff += hdr.byteOffsetFrames; + iOff = (i64)zonefileGet32(&aEntry[12]); + sz = (int)zonefileGet32(&aEntry[16]); sqlite3_bind_int64(pTab->pInsertIdx, 1, (i64)zonefileGet64(&aEntry[0])); sqlite3_bind_int64(pTab->pInsertIdx, 2, iFileid); - sqlite3_bind_int64(pTab->pInsertIdx, 3, (i64)zonefileGet32(&aEntry[8])); - sqlite3_bind_int64(pTab->pInsertIdx, 4, (i64)zonefileGet32(&aEntry[12])); - sqlite3_bind_int64(pTab->pInsertIdx, 5, (i64)zonefileGet32(&aEntry[16])); + if( hdr.encryptionType || hdr.compressionTypeContent ){ + sqlite3_bind_int64(pTab->pInsertIdx, 5, iOff); + sqlite3_bind_int(pTab->pInsertIdx, 6, sz); + }else{ + iFrameOff += iOff; + szFrame = sz; + } + sqlite3_bind_int64(pTab->pInsertIdx, 3, iFrameOff); + sqlite3_bind_int(pTab->pInsertIdx, 4, szFrame); sqlite3_step(pTab->pInsertIdx); rc = sqlite3_reset(pTab->pInsertIdx); @@ -1808,10 +1826,11 @@ static int zonefileCreateConnect( if( bCreate ){ char *zSql = sqlite3_mprintf( - "CREATE TABLE %Q.'%q_shadow_idx'(" + "CREATE TABLE %Q.'%q_shadow_idx'(" " k INTEGER PRIMARY KEY," " fileid INTEGER," - " frame INTEGER," + " fofst INTEGER," + " fsz INTEGER," " ofst INTEGER," " sz INTEGER" ");" @@ -1962,6 +1981,7 @@ static int zonefileDestroy(sqlite3_vtab *pVtab){ char *zSql = sqlite3_mprintf( "DROP TABLE IF EXISTS %Q.'%q_shadow_idx';" "DROP TABLE IF EXISTS %Q.'%q_shadow_file';" + "DROP TABLE IF EXISTS %Q.'%q_shadow_frame';" "DROP TABLE IF EXISTS %Q.'%q_files';", pTab->zDb, pTab->zName, pTab->zDb, pTab->zName, pTab->zDb, pTab->zName ); @@ -2053,7 +2073,7 @@ static int zonefileFilter( } rc = zonefilePrepare(pTab->db, &pCsr->pSelect, &pTab->base.zErrMsg, - "SELECT k, fileid, frame, ofst, sz FROM %Q.'%q_shadow_idx'%s%s%s%s", + "SELECT k, fileid, fofst, fsz, ofst, sz FROM %Q.'%q_shadow_idx'%s%s%s%s", pTab->zDb, pTab->zName, z1 ? " WHERE " : "", z1, z2 ? " AND " : "", z2 @@ -2119,21 +2139,14 @@ static int zonefileFindKey(ZonefileTab *pTab, i64 iFileid, const char **pzKey){ return 0; } -static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ +static int zonefileGetFile( + sqlite3_context *pCtx, /* Leave error message here */ + ZonefileCsr *pCsr, /* Cursor object */ + const char **pzFile /* OUT: Pointer to current file */ +){ ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab; - const char *zFile = 0; - char *zErr = 0; - FILE *pFd = 0; int rc = SQLITE_OK; - u32 iOff = 0; /* Offset of frame in file */ - u32 szFrame = 0; /* Size of frame in bytes */ - int iKeyOff = 0; /* Offset of record within frame */ - int szKey = 0; /* Uncompressed size of record in bytes */ - i64 iFileid = 0; - ZonefileHeader hdr; - ZonefileCompress *pCmpMethod = 0; - ZonefileCodec *pCodec = 0; - void *pCmp = 0; + i64 iFileid; if( pTab->pIdToName==0 ){ rc = zonefilePrepare(pTab->db, &pTab->pIdToName, &pTab->base.zErrMsg, @@ -2146,25 +2159,43 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ } } - iKeyOff = sqlite3_column_int(pCsr->pSelect, 3); - szKey = sqlite3_column_int(pCsr->pSelect, 4); - - /* Open the file to read the blob from */ iFileid = sqlite3_column_int64(pCsr->pSelect,1); sqlite3_bind_int64(pTab->pIdToName, 1, iFileid); if( SQLITE_ROW==sqlite3_step(pTab->pIdToName) ){ - zFile = (const char*)sqlite3_column_text(pTab->pIdToName, 0); - pFd = zonefileFileOpen(zFile, 0, &zErr); - } - if( zFile==0 ){ + *pzFile = (const char*)sqlite3_column_text(pTab->pIdToName, 0); + }else{ rc = sqlite3_reset(pTab->pIdToName); - if( rc!=SQLITE_OK ){ - zonefileTransferError(pCtx); - }else{ + if( rc==SQLITE_OK ){ rc = SQLITE_CORRUPT_VTAB; + }else{ + zonefileTransferError(pCtx); } - }else if( pFd==0 ){ - rc = SQLITE_ERROR; + } + + return rc; +} + +static void zonefileReleaseFile(ZonefileCsr *pCsr){ + ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab; + sqlite3_reset(pTab->pIdToName); +} + +static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ + ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab; + const char *zFile = 0; + char *zErr = 0; + FILE *pFd = 0; + int rc = SQLITE_OK; + ZonefileHeader hdr; + ZonefileCompress *pCmpMethod = 0; + ZonefileCodec *pCodec = 0; + void *pCmp = 0; + + /* Open the file to read the blob from */ + rc = zonefileGetFile(pCtx, pCsr, &zFile); + if( rc==SQLITE_OK ){ + pFd = zonefileFileOpen(zFile, 0, &zErr); + if( pFd==0 ) rc = SQLITE_ERROR; } /* Read the zonefile header */ @@ -2172,7 +2203,7 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr); } - /* Find the compression method and open the compressor instance */ + /* Find the compression method and open the compressor handle. */ if( rc==SQLITE_OK ){ rc = zfFindCompress(hdr.compressionTypeContent, &pCmpMethod, &zErr); } @@ -2195,88 +2226,64 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ sqlite3_free(aDict); } + /* Find the encryption method and key. */ if( hdr.encryptionType ){ - const char *zKey = 0; - int nKey = zonefileFindKey(pTab, iFileid, &zKey); + const char *z= 0; + int nKey = zonefileFindKey(pTab, sqlite3_column_int64(pCsr->pSelect,1), &z); if( nKey==0 ){ zErr = sqlite3_mprintf("missing encryption key for file \"%s\"", zFile); rc = SQLITE_ERROR; }else{ - rc = zonefileCodecCreate(hdr.encryptionType,(u8*)zKey,nKey,&pCodec,&zErr); - } - } - - /* Find the offset (iOff) and size (szFrame) of the frame that the - ** record is stored in. */ - if( rc==SQLITE_OK ){ - int iFrame = sqlite3_column_int(pCsr->pSelect, 2); - u8 aSpace[8] = {0,0,0,0,0,0,0,0}; - u8 *aOff = aSpace; - u8 *aFree = 0; - if( hdr.compressionTypeIndexData ){ - int nFree = 0; - rc = zonefileLoadIndex(&hdr, pFd, &aFree, &nFree, &zErr); - if( rc==SQLITE_OK ) aOff = &aFree[4*(iFrame-1)]; - }else{ - rc = zonefileFileRead(pFd, aOff, 8, - ZONEFILE_SZ_HEADER + hdr.extendedHeaderSize + 4 * (iFrame-1) - ); + rc = zonefileCodecCreate(hdr.encryptionType,(u8*)z, nKey, &pCodec, &zErr); } - szFrame = zonefileGet32(&aOff[4]); - if( iFrame>0 ){ - iOff = zonefileGet32(aOff); - szFrame = szFrame - iOff; - } - sqlite3_free(aFree); } /* Read some data into memory. If the data is uncompressed, then just ** the required record is read. Otherwise, the entire frame is read ** into memory. */ if( rc==SQLITE_OK ){ - int nRead; /* Bytes of data to read */ - i64 iRead; /* Offset to read at */ + i64 iFrameOff = sqlite3_column_int64(pCsr->pSelect, 2); + int szFrame = sqlite3_column_int(pCsr->pSelect, 3); + u8 *aBuf = sqlite3_malloc(szFrame); - if( pCmpMethod || pCodec ){ - nRead = szFrame; - iRead = iOff; - }else{ - nRead = szKey; - iRead = iOff + iKeyOff; - } - - u8 *aBuf = sqlite3_malloc(nRead); if( aBuf==0 ){ rc = SQLITE_NOMEM; }else{ - rc = zonefileFileRead(pFd, aBuf, nRead, hdr.byteOffsetFrames + iRead); - if( rc==SQLITE_OK ){ + rc = zonefileFileRead(pFd, aBuf, szFrame, iFrameOff); + if( rc!=SQLITE_OK ){ + zErr = sqlite3_mprintf( + "failed to read %d bytes at offset %d from file \"%s\"", + szFrame, iFrameOff, zFile + ); + } + } + + if( rc==SQLITE_OK ){ + if( pCodec==0 && pCmpMethod==0 ){ + sqlite3_result_blob(pCtx, aBuf, szFrame, zonefileFree); + aBuf = 0; + }else{ + i64 iOff = sqlite3_column_int64(pCsr->pSelect, 4); + int sz = sqlite3_column_int(pCsr->pSelect, 5); + + /* Decrypt the data if necessary */ if( pCodec ){ - zonefileCodecDecode(pCodec, aBuf, nRead); + zonefileCodecDecode(pCodec, aBuf, szFrame); + szFrame -= zonefileCodecNonceSize(pCodec); } - if( pCmpMethod==0 ){ - if( pCodec==0 ){ - sqlite3_result_blob(pCtx, aBuf, szKey, zonefileFree); - aBuf = 0; - }else{ - sqlite3_result_blob(pCtx, &aBuf[iKeyOff], szKey, SQLITE_TRANSIENT); - } - }else{ + if( pCmpMethod ){ rc = zonefileCtxUncompress( - pCtx, pCmpMethod, pCmp, aBuf, szFrame, iKeyOff, szKey + pCtx, pCmpMethod, pCmp, aBuf, szFrame, iOff, sz ); + }else{ + sqlite3_result_blob(pCtx, &aBuf[iOff], sz, SQLITE_TRANSIENT); } - }else{ - zErr = sqlite3_mprintf( - "failed to read %d bytes at offset %d from file \"%s\"", - nRead, (int)(hdr.byteOffsetFrames+iRead), zFile - ); } - sqlite3_free(aBuf); } + sqlite3_free(aBuf); } - sqlite3_reset(pTab->pIdToName); + zonefileReleaseFile(pCsr); if( zErr ){ assert( rc!=SQLITE_OK ); sqlite3_result_error(pCtx, zErr, -1); @@ -2305,18 +2312,19 @@ static int zonefileColumn( case 1: /* v */ rc = zonefileGetValue(pCtx, pCsr); break; - case 2: /* fileid */ + case 2: /* file */ sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, 1)); break; - case 3: /* frame */ - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, 2)); - break; - case 4: /* ofst */ - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, 3)); - break; - default: /* sz */ - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, 4)); + default: { /* sz */ + int iCol; + if( sqlite3_column_type(pCsr->pSelect, 5)==SQLITE_NULL ){ + iCol = 3; + }else{ + iCol = 5; + } + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, iCol)); break; + } } return rc; } diff --git a/ext/zonefile/zonefile1.test b/ext/zonefile/zonefile1.test index 1bb6f850ab..ca1a06f43a 100644 --- a/ext/zonefile/zonefile1.test +++ b/ext/zonefile/zonefile1.test @@ -47,9 +47,11 @@ do_execsql_test 1.3 { do_execsql_test 1.4 { SELECT count(*) FROM z1_shadow_idx } 3 do_execsql_test 1.5.1 { SELECT k FROM z1 } {1 2 3} -do_execsql_test 1.5.2 { SELECT fileid FROM z1 } {1 1 1} -do_execsql_test 1.5.3 { SELECT frame FROM z1 } {0 0 0} +do_execsql_test 1.5.2 { SELECT file FROM z1 } { + test.zonefile test.zonefile test.zonefile +} do_execsql_test 1.5.4 { SELECT sz FROM z1 } {100 100 100} +exit do_execsql_test 1.5.5 { SELECT zz.v==z1.v FROM zz, z1 WHERE zz.k=z1.k diff --git a/manifest b/manifest index 638de471a3..5e2c7a9e30 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sinvoking\sencryption\shooks\sto\szonefile.\sAnd\smock\sencryption\nmethod\s"xor"\sfor\stesting. -D 2018-02-19T21:07:20.566 +C Instead\sof\sjust\sthe\sframe\snumber,\sstore\sframe\ssizes\sand\soffsets\sin\szonefile\nshadow\stable\s%_shadow_idx. +D 2018-02-20T18:47:24.294 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -409,8 +409,8 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F ext/zonefile/README.md 1a95a93c865e196bc6301581e5a702f449ea9ce0e307cdbdbbdfd58377f1ec7e -F ext/zonefile/zonefile.c afee047df23badc77ff088613723f08fdda8c645e6e7df60fff7d6f8487c560f -F ext/zonefile/zonefile1.test 6eb9e28b7cb5cd9dcac40b2c2b206e9763242d07f7a44e099221d1cfdcb2b30b +F ext/zonefile/zonefile.c 0a15cb181d02f5afb7ca8dd3ae939cfa765bbf11a6a44c4e3f358e978d39795e +F ext/zonefile/zonefile1.test 12e32e0664338fe5002d78036f057c46cf33214d73550c6e820e90e946a90f52 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 78267a091307e2c29a4fb1606fa9c79939fe010b801749614f4c48dc8715810e -R c6e7b11c5fa6bd69655eefd536ffcebb +P 55cf920c5a13473d04f0cb885117c04b2bc054bfed6ee549be84cb9485c104d2 +R 032579fc2be331b3506eb489f2f4da6b U dan -Z 818fddc92c78d4ee6aea120c041035ad +Z 6da1c9e76b52b238efa7bf0ed02b77c3 diff --git a/manifest.uuid b/manifest.uuid index 4faea7be78..982449f30b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55cf920c5a13473d04f0cb885117c04b2bc054bfed6ee549be84cb9485c104d2 \ No newline at end of file +56801c461c5d19cf96146fe0fa7f725c81da5cd7495c9608cd044125d00fecfe \ No newline at end of file