From: dan Date: Sat, 10 Feb 2018 17:41:01 +0000 (+0000) Subject: Add the start of the "zonefile" extension. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=087529aae46db4b85965c4e0495436f306062317;p=thirdparty%2Fsqlite.git Add the start of the "zonefile" extension. FossilOrigin-Name: c125b4c380d1a20c7d71b413e96183eca9987aed3d0ba28395aa79c7c31bb6fd --- diff --git a/ext/zonefile/README.md b/ext/zonefile/README.md new file mode 100644 index 0000000000..4f17c4e528 --- /dev/null +++ b/ext/zonefile/README.md @@ -0,0 +1,14 @@ + +Notes: + + * Using 32-bit frame numbers (not 16-bit). + + * The ZonefileHeader object is 26 bytes in size. Which means that the + ZoneFileIndex will not be 8-byte aligned. Problem? + + * The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are + relative to the offset in ZoneFileHeader.byteOffsetFrames. This is + necessary as we may not know the offset of the start of the frame data + until after the ZoneFileIndex structure is compressed. + + diff --git a/ext/zonefile/zonefile.c b/ext/zonefile/zonefile.c new file mode 100644 index 0000000000..a52b4cf626 --- /dev/null +++ b/ext/zonefile/zonefile.c @@ -0,0 +1,374 @@ +/* +** 2018-02-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#ifndef SQLITE_OMIT_VIRTUALTABLE + +#ifndef SQLITE_AMALGAMATION +typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned long u32; +#define MIN(a,b) ((a)<(b) ? (a) : (b)) + +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif +#endif /* SQLITE_AMALGAMATION */ + +#define ZONEFILE_MAGIC_NUMBER 0x464B3138 + +#define ZONEFILE_SZ_HEADER 26 + +#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024) +#define ZONEFILE_DEFAULT_ENCRYPTION 0 +#define ZONEFILE_DEFAULT_COMPRESSION 0 + + +#include +#include +#include + +typedef struct ZonefileWrite ZonefileWrite; +struct ZonefileWrite { + int compressionTypeIndexData; + int compressionTypeContent; + int encryptionType; + int maxAutoFrameSize; +}; + +typedef struct ZonefileBuffer ZonefileBuffer; +struct ZonefileBuffer { + u8 *a; + int n; + int nAlloc; +}; + +/* +** Set the error message contained in context ctx to the results of +** vprintf(zFmt, ...). +*/ +static void zonefileCtxError(sqlite3_context *ctx, const char *zFmt, ...){ + char *zMsg = 0; + va_list ap; + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + va_end(ap); +} + +static void zonefileTransferError(sqlite3_context *pCtx){ + sqlite3 *db = sqlite3_context_db_handle(pCtx); + const char *zErr = sqlite3_errmsg(db); + sqlite3_result_error(pCtx, zErr, -1); +} + +static sqlite3_stmt *zonefilePrepare( + sqlite3_context *pCtx, + const char *zFmt, + ... +){ + sqlite3_stmt *pRet = 0; + va_list ap; + char *zSql; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( zSql ){ + sqlite3 *db = sqlite3_context_db_handle(pCtx); + int rc = sqlite3_prepare(db, zSql, -1, &pRet, 0); + if( rc!=SQLITE_OK ){ + zonefileTransferError(pCtx); + } + }else{ + sqlite3_result_error_nomem(pCtx); + } + return pRet; +} + +/* +** Return zero if the two SQL values passed as arguments are equal, or +** non-zero otherwise. Values with different types are considered unequal, +** even if they both contain the same numeric value (e.g. 2 and 2.0). +*/ +static int zonefileCompareValue(sqlite3_value *p1, sqlite3_value *p2){ + int eType; + assert( p1 ); + if( p2==0 ) return 1; + eType = sqlite3_value_type(p1); + if( sqlite3_value_type(p2)!=eType ) return 1; + switch( eType ){ + case SQLITE_INTEGER: + return sqlite3_value_int64(p1)==sqlite3_value_int64(p2); + case SQLITE_FLOAT: + return sqlite3_value_double(p1)==sqlite3_value_double(p2); + case SQLITE_TEXT: + case SQLITE_BLOB: { + int n1 = sqlite3_value_bytes(p1); + int n2 = sqlite3_value_bytes(p2); + if( n1!=n2 ) return 1; + return memcmp(sqlite3_value_blob(p1), sqlite3_value_blob(p2), n1); + } + default: + assert( eType==SQLITE_NULL); + } + + return 0; +} + +int zonefileIsAutoFrame(sqlite3_value *pFrame){ + return ( + sqlite3_value_type(pFrame)==SQLITE_INTEGER + && sqlite3_value_int64(pFrame)==-1 + ); +} + +static int zonefileGetParams( + sqlite3_context *pCtx, /* Leave any error message here */ + const char *zJson, /* JSON configuration parameter (or NULL) */ + ZonefileWrite *p /* Populate this object before returning */ +){ + memset(p, 0, sizeof(ZonefileWrite)); + p->maxAutoFrameSize = ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE; + return SQLITE_OK; +} + +/* +** Check that there is room in buffer pBuf for at least nByte bytes more +** data. If not, attempt to allocate more space. If the allocation attempt +** fails, leave an error message in context pCtx and return SQLITE_ERROR. +** +** If no error occurs, SQLITE_OK is returned. +*/ +static int zonefileBufferGrow( + sqlite3_context *pCtx, + ZonefileBuffer *pBuf, + int nByte +){ + int nReq = pBuf->n + nByte; + if( nReq>pBuf->nAlloc ){ + u8 *aNew; + int nNew = pBuf->nAlloc ? pBuf->nAlloc*2 : 128; + while( nNewa, nNew); + if( aNew==0 ){ + sqlite3_result_error_nomem(pCtx); + return SQLITE_ERROR; + } + pBuf->a = aNew; + pBuf->nAlloc = nNew; + } + return SQLITE_OK; +} + +static void zonefileBufferFree(ZonefileBuffer *pBuf){ + sqlite3_free(pBuf->a); + memset(pBuf, 0, sizeof(ZonefileBuffer)); +} + +static void zonefilePut32(u8 *aBuf, u32 v){ + aBuf[0] = (v >> 24) & 0xFF; + aBuf[1] = (v >> 16) & 0xFF; + aBuf[2] = (v >> 8) & 0xFF; + aBuf[3] = v & 0xFF; +} + +static void zonefileAppend32(ZonefileBuffer *pBuf, u32 v){ + zonefilePut32(&pBuf->a[pBuf->n], v); + pBuf->n += 4; +} + +static void zonefileAppend64(ZonefileBuffer *pBuf, u64 v){ + zonefileAppend32(pBuf, v>>32); + zonefileAppend32(pBuf, v & 0xFFFFFFFF); +} + +static void zonefileAppendBlob(ZonefileBuffer *pBuf, const u8 *p, int n){ + memcpy(&pBuf->a[pBuf->n], p, n); + pBuf->n += n; +} + +static int zonefileWrite(FILE *pFd, const u8 *aBuf, int nBuf){ + size_t res = fwrite(aBuf, 1, nBuf, pFd); + return res!=nBuf ? SQLITE_ERROR : SQLITE_OK; +} + +/* +** Function: zonefile_write(F,T[,J]) +*/ +static void zonefileWriteFunc( + sqlite3_context *pCtx, /* Context object */ + int objc, /* Number of SQL arguments */ + sqlite3_value **objv /* Array of SQL arguments */ +){ + const char *zFile = 0; /* File to write to */ + const char *zTbl = 0; /* Database object to read from */ + const char *zJson = 0; /* JSON configuration parameters */ + ZonefileWrite sWrite; /* Decoded JSON parameters */ + int nKey = 0; /* Number of keys in new zonefile */ + int nFrame = 0; /* Number of frames in new zonefile */ + int szFrame = 0; /* Size of current frame */ + sqlite3_stmt *pStmt = 0; /* SQL used to read data from source table */ + FILE *pFd = 0; + int rc; + sqlite3_value *pPrev = 0; + + ZonefileBuffer sFrameIdx = {0, 0, 0}; + ZonefileBuffer sKeyIdx = {0, 0, 0}; + ZonefileBuffer sFrames = {0, 0, 0}; + u8 aHdr[ZONEFILE_SZ_HEADER]; /* Space to assemble zonefile header */ + + assert( objc==2 || objc==3 ); + zFile = (const char*)sqlite3_value_text(objv[0]); + zTbl = (const char*)sqlite3_value_text(objv[1]); + if( objc==3 ){ + zJson = (const char*)sqlite3_value_text(objv[2]); + } + if( zonefileGetParams(pCtx, zJson, &sWrite) ) return; + + /* Prepare the SQL statement used to read data from the source table. This + ** also serves to verify the suitability of the source table schema. */ + pStmt = zonefilePrepare(pCtx, + "SELECT k, frame, v FROM %Q ORDER BY frame, idx, k", zTbl + ); + if( pStmt==0 ) return; + + /* Open a file-handle used to write out the zonefile */ + pFd = fopen(zFile, "w"); + if( pFd==0 ){ + zonefileCtxError(pCtx, "error opening file \"%s\" (fopen())", zFile); + sqlite3_finalize(pStmt); + return; + } + + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + sqlite3_int64 k = sqlite3_column_int64(pStmt, 0); + sqlite3_value *pFrame = sqlite3_column_value(pStmt, 1); + int nBlob = sqlite3_column_bytes(pStmt, 2); + const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2); + + int bAuto = zonefileIsAutoFrame(pFrame); + if( zonefileCompareValue(pFrame, pPrev) + || (bAuto && szFrame && (szFrame+nBlob)>sWrite.maxAutoFrameSize) + ){ + /* Add new entry to sFrameIdx */ + szFrame = 0; + if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) ) goto zone_write_out; + zonefileAppend32(&sFrameIdx, sFrames.n); + sqlite3_value_free(pPrev); + pPrev = sqlite3_value_dup(pFrame); + if( pPrev==0 ){ + sqlite3_result_error_nomem(pCtx); + goto zone_write_out; + } + nFrame++; + } + + /* Add new entry to sKeyIdx */ + if( zonefileBufferGrow(pCtx, &sKeyIdx, 20) ) goto zone_write_out; + zonefileAppend64(&sKeyIdx, k); + zonefileAppend32(&sKeyIdx, nFrame-1); + zonefileAppend32(&sKeyIdx, szFrame); + zonefileAppend32(&sKeyIdx, nBlob); + + /* Add data for new entry to sFrames */ + if( zonefileBufferGrow(pCtx, &sFrames, nBlob) ) goto zone_write_out; + zonefileAppendBlob(&sFrames, pBlob, nBlob); + szFrame += nBlob; + nKey++; + } + + /* Create the zonefile header in the in-memory buffer */ + zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER); + aHdr[4] = sWrite.compressionTypeIndexData; + aHdr[5] = sWrite.compressionTypeContent; + zonefilePut32(&aHdr[6], 0); /* Compression dictionary byte offset */ + zonefilePut32(&aHdr[10], ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n); + zonefilePut32(&aHdr[14], nFrame); + zonefilePut32(&aHdr[18], nKey); + aHdr[22] = sWrite.encryptionType; + aHdr[23] = 0; /* Encryption key index */ + aHdr[24] = 0; /* extended header version */ + aHdr[25] = 0; /* extended header size */ + assert( ZONEFILE_SZ_HEADER==26 ); + + rc = zonefileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER); + if( rc==SQLITE_OK ) rc = zonefileWrite(pFd, sFrameIdx.a, sFrameIdx.n); + if( rc==SQLITE_OK ) rc = zonefileWrite(pFd, sKeyIdx.a, sKeyIdx.n); + if( rc==SQLITE_OK ) rc = zonefileWrite(pFd, sFrames.a, sFrames.n); + if( rc ){ + zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile); + goto zone_write_out; + } + + if( fclose(pFd) ){ + zonefileCtxError(pCtx, "error writing file \"%s\" (fclose())", zFile); + } + pFd = 0; + + zone_write_out: + if( pFd ) fclose(pFd); + sqlite3_finalize(pStmt); + zonefileBufferFree(&sFrameIdx); + zonefileBufferFree(&sKeyIdx); + zonefileBufferFree(&sFrames); +} + +/* +** Register the "zonefile" extensions. +*/ +static int zonefileRegister(sqlite3 *db){ + struct Func { + const char *z; + int n; + void (*x)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "zonefile_write", 2, zonefileWriteFunc }, + { "zonefile_write", 3, zonefileWriteFunc }, + }; + int rc = SQLITE_OK; + int i; + + for(i=0; rc==SQLITE_OK && iz, p->n, SQLITE_ANY, 0, p->x, 0, 0); + } + + return rc; +} + +#else /* SQLITE_OMIT_VIRTUALTABLE */ +# define zonefileRegister(x) SQLITE_OK +#endif + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_zonefile_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + return zonefileRegister(db); +} diff --git a/ext/zonefile/zonefile1.test b/ext/zonefile/zonefile1.test new file mode 100644 index 0000000000..9aae50c2ee --- /dev/null +++ b/ext/zonefile/zonefile1.test @@ -0,0 +1,34 @@ +# 2018 Feb 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the zonefile extension. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join $testdir tester.tcl] +set testprefix zonefile1 +load_static_extension db zonefile + + +do_execsql_test 1.0 { + CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB); + INSERT INTO zz VALUES(1, -1, -1, randomblob(100)); + INSERT INTO zz VALUES(2, -1, -1, randomblob(100)); + INSERT INTO zz VALUES(3, -1, -1, randomblob(100)); +} + +do_execsql_test 1.1 { + SELECT zonefile_write('test.zonefile', 'zz'); +} + +finish_test diff --git a/main.mk b/main.mk index 2eddfc6f27..59072e01cd 100644 --- a/main.mk +++ b/main.mk @@ -372,6 +372,7 @@ TESTSRC += \ $(TOP)/ext/misc/wholenumber.c \ $(TOP)/ext/misc/vfslog.c \ $(TOP)/ext/misc/zipfile.c \ + $(TOP)/ext/zonefile/zonefile.c \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ $(TOP)/ext/fts5/fts5_test_tok.c diff --git a/manifest b/manifest index bb81d9a18e..2f28794302 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smisplaced\stestcase()\smacros\sfrom\sthe\sprevious\scheck-in. -D 2018-02-10T02:31:30.872 +C Add\sthe\sstart\sof\sthe\s"zonefile"\sextension. +D 2018-02-10T17:41:01.882 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -408,10 +408,13 @@ F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f +F ext/zonefile/README.md d675dbdffe2cd6d54681b958a13c88433a5022d39ab9ef3d4f487a272603f902 +F ext/zonefile/zonefile.c da7c63bfc0332e61cff28712ff75b08164c3a5eceef6273cac82f1fe2703b1ce +F ext/zonefile/zonefile1.test ba4ca1166bb39a4121e6e7b3ef40a57cdff47a7432c816ab90ca61c02d9648d3 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk c8c473bd91d553acab3fb0608ddb69fc769c7bcf6d9e258800504bfda86c792b +F main.mk 0efdf6ac125eb810494d72dd7016ab07ddf3e3fa8615a14aef8491b69818fbe4 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -498,7 +501,7 @@ F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 F src/tclsqlite.c 11a2618c227fd13ccad73ee02d1199f9880c59db2b3144fd7432db1980a2577d -F src/test1.c 1ab7cbbb6693e08364c1a9241e2aee17f8c4925e4cc52396be77ae6845a05828 +F src/test1.c 6b0dfd4f0a1996805bc04381b45df71ead9ed84634d2a47a757ab292febf9026 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 @@ -1705,7 +1708,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928 -R ec6c39c5f311f4ee60e0b56f01a82df2 -U drh -Z 0f7daa30b8fefb9cfc3cc8e0bb044dd7 +P 3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984 +R 49b6c60b83f70d70fd5fce453b29eb31 +T *branch * zonefile +T *sym-zonefile * +T -sym-trunk * +U dan +Z c26dc1aef1bc29dda740f0f5df0c6eab diff --git a/manifest.uuid b/manifest.uuid index 611a8eb076..0991015f78 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984 \ No newline at end of file +c125b4c380d1a20c7d71b413e96183eca9987aed3d0ba28395aa79c7c31bb6fd \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index bc8f389dbd..c76f0008da 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6992,6 +6992,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd( #ifdef SQLITE_HAVE_ZLIB extern int sqlite3_zipfile_init(sqlite3*,char**,const sqlite3_api_routines*); #endif + extern int sqlite3_zonefile_init(sqlite3*,char**,const sqlite3_api_routines*); static const struct { const char *zExtName; int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*); @@ -7016,6 +7017,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd( #ifdef SQLITE_HAVE_ZLIB { "zipfile", sqlite3_zipfile_init }, #endif + { "zonefile", sqlite3_zonefile_init }, }; sqlite3 *db; const char *zName;