From: dan Date: Fri, 29 Dec 2017 20:19:03 +0000 (+0000) Subject: Update ext/misc/zipfile.c to support creating and adding entries to existing X-Git-Tag: version-3.22.0~108^2~17 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=373dc3bb3f00c19e84fab3a997194c7316f4f0e5;p=thirdparty%2Fsqlite.git Update ext/misc/zipfile.c to support creating and adding entries to existing zip archives. FossilOrigin-Name: 2dec2dec592c0726ebe87b841b9c8d493dea7074a99f278eb1bf0b744d658a9d --- diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 188d4fc2ea..5824965032 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -26,6 +26,8 @@ SQLITE_EXTENSION_INIT1 #include #include +#include + #ifndef SQLITE_OMIT_VIRTUALTABLE #ifndef SQLITE_AMALGAMATION @@ -47,11 +49,40 @@ typedef unsigned long u32; ");" #define ZIPFILE_F_COLUMN_IDX 6 /* Index of column "f" in the above */ - #define ZIPFILE_BUFFER_SIZE (64*1024) -#define ZIPFILE_EXTRA_TIMESTAMP 0x5455 +/* +** Magic numbers used to read and write zip files. +** +** ZIPFILE_NEWENTRY_MADEBY: +** Use this value for the "version-made-by" field in new zip file +** entries. The upper byte indicates "unix", and the lower byte +** indicates that the zip file matches pkzip specification 3.0. +** This is what info-zip seems to do. +** +** ZIPFILE_NEWENTRY_REQUIRED: +** Value for "version-required-to-extract" field of new entries. +** Version 2.0 is required to support folders and deflate compression. +** +** ZIPFILE_NEWENTRY_FLAGS: +** Value for "general-purpose-bit-flags" field of new entries. Bit +** 11 means "utf-8 filename and comment". +** +** ZIPFILE_SIGNATURE_CDS: +** First 4 bytes of a valid CDS record. +** +** ZIPFILE_SIGNATURE_LFH: +** First 4 bytes of a valid LFH record. +*/ +#define ZIPFILE_EXTRA_TIMESTAMP 0x5455 +#define ZIPFILE_NEWENTRY_MADEBY ((3<<8) + 30) +#define ZIPFILE_NEWENTRY_REQUIRED 20 +#define ZIPFILE_NEWENTRY_FLAGS 0x800 +#define ZIPFILE_SIGNATURE_CDS 0x02014b50 +#define ZIPFILE_SIGNATURE_LFH 0x04034b50 +#define ZIPFILE_SIGNATURE_EOCD 0x06054b50 +#define ZIPFILE_LFH_FIXED_SZ 30 /* ** Set the error message contained in context ctx to the results of @@ -188,18 +219,58 @@ struct ZipfileCsr { i64 iDataOff; /* Offset in zipfile to data */ u32 mTime; /* Extended mtime value */ int flags; - u8 *aBuffer; /* Buffer used for various tasks */ }; +/* +** Values for ZipfileCsr.flags. +*/ #define ZIPFILE_MTIME_VALID 0x0001 +typedef struct ZipfileEntry ZipfileEntry; +struct ZipfileEntry { + char *zPath; /* Path of zipfile entry */ + u8 *aCdsEntry; /* Buffer containing entire CDS entry */ + int nCdsEntry; /* Size of buffer aCdsEntry[] in bytes */ + ZipfileEntry *pNext; +}; + typedef struct ZipfileTab ZipfileTab; struct ZipfileTab { sqlite3_vtab base; /* Base class - must be first */ + char *zFile; /* Zip file this table accesses (may be NULL) */ + u8 *aBuffer; /* Temporary buffer used for various tasks */ + + /* The following are used by write transactions only */ + ZipfileEntry *pEntry; /* Linked list of all files (if pWriteFd!=0) */ + FILE *pWriteFd; /* File handle open on zip archive */ + i64 szCurrent; /* Current size of zip archive */ + i64 szOrig; /* Size of archive at start of transaction */ }; +static void zipfileDequote(char *zIn){ + char q = zIn[0]; + if( q=='"' || q=='\'' || q=='`' || q=='[' ){ + char c; + int iIn = 1; + int iOut = 0; + if( q=='[' ) q = ']'; + while( (c = zIn[iIn++]) ){ + if( c==q ){ + if( zIn[iIn++]!=q ) break; + } + zIn[iOut++] = c; + } + zIn[iOut] = '\0'; + } +} + /* ** Construct a new ZipfileTab virtual table object. +** +** argv[0] -> module name ("zipfile") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. */ static int zipfileConnect( sqlite3 *db, @@ -208,14 +279,28 @@ static int zipfileConnect( sqlite3_vtab **ppVtab, char **pzErr ){ + int nByte = sizeof(ZipfileTab) + ZIPFILE_BUFFER_SIZE; + int nFile = 0; + const char *zFile = 0; ZipfileTab *pNew = 0; int rc; + if( argc>3 ){ + zFile = argv[3]; + nFile = strlen(zFile)+1; + } + rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA); if( rc==SQLITE_OK ){ - pNew = (ZipfileTab*)sqlite3_malloc( sizeof(*pNew) ); + pNew = (ZipfileTab*)sqlite3_malloc(nByte+nFile); if( pNew==0 ) return SQLITE_NOMEM; - memset(pNew, 0, sizeof(*pNew)); + memset(pNew, 0, nByte+nFile); + pNew->aBuffer = (u8*)&pNew[1]; + if( zFile ){ + pNew->zFile = (char*)&pNew->aBuffer[ZIPFILE_BUFFER_SIZE]; + memcpy(pNew->zFile, zFile, nFile); + zipfileDequote(pNew->zFile); + } } *ppVtab = (sqlite3_vtab*)pNew; return rc; @@ -234,11 +319,12 @@ static int zipfileDisconnect(sqlite3_vtab *pVtab){ */ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ ZipfileCsr *pCsr; - pCsr = sqlite3_malloc( sizeof(*pCsr) + ZIPFILE_BUFFER_SIZE); - if( pCsr==0 ) return SQLITE_NOMEM; + pCsr = sqlite3_malloc(sizeof(*pCsr)); + *ppCsr = (sqlite3_vtab_cursor*)pCsr; + if( pCsr==0 ){ + return SQLITE_NOMEM; + } memset(pCsr, 0, sizeof(*pCsr)); - pCsr->aBuffer = (u8*)&pCsr[1]; - *ppCsr = &pCsr->base; return SQLITE_OK; } @@ -247,6 +333,8 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ ** by zipfileOpen(). */ static void zipfileResetCursor(ZipfileCsr *pCsr){ + sqlite3_free(pCsr->cds.zFile); + pCsr->cds.zFile = 0; pCsr->iRowid = 0; pCsr->bEof = 0; if( pCsr->pFile ){ @@ -276,17 +364,39 @@ static void zipfileSetErrmsg(ZipfileCsr *pCsr, const char *zFmt, ...){ va_end(ap); } -static int zipfileReadData(ZipfileCsr *pCsr, u8 *aRead, int nRead, i64 iOff){ +static int zipfileReadData( + FILE *pFile, /* Read from this file */ + u8 *aRead, /* Read into this buffer */ + int nRead, /* Number of bytes to read */ + i64 iOff, /* Offset to read from */ + char **pzErrmsg /* OUT: Error message (from sqlite3_malloc) */ +){ size_t n; - fseek(pCsr->pFile, iOff, SEEK_SET); - n = fread(aRead, 1, nRead, pCsr->pFile); + fseek(pFile, iOff, SEEK_SET); + n = fread(aRead, 1, nRead, pFile); if( n!=nRead ){ - zipfileSetErrmsg(pCsr, "error in fread()"); + *pzErrmsg = sqlite3_mprintf("error in fread()"); return SQLITE_ERROR; } return SQLITE_OK; } +static int zipfileAppendData( + ZipfileTab *pTab, + const u8 *aWrite, + int nWrite +){ + size_t n; + fseek(pTab->pWriteFd, pTab->szCurrent, SEEK_SET); + n = fwrite(aWrite, 1, nWrite, pTab->pWriteFd); + if( n!=nWrite ){ + pTab->base.zErrMsg = sqlite3_mprintf("error in fwrite()"); + return SQLITE_ERROR; + } + pTab->szCurrent += nWrite; + return SQLITE_OK; +} + static u16 zipfileGetU16(const u8 *aBuf){ return (aBuf[1] << 8) + aBuf[0]; } @@ -297,18 +407,47 @@ static u32 zipfileGetU32(const u8 *aBuf){ + ((u32)(aBuf[0]) << 0); } +static void zipfilePutU16(u8 *aBuf, u16 val){ + aBuf[0] = val & 0xFF; + aBuf[1] = (val>>8) & 0xFF; +} +static void zipfilePutU32(u8 *aBuf, u32 val){ + aBuf[0] = val & 0xFF; + aBuf[1] = (val>>8) & 0xFF; + aBuf[2] = (val>>16) & 0xFF; + aBuf[3] = (val>>24) & 0xFF; +} + #define zipfileRead32(aBuf) ( aBuf+=4, zipfileGetU32(aBuf-4) ) #define zipfileRead16(aBuf) ( aBuf+=2, zipfileGetU16(aBuf-2) ) +#define zipfileWrite32(aBuf,val) { zipfilePutU32(aBuf,val); aBuf+=4; } +#define zipfileWrite16(aBuf,val) { zipfilePutU16(aBuf,val); aBuf+=2; } + +static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){ + return ((ZipfileTab*)(pCsr->base.pVtab))->aBuffer; +} + +/* +** Magic numbers used to read CDS records. +*/ +#define ZIPFILE_CDS_FIXED_SZ 46 +#define ZIPFILE_CDS_NFILE_OFF 28 + static int zipfileReadCDS(ZipfileCsr *pCsr){ - static const int szFix = 46; /* Size of fixed-size part of CDS */ - u8 *aRead = pCsr->aBuffer; + char **pzErr = &pCsr->base.pVtab->zErrMsg; + u8 *aRead = zipfileCsrBuffer(pCsr); int rc; - rc = zipfileReadData(pCsr, aRead, szFix, pCsr->iNextOff); + sqlite3_free(pCsr->cds.zFile); + pCsr->cds.zFile = 0; + + rc = zipfileReadData( + pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr + ); if( rc==SQLITE_OK ){ u32 sig = zipfileRead32(aRead); - if( sig!=0x02014b50 ){ + if( sig!=ZIPFILE_SIGNATURE_CDS ){ zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff); rc = SQLITE_ERROR; }else{ @@ -322,6 +461,7 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){ pCsr->cds.crc32 = zipfileRead32(aRead); pCsr->cds.szCompressed = zipfileRead32(aRead); pCsr->cds.szUncompressed = zipfileRead32(aRead); + assert( aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_NFILE_OFF ); pCsr->cds.nFile = zipfileRead16(aRead); pCsr->cds.nExtra = zipfileRead16(aRead); pCsr->cds.nComment = zipfileRead16(aRead); @@ -330,15 +470,15 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){ pCsr->cds.iExternalAttr = zipfileRead32(aRead); pCsr->cds.iOffset = zipfileRead32(aRead); - assert( aRead==&pCsr->aBuffer[szFix] ); + assert( aRead==zipfileCsrBuffer(pCsr)+ZIPFILE_CDS_FIXED_SZ ); nRead = pCsr->cds.nFile + pCsr->cds.nExtra; - aRead = pCsr->aBuffer; - rc = zipfileReadData(pCsr, aRead, nRead, pCsr->iNextOff+szFix); + aRead = zipfileCsrBuffer(pCsr); + pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; + rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr); if( rc==SQLITE_OK ){ pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead); - pCsr->iNextOff += szFix; pCsr->iNextOff += pCsr->cds.nFile; pCsr->iNextOff += pCsr->cds.nExtra; pCsr->iNextOff += pCsr->cds.nComment; @@ -381,14 +521,15 @@ static int zipfileReadCDS(ZipfileCsr *pCsr){ } static int zipfileReadLFH(ZipfileCsr *pCsr){ - static const int szFix = 30; /* Size of fixed-size part of LFH */ - u8 *aRead = pCsr->aBuffer; + char **pzErr = &pCsr->base.pVtab->zErrMsg; + static const int szFix = ZIPFILE_LFH_FIXED_SZ; + u8 *aRead = zipfileCsrBuffer(pCsr); int rc; - rc = zipfileReadData(pCsr, aRead, szFix, pCsr->cds.iOffset); + rc = zipfileReadData(pCsr->pFile, aRead, szFix, pCsr->cds.iOffset, pzErr); if( rc==SQLITE_OK ){ u32 sig = zipfileRead32(aRead); - if( sig!=0x04034b50 ){ + if( sig!=ZIPFILE_SIGNATURE_LFH ){ zipfileSetErrmsg(pCsr, "failed to read LFH at offset %d", (int)pCsr->cds.iOffset ); @@ -404,7 +545,7 @@ static int zipfileReadLFH(ZipfileCsr *pCsr){ pCsr->lfh.szUncompressed = zipfileRead32(aRead); pCsr->lfh.nFile = zipfileRead16(aRead); pCsr->lfh.nExtra = zipfileRead16(aRead); - assert( aRead==&pCsr->aBuffer[szFix] ); + assert( aRead==zipfileCsrBuffer(pCsr)+szFix ); pCsr->iDataOff = pCsr->cds.iOffset+szFix+pCsr->lfh.nFile+pCsr->lfh.nExtra; } } @@ -460,6 +601,22 @@ static time_t zipfileMtime(ZipfileCsr *pCsr){ return mktime(&t); } +static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){ + time_t t = (time_t)mTime; + struct tm res; + localtime_r(&t, &res); + + pCds->mTime = + (res.tm_sec / 2) + + (res.tm_min << 5) + + (res.tm_hour << 11); + + pCds->mDate = + (res.tm_mday-1) + + ((res.tm_mon+1) << 5) + + ((res.tm_year-80) << 9); +} + /* ** Return values of columns for the row at which the series_cursor ** is currently pointing. @@ -499,7 +656,9 @@ static int zipfileColumn( if( aBuf==0 ){ rc = SQLITE_NOMEM; }else{ - rc = zipfileReadData(pCsr, aBuf, sz, pCsr->iDataOff); + rc = zipfileReadData(pCsr->pFile, aBuf, sz, pCsr->iDataOff, + &pCsr->base.pVtab->zErrMsg + ); } if( rc==SQLITE_OK ){ sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT); @@ -537,17 +696,26 @@ static int zipfileEof(sqlite3_vtab_cursor *cur){ } /* -** The zip file has been successfully opened (so pCsr->pFile is valid). -** This function attempts to locate and read the End of central -** directory record from the file. -** */ -static int zipfileReadEOCD(ZipfileCsr *pCsr, ZipfileEOCD *pEOCD){ - u8 *aRead = pCsr->aBuffer; - int nRead = (int)(MIN(pCsr->nByte, ZIPFILE_BUFFER_SIZE)); - i64 iOff = pCsr->nByte - nRead; +static int zipfileReadEOCD( + ZipfileTab *pTab, /* Return errors here */ + FILE *pFile, /* Read from this file */ + ZipfileEOCD *pEOCD /* Object to populate */ +){ + u8 *aRead = pTab->aBuffer; /* Temporary buffer */ + i64 szFile; /* Total size of file in bytes */ + int nRead; /* Bytes to read from file */ + i64 iOff; /* Offset to read from */ + + fseek(pFile, 0, SEEK_END); + szFile = (i64)ftell(pFile); + if( szFile==0 ){ + return SQLITE_EMPTY; + } + nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE)); + iOff = szFile - nRead; - int rc = zipfileReadData(pCsr, aRead, nRead, iOff); + int rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg); if( rc==SQLITE_OK ){ int i; @@ -560,7 +728,9 @@ static int zipfileReadEOCD(ZipfileCsr *pCsr, ZipfileEOCD *pEOCD){ } } if( i<0 ){ - zipfileSetErrmsg(pCsr, "cannot find end of central directory record"); + pTab->base.zErrMsg = sqlite3_mprintf( + "cannot find end of central directory record" + ); return SQLITE_ERROR; } @@ -592,6 +762,7 @@ static int zipfileFilter( int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ + ZipfileTab *pTab = (ZipfileTab*)cur->pVtab; ZipfileCsr *pCsr = (ZipfileCsr*)cur; const char *zFile; /* Zip file to scan */ int rc = SQLITE_OK; /* Return Code */ @@ -600,23 +771,29 @@ static int zipfileFilter( assert( idxNum==argc && (idxNum==0 || idxNum==1) ); if( idxNum==0 ){ - /* Error. User did not supply a file name. */ - zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument"); - return SQLITE_ERROR; + ZipfileTab *pTab = (ZipfileTab*)cur->pVtab; + zFile = pTab->zFile; + if( zFile==0 ){ + /* Error. This is an eponymous virtual table and the user has not + ** supplied a file name. */ + zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument"); + return SQLITE_ERROR; + } + }else{ + zFile = (const char*)sqlite3_value_text(argv[0]); } - - zFile = sqlite3_value_text(argv[0]); pCsr->pFile = fopen(zFile, "rb"); if( pCsr->pFile==0 ){ zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile); rc = SQLITE_ERROR; }else{ - fseek(pCsr->pFile, 0, SEEK_END); - pCsr->nByte = (i64)ftell(pCsr->pFile); - rc = zipfileReadEOCD(pCsr, &pCsr->eocd); + rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd); if( rc==SQLITE_OK ){ pCsr->iNextOff = pCsr->eocd.iOffset; rc = zipfileNext(cur); + }else if( rc==SQLITE_EMPTY ){ + rc = SQLITE_OK; + pCsr->bEof = 1; } } @@ -653,17 +830,404 @@ static int zipfileBestIndex( return SQLITE_OK; } +static int zipfileLoadDirectory(ZipfileTab *pTab){ + ZipfileEOCD eocd; + int rc; + + rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd); + if( rc==SQLITE_OK ){ + int i; + int iOff = 0; + u8 *aBuf = sqlite3_malloc(eocd.nSize); + if( aBuf==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = zipfileReadData( + pTab->pWriteFd, aBuf, eocd.nSize, eocd.iOffset, &pTab->base.zErrMsg + ); + } + + for(i=0; rc==SQLITE_OK && izPath = (char*)&pNew[1]; + memcpy(pNew->zPath, &aBuf[ZIPFILE_CDS_FIXED_SZ], nFile); + pNew->zPath[nFile] = '\0'; + pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1]; + pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment; + memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry); + + pNew->pNext = pTab->pEntry; + pTab->pEntry = pNew; + } + + iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment; + } + + sqlite3_free(aBuf); + }else if( rc==SQLITE_EMPTY ){ + rc = SQLITE_OK; + } + + return rc; +} + +static ZipfileEntry *zipfileNewEntry( + ZipfileCDS *pCds, /* Values for fixed size part of CDS */ + const char *zPath, /* Path for new entry */ + int nPath, /* strlen(zPath) */ + u32 mTime /* Modification time (or 0) */ +){ + u8 *aWrite; + ZipfileEntry *pNew; + pCds->nFile = nPath; + pCds->nExtra = mTime ? 9 : 0; + pNew = (ZipfileEntry*)sqlite3_malloc( + sizeof(ZipfileEntry) + + nPath+1 + + ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra + ); + + if( pNew ){ + pNew->zPath = (char*)&pNew[1]; + pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1]; + pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra; + memcpy(pNew->zPath, zPath, nPath+1); + + aWrite = pNew->aCdsEntry; + zipfileWrite32(aWrite, ZIPFILE_SIGNATURE_CDS); + zipfileWrite16(aWrite, pCds->iVersionMadeBy); + zipfileWrite16(aWrite, pCds->iVersionExtract); + zipfileWrite16(aWrite, pCds->flags); + zipfileWrite16(aWrite, pCds->iCompression); + zipfileWrite16(aWrite, pCds->mTime); + zipfileWrite16(aWrite, pCds->mDate); + zipfileWrite32(aWrite, pCds->crc32); + zipfileWrite32(aWrite, pCds->szCompressed); + zipfileWrite32(aWrite, pCds->szUncompressed); + zipfileWrite16(aWrite, pCds->nFile); + zipfileWrite16(aWrite, pCds->nExtra); + zipfileWrite16(aWrite, pCds->nComment); assert( pCds->nComment==0 ); + zipfileWrite16(aWrite, pCds->iDiskStart); + zipfileWrite16(aWrite, pCds->iInternalAttr); + zipfileWrite32(aWrite, pCds->iExternalAttr); + zipfileWrite32(aWrite, pCds->iOffset); + assert( aWrite==&pNew->aCdsEntry[ZIPFILE_CDS_FIXED_SZ] ); + memcpy(aWrite, zPath, nPath); + if( pCds->nExtra ){ + aWrite += nPath; + zipfileWrite16(aWrite, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(aWrite, 5); + *aWrite++ = 0x01; + zipfileWrite32(aWrite, mTime); + } + } + + return pNew; +} + +static int zipfileAppendEntry( + ZipfileTab *pTab, + ZipfileCDS *pCds, + const char *zPath, /* Path for new entry */ + int nPath, /* strlen(zPath) */ + const u8 *pData, + int nData, + u32 mTime +){ + u8 *aBuf = pTab->aBuffer; + int rc; + + zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_LFH); + zipfileWrite16(aBuf, pCds->iVersionExtract); + zipfileWrite16(aBuf, pCds->flags); + zipfileWrite16(aBuf, pCds->iCompression); + zipfileWrite16(aBuf, pCds->mTime); + zipfileWrite16(aBuf, pCds->mDate); + zipfileWrite32(aBuf, pCds->crc32); + zipfileWrite32(aBuf, pCds->szCompressed); + zipfileWrite32(aBuf, pCds->szUncompressed); + zipfileWrite16(aBuf, nPath); + zipfileWrite16(aBuf, pCds->nExtra); + assert( aBuf==&pTab->aBuffer[ZIPFILE_LFH_FIXED_SZ] ); + rc = zipfileAppendData(pTab, pTab->aBuffer, aBuf - pTab->aBuffer); + if( rc==SQLITE_OK ){ + rc = zipfileAppendData(pTab, (const u8*)zPath, nPath); + } + + if( rc==SQLITE_OK && pCds->nExtra ){ + aBuf = pTab->aBuffer; + zipfileWrite16(aBuf, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(aBuf, 5); + *aBuf++ = 0x01; + zipfileWrite32(aBuf, mTime); + rc = zipfileAppendData(pTab, pTab->aBuffer, 9); + } + + if( rc==SQLITE_OK ){ + rc = zipfileAppendData(pTab, pData, nData); + } + + return rc; +} + +static int zipfileGetMode(ZipfileTab *pTab, sqlite3_value *pVal, int *pMode){ + const char *z = (const char*)sqlite3_value_text(pVal); + int mode = 0; + if( z==0 || (z[0]>=0 && z[0]<=9) ){ + mode = sqlite3_value_int(pVal); + }else{ + const char zTemplate[10] = "-rwxrwxrwx"; + int i; + if( strlen(z)!=10 ) goto parse_error; + switch( z[0] ){ + case '-': mode |= S_IFREG; break; + case 'd': mode |= S_IFDIR; break; + case 'l': mode |= S_IFLNK; break; + default: goto parse_error; + } + for(i=1; i<10; i++){ + if( z[i]==zTemplate[i] ) mode |= 1 << (9-i); + else if( z[i]!='-' ) goto parse_error; + } + } + *pMode = mode; + return SQLITE_OK; + + parse_error: + pTab->base.zErrMsg = sqlite3_mprintf("zipfile: parse error in mode: %s", z); + return SQLITE_ERROR; +} + +/* +** xUpdate method. +*/ +static int zipfileUpdate( + sqlite3_vtab *pVtab, + int nVal, + sqlite3_value **apVal, + sqlite_int64 *pRowid +){ + ZipfileTab *pTab = (ZipfileTab*)pVtab; + int rc = SQLITE_OK; /* Return Code */ + ZipfileEntry *pNew = 0; /* New in-memory CDS entry */ + + int mode; /* Mode for new entry */ + i64 mTime; /* Modification time for new entry */ + i64 sz; /* Uncompressed size */ + const char *zPath; /* Path for new entry */ + int nPath; /* strlen(zPath) */ + const u8 *pData; /* Pointer to buffer containing content */ + int nData; /* Size of pData buffer in bytes */ + int iMethod = 0; /* Compression method for new entry */ + u8 *pFree = 0; /* Free this */ + ZipfileCDS cds; /* New Central Directory Structure entry */ + + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: UPDATE/DELETE not yet supported" + ); + return SQLITE_ERROR; + } + + /* This table is only writable if a default archive path was specified + ** as part of the CREATE VIRTUAL TABLE statement. */ + if( pTab->zFile==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: writing requires a default archive" + ); + return SQLITE_ERROR; + } + + /* Open a write fd on the file. Also load the entire central directory + ** structure into memory. During the transaction any new file data is + ** appended to the archive file, but the central directory is accumulated + ** in main-memory until the transaction is committed. */ + if( pTab->pWriteFd==0 ){ + pTab->pWriteFd = fopen(pTab->zFile, "ab+"); + if( pTab->pWriteFd==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: failed to open file %s for writing", pTab->zFile + ); + return SQLITE_ERROR; + } + fseek(pTab->pWriteFd, 0, SEEK_END); + pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); + rc = zipfileLoadDirectory(pTab); + if( rc!=SQLITE_OK ) return rc; + } + + zPath = (const char*)sqlite3_value_text(apVal[2]); + nPath = strlen(zPath); + rc = zipfileGetMode(pTab, apVal[3], &mode); + if( rc!=SQLITE_OK ) return rc; + mTime = sqlite3_value_int64(apVal[4]); + sz = sqlite3_value_int(apVal[5]); + pData = sqlite3_value_blob(apVal[6]); + nData = sqlite3_value_bytes(apVal[6]); + + /* If a NULL value is inserted into the 'method' column, do automatic + ** compression. */ + if( nData>0 && sqlite3_value_type(apVal[7])==SQLITE_NULL ){ + pFree = (u8*)sqlite3_malloc(nData); + if( pFree==0 ){ + rc = SQLITE_NOMEM; + }else{ + int res; + z_stream str; + memset(&str, 0, sizeof(str)); + str.next_in = (z_const Bytef*)pData; + str.avail_in = nData; + str.next_out = pFree; + str.avail_out = nData; + deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); + res = deflate(&str, Z_FINISH); + if( res==Z_STREAM_END ){ + pData = pFree; + nData = str.total_out; + iMethod = 8; + }else if( res!=Z_OK ){ + pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error"); + rc = SQLITE_ERROR; + } + deflateEnd(&str); + } + }else{ + iMethod = sqlite3_value_int(apVal[7]); + if( iMethod<0 || iMethod>65535 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: invalid compression method: %d", iMethod + ); + rc = SQLITE_ERROR; + } + } + + if( rc==SQLITE_OK ){ + /* Create the new CDS record. */ + memset(&cds, 0, sizeof(cds)); + cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; + cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; + cds.flags = ZIPFILE_NEWENTRY_FLAGS; + cds.iCompression = iMethod; + zipfileMtimeToDos(&cds, (u32)mTime); + cds.crc32 = crc32(0, pData, nData); + cds.szCompressed = nData; + cds.szUncompressed = sz; + cds.iExternalAttr = (mode<<16); + cds.iOffset = pTab->szCurrent; + pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pNew->pNext = pTab->pEntry; + pTab->pEntry = pNew; + } + } + + /* Append the new header+file to the archive */ + if( rc==SQLITE_OK ){ + rc = zipfileAppendEntry(pTab, &cds, zPath, nPath, pData, nData, (u32)mTime); + } + + sqlite3_free(pFree); + return rc; +} + +static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){ + u8 *aBuf = pTab->aBuffer; + + zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_EOCD); + zipfileWrite16(aBuf, p->iDisk); + zipfileWrite16(aBuf, p->iFirstDisk); + zipfileWrite16(aBuf, p->nEntry); + zipfileWrite16(aBuf, p->nEntryTotal); + zipfileWrite32(aBuf, p->nSize); + zipfileWrite32(aBuf, p->iOffset); + zipfileWrite16(aBuf, 0); /* Size of trailing comment in bytes*/ + + assert( (aBuf-pTab->aBuffer)==22 ); + return zipfileAppendData(pTab, pTab->aBuffer, aBuf - pTab->aBuffer); +} + +static void zipfileCleanupTransaction(ZipfileTab *pTab){ + ZipfileEntry *pEntry; + ZipfileEntry *pNext; + + for(pEntry=pTab->pEntry; pEntry; pEntry=pNext){ + pNext = pEntry->pNext; + sqlite3_free(pEntry); + } + pTab->pEntry = 0; + fclose(pTab->pWriteFd); + pTab->pWriteFd = 0; + pTab->szCurrent = 0; + pTab->szOrig = 0; +} + +static int zipfileBegin(sqlite3_vtab *pVtab){ + return SQLITE_OK; +} + +static int zipfileCommit(sqlite3_vtab *pVtab){ + ZipfileTab *pTab = (ZipfileTab*)pVtab; + int rc = SQLITE_OK; + if( pTab->pWriteFd ){ + i64 iOffset = pTab->szCurrent; + ZipfileEntry *pEntry; + ZipfileEOCD eocd; + int nEntry = 0; + + /* Write out all entries */ + for(pEntry=pTab->pEntry; rc==SQLITE_OK && pEntry; pEntry=pEntry->pNext){ + rc = zipfileAppendData(pTab, pEntry->aCdsEntry, pEntry->nCdsEntry); + nEntry++; + } + + /* Write out the EOCD record */ + eocd.iDisk = 0; + eocd.iFirstDisk = 0; + eocd.nEntry = nEntry; + eocd.nEntryTotal = nEntry; + eocd.nSize = pTab->szCurrent - iOffset;; + eocd.iOffset = iOffset; + rc = zipfileAppendEOCD(pTab, &eocd); + + zipfileCleanupTransaction(pTab); + } + return rc; +} + +static int zipfileRollback(sqlite3_vtab *pVtab){ + return zipfileCommit(pVtab); +} + /* ** Register the "zipfile" virtual table. */ static int zipfileRegister(sqlite3 *db){ static sqlite3_module zipfileModule = { - 0, /* iVersion */ - 0, /* xCreate */ + 1, /* iVersion */ + zipfileConnect, /* xCreate */ zipfileConnect, /* xConnect */ zipfileBestIndex, /* xBestIndex */ zipfileDisconnect, /* xDisconnect */ - 0, /* xDestroy */ + zipfileDisconnect, /* xDestroy */ zipfileOpen, /* xOpen - open a cursor */ zipfileClose, /* xClose - close a cursor */ zipfileFilter, /* xFilter - configure scan constraints */ @@ -671,11 +1235,11 @@ static int zipfileRegister(sqlite3 *db){ zipfileEof, /* xEof - check for end of scan */ zipfileColumn, /* xColumn - read data */ zipfileRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ + zipfileUpdate, /* xUpdate */ + zipfileBegin, /* xBegin */ 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ + zipfileCommit, /* xCommit */ + zipfileRollback, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ }; @@ -687,8 +1251,6 @@ static int zipfileRegister(sqlite3 *db){ # define zipfileRegister(x) SQLITE_OK #endif -#include - /* ** zipfile_uncompress(DATA, SZ, METHOD) */ diff --git a/main.mk b/main.mk index ff379da861..9cdcc051e3 100644 --- a/main.mk +++ b/main.mk @@ -370,6 +370,7 @@ TESTSRC += \ $(TOP)/ext/misc/unionvtab.c \ $(TOP)/ext/misc/wholenumber.c \ $(TOP)/ext/misc/vfslog.c \ + $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ $(TOP)/ext/fts5/fts5_test_tok.c diff --git a/manifest b/manifest index d6c9a543e5..c8433dc8ca 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\sshell\stool\s".ar\s--list\s--verbose"\scommand. -D 2017-12-27T21:13:21.423 +C Update\sext/misc/zipfile.c\sto\ssupport\screating\sand\sadding\sentries\sto\sexisting\nzip\sarchives. +D 2017-12-29T20:19:03.196 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 @@ -302,7 +302,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c ced1aa768904cf8182c3da93d42df4e3cf30fe7a2e66c9216f6bc64a8df4fd5b +F ext/misc/zipfile.c d9e0b0a4fdb4cb82ad22fd19d75563efdb1d6f812d62438bd30819781c282357 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -405,7 +405,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk fc13303745f7a06e2eac69406ee0e7ba481f6df37a528ceb6ebacc03e310a020 +F main.mk 9109d5786263a20fcf321a77c463d39c9ac84c871ea3774b0d213d8311dda105 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -492,7 +492,7 @@ F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 F src/tclsqlite.c 1833388c01e3b77f4c712185ee7250b9423ee0981ce6ae7e401e47db0319a696 -F src/test1.c 8ef15f7a357f85dfc41c6c748ce9c947b4f676e01bb5ae6a45bee4923dff8b51 +F src/test1.c 1879ff5095def6091f83f54c6233bc19e2b7223068fb02da41f2396d55729764 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 @@ -1596,6 +1596,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e +F test/zipfile.test c5167d0d13ed7165b1982ac46b8aa94d65bdf41d94dd82f1d17ad2250650356c F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1692,7 +1693,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a532a0f6fd59e81086d46f09151ba7fb26725198231d902c71d0f95cb01dbe91 -R 7075030d907b0ffe212621eb709bb967 +P b64681a644c419bb98d00980a6cb56ef5a0aff5ef5321955631f0b4c88aac283 +R 407639481a191f472cd3497cc84b9ac5 U dan -Z 1c430e112380037cc2bfad44bf8be92e +Z a9a0bf4010ca982b681b075b8e4b53ca diff --git a/manifest.uuid b/manifest.uuid index aaba04f0ec..d0d83a4e81 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b64681a644c419bb98d00980a6cb56ef5a0aff5ef5321955631f0b4c88aac283 \ No newline at end of file +2dec2dec592c0726ebe87b841b9c8d493dea7074a99f278eb1bf0b744d658a9d \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 446317d803..f9b3b69711 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6960,6 +6960,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd( extern int sqlite3_totype_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_unionvtab_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_zipfile_init(sqlite3*,char**,const sqlite3_api_routines*); static const struct { const char *zExtName; int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*); @@ -6981,6 +6982,7 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd( { "totype", sqlite3_totype_init }, { "unionvtab", sqlite3_unionvtab_init }, { "wholenumber", sqlite3_wholenumber_init }, + { "zipfile", sqlite3_zipfile_init }, }; sqlite3 *db; const char *zName; diff --git a/test/zipfile.test b/test/zipfile.test new file mode 100644 index 0000000000..6c0ed4ba62 --- /dev/null +++ b/test/zipfile.test @@ -0,0 +1,60 @@ +# 2017 December 9 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix zipfile + +load_static_extension db zipfile + +forcedelete test.zip +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE temp.zz USING zipfile('test.zip'); + PRAGMA table_info(zz); +} { + 0 name {} 0 {} 0 + 1 mode {} 0 {} 0 + 2 mtime {} 0 {} 0 + 3 sz {} 0 {} 0 + 4 data {} 0 {} 0 + 5 method {} 0 {} 0 +} + +do_execsql_test 1.1 { + INSERT INTO zz VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0); + INSERT INTO zz VALUES('g.txt', '-rw-r--r--', 1000000002, 5, '12345', 0); +} + +do_execsql_test 1.2 { + SELECT name, mtime, data FROM zipfile('test.zip'); +} { + g.txt 1000000002 12345 + f.txt 1000000000 abcde +} + +do_execsql_test 1.3 { + INSERT INTO zz VALUES('h.txt', + '-rw-r--r--', 1000000004, 20, 'aaaaaaaaaabbbbbbbbbb', NULL + ); +} + +do_execsql_test 1.4 { + SELECT name, mtime, zipfile_uncompress(data, sz, method), method + FROM zipfile('test.zip'); +} { + h.txt 1000000004 aaaaaaaaaabbbbbbbbbb 8 + f.txt 1000000000 abcde 0 + g.txt 1000000002 12345 0 +} + +finish_test +