From: jo620kix <81214276+jo620kix@users.noreply.github.com> Date: Mon, 29 Mar 2021 05:02:57 +0000 (+0900) Subject: ZIP reader: added support for Zstd decompression X-Git-Tag: v3.6.0~17^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b0ad184f6aea44e4a4d35487ccfc529eea3f5c6b;p=thirdparty%2Flibarchive.git ZIP reader: added support for Zstd decompression --- diff --git a/libarchive/archive_read_support_format_zip.c b/libarchive/archive_read_support_format_zip.c index 5e8895af0..178122d62 100644 --- a/libarchive/archive_read_support_format_zip.c +++ b/libarchive/archive_read_support_format_zip.c @@ -58,6 +58,9 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_read_support_format_zip.c 201102 #ifdef HAVE_LZMA_H #include #endif +#ifdef HAVE_ZSTD_H +#include +#endif #include "archive.h" #include "archive_digest_private.h" @@ -191,6 +194,11 @@ struct zip { char bzstream_valid; #endif +#if HAVE_ZSTD_H && HAVE_LIBZSTD + ZSTD_DStream *zstdstream; + char zstdstream_valid; +#endif + IByteIn zipx_ppmd_stream; ssize_t zipx_ppmd_read_compressed; CPpmd8 ppmd8; @@ -424,6 +432,7 @@ static const struct { {17, "reserved"}, /* Reserved by PKWARE */ {18, "ibm-terse-new"}, /* File is compressed using IBM TERSE (new) */ {19, "ibm-lz777"},/* IBM LZ77 z Architecture (PFS) */ + {93, "zstd"}, /* Zstandard (zstd) Compression */ {95, "xz"}, /* XZ compressed data */ {96, "jpeg"}, /* JPEG compressed data */ {97, "wav-pack"}, /* WavPack compressed data */ @@ -2299,6 +2308,129 @@ zip_read_data_zipx_bzip2(struct archive_read *a, const void **buff, #endif +#if HAVE_ZSTD_H && HAVE_LIBZSTD +static int +zipx_zstd_init(struct archive_read *a, struct zip *zip) +{ + size_t r; + + /* Deallocate already existing Zstd decompression context if it + * exists. */ + if(zip->zstdstream_valid) { + ZSTD_freeDStream(zip->zstdstream); + zip->zstdstream_valid = 0; + } + + /* Allocate a new Zstd decompression context. */ + zip->zstdstream = ZSTD_createDStream(); + + r = ZSTD_initDStream(zip->zstdstream); + if (ZSTD_isError(r)) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Error initializing zstd decompressor: %s", + ZSTD_getErrorName(r)); + + return ARCHIVE_FAILED; + } + + /* Mark the zstdstream field to be released in cleanup phase. */ + zip->zstdstream_valid = 1; + + /* (Re)allocate the buffer that will contain decompressed bytes. */ + free(zip->uncompressed_buffer); + + zip->uncompressed_buffer_size = ZSTD_DStreamOutSize(); + zip->uncompressed_buffer = + (uint8_t*) malloc(zip->uncompressed_buffer_size); + if (zip->uncompressed_buffer == NULL) { + archive_set_error(&a->archive, ENOMEM, + "No memory for Zstd decompression"); + + return ARCHIVE_FATAL; + } + + /* Initialization done. */ + zip->decompress_init = 1; + return ARCHIVE_OK; +} + +static int +zip_read_data_zipx_zstd(struct archive_read *a, const void **buff, + size_t *size, int64_t *offset) +{ + struct zip *zip = (struct zip *)(a->format->data); + ssize_t bytes_avail = 0, in_bytes, to_consume; + const void *compressed_buff; + int r; + size_t ret; + uint64_t total_out; + ZSTD_outBuffer out; + ZSTD_inBuffer in; + + /* Initialize decompression context if we're here for the first time. */ + if(!zip->decompress_init) { + r = zipx_zstd_init(a, zip); + if(r != ARCHIVE_OK) + return r; + } + + /* Fetch more compressed bytes */ + compressed_buff = __archive_read_ahead(a, 1, &bytes_avail); + if(bytes_avail < 0) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Truncated zstd file body"); + return (ARCHIVE_FATAL); + } + + in_bytes = zipmin(zip->entry_bytes_remaining, bytes_avail); + + /* Setup buffer boundaries */ + in.src = compressed_buff; + in.size = in_bytes; + in.pos = 0; + out = (ZSTD_outBuffer) { zip->uncompressed_buffer, zip->uncompressed_buffer_size, 0 }; + + /* Perform the decompression. */ + ret = ZSTD_decompressStream(zip->zstdstream, &out, &in); + if (ZSTD_isError(ret)) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Error during zstd decompression: %s", + ZSTD_getErrorName(ret)); + return (ARCHIVE_FATAL); + } + + /* Check end of the stream. */ + if (ret == 0) { + if ((in.pos == in.size) && (out.pos < out.size)) { + zip->end_of_entry = 1; + ZSTD_freeDStream(zip->zstdstream); + zip->zstdstream_valid = 0; + } + } + + /* Update the pointers so decompressor can continue decoding. */ + to_consume = in.pos; + __archive_read_consume(a, to_consume); + + total_out = out.pos; + + zip->entry_bytes_remaining -= to_consume; + zip->entry_compressed_bytes_read += to_consume; + zip->entry_uncompressed_bytes_read += total_out; + + /* Give libarchive its due. */ + *size = total_out; + *buff = zip->uncompressed_buffer; + + /* Seek for optional marker, like in other entries. */ + r = consume_optional_marker(a, zip); + if(r != ARCHIVE_OK) + return r; + + return ARCHIVE_OK; +} +#endif + #ifdef HAVE_ZLIB_H static int zip_deflate_init(struct archive_read *a, struct zip *zip) @@ -2918,6 +3050,11 @@ archive_read_format_zip_read_data(struct archive_read *a, case 95: /* ZIPx XZ compression. */ r = zip_read_data_zipx_xz(a, buff, size, offset); break; +#endif +#if HAVE_ZSTD_H && HAVE_LIBZSTD + case 93: /* ZIPx Zstd compression. */ + r = zip_read_data_zipx_zstd(a, buff, size, offset); + break; #endif /* PPMd support is built-in, so we don't need any #if guards. */ case 98: /* ZIPx PPMd compression. */ @@ -3009,6 +3146,12 @@ archive_read_format_zip_cleanup(struct archive_read *a) } #endif +#if HAVE_ZSTD_H && HAVE_LIBZSTD + if (zip->zstdstream_valid) { + ZSTD_freeDStream(zip->zstdstream); + } +#endif + free(zip->uncompressed_buffer); if (zip->ppmd8_valid) diff --git a/libarchive/test/test_read_format_zip.c b/libarchive/test/test_read_format_zip.c index 31f66f012..fdbfdca61 100644 --- a/libarchive/test/test_read_format_zip.c +++ b/libarchive/test/test_read_format_zip.c @@ -736,6 +736,130 @@ DEFINE_TEST(test_read_format_zip_bzip2_multi_blockread) assertEqualIntA(a, ARCHIVE_OK, archive_read_free(a)); } +DEFINE_TEST(test_read_format_zip_zstd_one_file) +{ + const char *refname = "test_read_format_zip_zstd.zipx"; + struct archive *a; + struct archive_entry *ae; + + assert((a = archive_read_new()) != NULL); + if (ARCHIVE_OK != archive_read_support_filter_zstd(a)) { + skipping("zstd is not fully supported on this platform"); + archive_read_close(a); + return; + } + extract_reference_file(refname); + + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_zip(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_open_filename(a, refname, 37)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); + assertEqualString("ZIP 2.0 (zstd)", archive_format_name(a)); + assertEqualString("vimrc", archive_entry_pathname(ae)); + assertEqualIntA(a, 0, extract_one(a, ae, 0xBA8E3BAA)); + assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_free(a)); +} + +DEFINE_TEST(test_read_format_zip_zstd_one_file_blockread) +{ + const char *refname = "test_read_format_zip_zstd.zipx"; + struct archive *a; + struct archive_entry *ae; + + assert((a = archive_read_new()) != NULL); + if (ARCHIVE_OK != archive_read_support_filter_zstd(a)) { + skipping("zstd is not fully supported on this platform"); + archive_read_close(a); + return; + } + extract_reference_file(refname); + + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_zip(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_open_filename(a, refname, 37)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); + assertEqualString("ZIP 2.0 (zstd)", archive_format_name(a)); + assertEqualString("vimrc", archive_entry_pathname(ae)); + assertEqualIntA(a, 0, extract_one_using_blocks(a, 13, 0xBA8E3BAA)); + assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_free(a)); +} + +DEFINE_TEST(test_read_format_zip_zstd_multi) +{ + const char *refname = "test_read_format_zip_zstd_multi.zipx"; + struct archive *a; + struct archive_entry *ae; + + assert((a = archive_read_new()) != NULL); + if (ARCHIVE_OK != archive_read_support_filter_zstd(a)) { + skipping("zstd is not fully supported on this platform"); + archive_read_close(a); + return; + } + extract_reference_file(refname); + + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_zip(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_open_filename(a, refname, 37)); + + assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); + assertEqualString("ZIP 2.0 (zstd)", archive_format_name(a)); + assertEqualString("smartd.conf", archive_entry_pathname(ae)); + assertEqualIntA(a, 0, extract_one(a, ae, 0x8DD7379E)); + + assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); + assertEqualString("ZIP 2.0 (zstd)", archive_format_name(a)); + assertEqualString("ts.conf", archive_entry_pathname(ae)); + assertEqualIntA(a, 0, extract_one(a, ae, 0x7AE59B31)); + + assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); + assertEqualString("ZIP 2.0 (zstd)", archive_format_name(a)); + assertEqualString("vimrc", archive_entry_pathname(ae)); + assertEqualIntA(a, 0, extract_one(a, ae, 0xBA8E3BAA)); + + assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_free(a)); +} + +DEFINE_TEST(test_read_format_zip_zstd_multi_blockread) +{ + const char *refname = "test_read_format_zip_zstd_multi.zipx"; + struct archive *a; + struct archive_entry *ae; + + assert((a = archive_read_new()) != NULL); + if (ARCHIVE_OK != archive_read_support_filter_zstd(a)) { + skipping("zstd is not fully supported on this platform"); + archive_read_close(a); + return; + } + extract_reference_file(refname); + + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_zip(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_open_filename(a, refname, 37)); + + assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); + assertEqualString("ZIP 2.0 (zstd)", archive_format_name(a)); + assertEqualString("smartd.conf", archive_entry_pathname(ae)); + assertEqualIntA(a, 0, extract_one_using_blocks(a, 12, 0x8DD7379E)); + + assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); + assertEqualString("ZIP 2.0 (zstd)", archive_format_name(a)); + assertEqualString("ts.conf", archive_entry_pathname(ae)); + assertEqualIntA(a, 0, extract_one_using_blocks(a, 13, 0x7AE59B31)); + + assertEqualIntA(a, ARCHIVE_OK, archive_read_next_header(a, &ae)); + assertEqualString("ZIP 2.0 (zstd)", archive_format_name(a)); + assertEqualString("vimrc", archive_entry_pathname(ae)); + assertEqualIntA(a, 0, extract_one_using_blocks(a, 14, 0xBA8E3BAA)); + + assertEqualIntA(a, ARCHIVE_EOF, archive_read_next_header(a, &ae)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_close(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_free(a)); +} + DEFINE_TEST(test_read_format_zip_xz_multi) { const char *refname = "test_read_format_zip_xz_multi.zipx"; diff --git a/libarchive/test/test_read_format_zip_zstd.zipx.uu b/libarchive/test/test_read_format_zip_zstd.zipx.uu new file mode 100644 index 000000000..58a295844 --- /dev/null +++ b/libarchive/test/test_read_format_zip_zstd.zipx.uu @@ -0,0 +1,18 @@ +begin 644 test_read_format_zip_zstd.zipx +M4$L#!!0```!=`#TQD4VJ.XZZ&P(``)`#```%````=FEMI.97\6@M?/P$SA1R6+RFRUD<*@L-#!$.$A`-#!,!;_I.?CH\]WK(I'HJ +MC1"P;([X:RD\"4NZL[X9?H=*YI9Q^9SUG`U"%GS.%'JF0M<\[PV?3#MR">63 +MU+);7<0EM,_'U]+()+7>.MS@IFGS.GUF +M'S5\-@V=Y=MW6",,P^\(`,FALG&E(V`ZX;1&#G$;^?8[^=%R^:Q[GB1^JPZA +M-SI]3O>XY5O/>)70LG3/QWO;L^DZJQ/Q)"5'6^UGS\CUT,:'[<)XDMY"B%WV +MPK$('N2MJ]2M61L`\D@@2E$0?$HA-^YDM0T959VUCN4BS.J<-=[ +M6P)HFD)TO47V[FWT;?<:9=HN25KX&K(SA?9*P8'"=&]T1G?9G8^[AR>)P11R +MY'H86B0%XTKA&U$O;*N]M,>LJ)E`K77)I'N2E`(B(!!&C.+TY**V^BU(V9R8 +M!ZLXW)6;"!YPP'E,4_F\OB!T"4Z:6/#"Z823+@5V#BNHGHC-+4@`)>?""C,) +M2+G'/1,#D-1'B&&R288:'(KD,I>"_/85-]>-'@L``"L:```+````+!_^R$5-5D[3>O4AU,N^VP>S3+^79Y-P+W?;_&!3.]_D3K-7OY*9 +MR]UN$D)9?2_/*/9(FB13Q!*YTT"H42F^;(9P^_V^L)?+=AI^5S^^6_'%I]C. +M9Z)^65,*(B*9*!+I`,KD;LHM>DSQNK>@Y,P6C^]6?!12PCM-YG)LFS/\JU3JFDR`5XNOHZ0N^P8+&#R!1]OO +MVOBBS4MIQG>:9>T9Q:[)S%W4/VZGC-8[OR,X![]8K^OG=J7OBK4\VOC@YV9W +MFHWR5[_MCQ4?:Y^R7Y3899^7>IAUK^2;'[3VLMQI&JC1>GRGD9AHN)X7\Y;: +M^7ZOWM5WNA/:>0C?=J=I`+@#PY2O0,.QX&@L(&@\@0,*@`$E-!B.,QP-R!>< +M!,"=IK&FC!A*V5%BT;9MB\5"UGU]@=_7TNR+R88[31-545)1L>].?^1Z*L:? +MLOK"5SW*USHUZU514E^883K="9+:RYTFHYC6C\_OLCR*\G4F;]#*=Y#!]VJW +MEJ6P^EZ<)N[5?Y^;47@$2V0Z@!QU6F17TJ,I0O;UK)^Z6:84_%S$)!55+`'Q +M84Z/04]X!7I"PI,`>2!?`)UZ<<\[-?BTBO+P*787=[K31`@)!V\)`U,VM1,# +M;ELI":%NY97\:E`QW[3R^)R&'\@5#-#:Y4[S($`4&@>WW*T##RHV96P(0GL( +MO0'Z!:+0Z&RIBGU\3D-!Z6Q+PNF/)1T^Y9R4A[76YELJD8?):1J$!V.!8!$Y +M/.!@+*)H,1V<)^A17D)7Z.IX@5FW<)H(4PCTX(RZJ;5.=YI;4<+K,7U`CL?= +MK=TS[-9#J%-D6DSYB"4R+2*6R)OG8[C3-`E?`)TK%%O$ASD-:X_A2IYP +M!0.93:UU;H9M7_:M^+J7OYC7&N"DI5W?%+BLEY#!KH'Y2X'Y2Z6#/5,,>7VQ(C%-+[J49OQ!=/B$7K*[.!. +M\V2F,B^U\2WC^=T`(N(1;]+%MP`C+XOI!.'6[L+@&'%61I@BTUO&@T?#P="M +M#X$>=U&S($RF`G8GPY^#*SB-@@/R>#@D.,/0]%U,4KJ#&DP%@DN`306CS/TQQ3?05=,[X`:49NP+`&$@#>Z77!V +MZ^)VQ4Y_Y#11J=;KZD=R:\8GH#,NH9:5^C[J75Q^:P7<:1@^K:Z&NE<&QH@_ +MHJ#VHMYQX>/?:2+NN%B4\V'B7L(0F:8H%PM-Y'8^7G>GB=Q&K?JSXIO-E]AM +MQ\469MA[!CY`=_%Q$(RJ2AZ+4^LSKF#NM?V9D$!.DY#@N`(!/I&N0(`W/M>" +MBCUJW?7\G&;C@Q;PYD;"BQICRGXTR +M3K6T"S9?]>4)A@`4&I^`GD"A<4D%O6&_T%#3:B<*8<4']Z+PO*]Z3C[<,CT0 +MQU"OWT20_.F'/?@J(DA_EHH[MZH>PN221WHF7]#4SBFI.!$Q(>/8E426\BW5*!/N4UZ$\&3:17\LPP\G\B?6I0!EVV8I[6 +M`*4UB!OLE#"H9,P[LXK]5*A9.\][4:W8M]4MXEX;Q0!*Y;R69GSP*1]&K[UD +MJN<>*NAI@#YE)7OJ]LDD,'@FZM2FK*IGLXSECRP)H=OJ30UO@'[W +M_F0N+FKSGVP7\I^YF.4G(4Z91#P\3@A5)1=1'E257'A1?FY!^1TB!UPNYC$_ +M;&[V9Z&2WRBCSH6RV0*Q>%@X/!#/L_"B&(GM_`/A8J9B?QYM_<)FF?Y.IT@D +M8'F!P:@Q,5/(S(B(B$B2)!D.<02$&,6>IE2R*7%*T&;?1.FK+IYT!&_5B-5DP39XE!D% +MU4I&#GB2.ZRXY2=4I)"45?,9GE*1:>%CGS]Y$T?]!<.L!XW0S]DV[J]^RZUO +MU-$)H:Y^.=7_&D-=/U?:N2=I9478#((F??EK=-I1S/+>G(J&!8#8Z!:SN/"0W!8-Q[JFX +MM_*65-,=T<$H_5R>$A*!B@?!QDT)A2&MH8F637:F%R6%";@](C+4@\6'16DZ +MU_HU?%3PJE\R1U+@1W,+`-#RU%+^+.>.!"M\UT,*C&33WQRGB1G26U$9^L)A +M)I%").6%$Y@_%NBBIU(1:BFRY39*=(')Z1KA$&)]OJ(+#I`/%M!VE>6H1^\D +M,#X7\>\,NQ2.]95_3M#!#:N(IC^E6(GHYRO+<3#5PU)A=@0# +M%[=9[_H%";U],PMX`<<#,:#K?!05",0@;1R(1:[N>SE]`N+W-AW20?"87U?W +MK[3+[+`]MO)(%2XN0H\@:S$OM_]EM=8!3:[TE;@FRJNCP6'V]PP'1<*B?KO! +MU0BW]X&T&QIBOH<8K/(#;Z@8%W[DDI%8F[B.+3D$C,J"\(`6XIX"'LOA0+`7 +MB)&X$P@K>MMV>X/MQ8*7P8-UY,(L_Q?%H5JB#>I4V^NXD34"X,<3K.RIA,_` +M?(#'XE!-6,8R!_)8X4;0H%5C7*"G;B"^H.GQJA^21KE)P!?PA)KD#RQ&%K(I +MX\:E'!D\^5@D'RPX,&RVW'G!BHTTI-]S4 +MD8845O$($J+3Q-60RN",_F/"AL>>K)JT2M]0$E`:/I!9&4\N/KB'Y^@,IEH' +M24"9!VN@A)H:Q?\A`.0A848 +M(`]O]_OU)'H%IJ@(?-M*9'K)&RCLM@T%=-)*31^"9Z1TS:;@S5]#J4..*-FE +M,/B;48CTK8-ZC76,F34<-E(M:`YL4:7L>QH0(`<:3R#$LW-\-LR\E^AG; +MI)[_`WT'-5!+`P04````70"(.)%-,9OE>IH!``!4`P``!P```'1S+F-O;F8H +MM2_]`%B-#`#6G$T?4$W3!D_TVC/9!E7:MTFZHMQL#\'FCNWU"^^36;A/#48` +M00!&`.H\A"5RGEETHH(5QQ%1V?#T(&R.*[7D]R#N_$0E[^.<6>(Y2SZSOC[& +M-:,6X0J5Q0$*P+/1^N-MYBU7C&((R=%@NK,V;$'M>)X8I2EYSU^EO7):4J]6LF)7DU;\IOL>*,<7S5>#A4! +MW:L#`DX$ID628(-DP6F6,+#@0UNE-8G\PZSD?UZ3A'UL]2XVSW:@7X5C(LE=*@ +M!P0),F$$.4@0"6:1X42K292OO'F(,_2?/@ZHC*Z015*;CIS?-8,)D!W9=FT'5M;6,=/5CUA)1X +M;1VP`U!+`P04````70`],9%-JCN.NAL"``"0`P``!0```'9I;7)C*+4O_0!8 +ME1``)BAL(R!OVP;,O[!>_2;=8,G_;[\MQ(&0),2W`I@S1AQ.5@F0@D!A90!? +M`%X`D5WYN&K*O.'RM]&3E$R[/8?P[:^%R]/I1X^K7-Q;OFKX`7;=U@C#,/O"`#)H;)QI2-@.N&T1@YQ&_GV._G1YXD +M?JL.H3)+> +M0HA=]L*W-DU=-H/GA"![DK:O4K5D;`/)(($I1$'Q*(3?N9+4-&56=M8[E(LS +MJG#7>UL":)I"=+U%]NYM]&WW&F7:+DE:^!JR,X7V2L&!PG1O=$9WV9V/NX-*X1M1+VRKO;3'K*B90*UUR:1[DI0"(B`01HSB].2BMOHM +M2-FK.-R5FP@><,!Y3%/YO+X@=`E.FECPPNF$DRX%=@XKJ)Z(S2U(`"7G +MP@HS"4BYQST3`Y#41XAALDF&&AR*Y#*7@OSV%7.AC5\0]IEE%P502P$"/P,4 +M````70"1.)%-GC?7C1X+```K&@``"P``````````````I($`````IH!``!4`P``!P`````````````` +MI(%'"P``=',N8V]N9E!+`0(_`Q0```!=`#TQD4VJ.XZZ&P(``)`#```%```` +I``````````"D@08-``!V:6UR8U!+!08``````P`#`*$```!$#P`````` +` +end