From: Michihiro NAKAJIMA Date: Mon, 19 Dec 2011 23:30:59 +0000 (-0500) Subject: Add support for PPMd to 7-Zip writer. X-Git-Tag: v3.0.2~24 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d92f5dd4e148b6d7fcb7f517ba1810ef58022273;p=thirdparty%2Flibarchive.git Add support for PPMd to 7-Zip writer. SVN-Revision: 3950 --- diff --git a/libarchive/archive_ppmd7.c b/libarchive/archive_ppmd7.c index 8aa492e3e..b2e8c3a34 100644 --- a/libarchive/archive_ppmd7.c +++ b/libarchive/archive_ppmd7.c @@ -965,6 +965,188 @@ static int Ppmd7_DecodeSymbol(CPpmd7 *p, IPpmd7_RangeDec *rc) } } +/* ---------- Encode ---------- Ppmd7Enc.c */ + +#define kTopValue (1 << 24) + +static void Ppmd7z_RangeEnc_Init(CPpmd7z_RangeEnc *p) +{ + p->Low = 0; + p->Range = 0xFFFFFFFF; + p->Cache = 0; + p->CacheSize = 1; +} + +static void RangeEnc_ShiftLow(CPpmd7z_RangeEnc *p) +{ + if ((UInt32)p->Low < (UInt32)0xFF000000 || (unsigned)(p->Low >> 32) != 0) + { + Byte temp = p->Cache; + do + { + p->Stream->Write(p->Stream, (Byte)(temp + (Byte)(p->Low >> 32))); + temp = 0xFF; + } + while(--p->CacheSize != 0); + p->Cache = (Byte)((UInt32)p->Low >> 24); + } + p->CacheSize++; + p->Low = (UInt32)p->Low << 8; +} + +static void RangeEnc_Encode(CPpmd7z_RangeEnc *p, UInt32 start, UInt32 size, UInt32 total) +{ + p->Low += start * (p->Range /= total); + p->Range *= size; + while (p->Range < kTopValue) + { + p->Range <<= 8; + RangeEnc_ShiftLow(p); + } +} + +static void RangeEnc_EncodeBit_0(CPpmd7z_RangeEnc *p, UInt32 size0) +{ + p->Range = (p->Range >> 14) * size0; + while (p->Range < kTopValue) + { + p->Range <<= 8; + RangeEnc_ShiftLow(p); + } +} + +static void RangeEnc_EncodeBit_1(CPpmd7z_RangeEnc *p, UInt32 size0) +{ + UInt32 newBound = (p->Range >> 14) * size0; + p->Low += newBound; + p->Range -= newBound; + while (p->Range < kTopValue) + { + p->Range <<= 8; + RangeEnc_ShiftLow(p); + } +} + +static void Ppmd7z_RangeEnc_FlushData(CPpmd7z_RangeEnc *p) +{ + unsigned i; + for (i = 0; i < 5; i++) + RangeEnc_ShiftLow(p); +} + + +#define MASK(sym) ((signed char *)charMask)[sym] + +static void Ppmd7_EncodeSymbol(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol) +{ + size_t charMask[256 / sizeof(size_t)]; + if (p->MinContext->NumStats != 1) + { + CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext); + UInt32 sum; + unsigned i; + if (s->Symbol == symbol) + { + RangeEnc_Encode(rc, 0, s->Freq, p->MinContext->SummFreq); + p->FoundState = s; + Ppmd7_Update1_0(p); + return; + } + p->PrevSuccess = 0; + sum = s->Freq; + i = p->MinContext->NumStats - 1; + do + { + if ((++s)->Symbol == symbol) + { + RangeEnc_Encode(rc, sum, s->Freq, p->MinContext->SummFreq); + p->FoundState = s; + Ppmd7_Update1(p); + return; + } + sum += s->Freq; + } + while (--i); + + p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]; + PPMD_SetAllBitsIn256Bytes(charMask); + MASK(s->Symbol) = 0; + i = p->MinContext->NumStats - 1; + do { MASK((--s)->Symbol) = 0; } while (--i); + RangeEnc_Encode(rc, sum, p->MinContext->SummFreq - sum, p->MinContext->SummFreq); + } + else + { + UInt16 *prob = Ppmd7_GetBinSumm(p); + CPpmd_State *s = Ppmd7Context_OneState(p->MinContext); + if (s->Symbol == symbol) + { + RangeEnc_EncodeBit_0(rc, *prob); + *prob = (UInt16)PPMD_UPDATE_PROB_0(*prob); + p->FoundState = s; + Ppmd7_UpdateBin(p); + return; + } + else + { + RangeEnc_EncodeBit_1(rc, *prob); + *prob = (UInt16)PPMD_UPDATE_PROB_1(*prob); + p->InitEsc = PPMD7_kExpEscape[*prob >> 10]; + PPMD_SetAllBitsIn256Bytes(charMask); + MASK(s->Symbol) = 0; + p->PrevSuccess = 0; + } + } + for (;;) + { + UInt32 escFreq; + CPpmd_See *see; + CPpmd_State *s; + UInt32 sum; + unsigned i, numMasked = p->MinContext->NumStats; + do + { + p->OrderFall++; + if (!p->MinContext->Suffix) + return; /* EndMarker (symbol = -1) */ + p->MinContext = Ppmd7_GetContext(p, p->MinContext->Suffix); + } + while (p->MinContext->NumStats == numMasked); + + see = Ppmd7_MakeEscFreq(p, numMasked, &escFreq); + s = Ppmd7_GetStats(p, p->MinContext); + sum = 0; + i = p->MinContext->NumStats; + do + { + int cur = s->Symbol; + if (cur == symbol) + { + UInt32 low = sum; + CPpmd_State *s1 = s; + do + { + sum += (s->Freq & (int)(MASK(s->Symbol))); + s++; + } + while (--i); + RangeEnc_Encode(rc, low, s1->Freq, sum + escFreq); + Ppmd_See_Update(see); + p->FoundState = s1; + Ppmd7_Update2(p); + return; + } + sum += (s->Freq & (int)(MASK(cur))); + MASK(cur) = 0; + s++; + } + while (--i); + + RangeEnc_Encode(rc, sum, escFreq, sum + escFreq); + see->Summ = (UInt16)(see->Summ + sum + escFreq); + } +} + const IPpmd7 __archive_ppmd7_functions = { &Ppmd7_Construct, @@ -975,5 +1157,8 @@ const IPpmd7 __archive_ppmd7_functions = &PpmdRAR_RangeDec_CreateVTable, &Ppmd7z_RangeDec_Init, &PpmdRAR_RangeDec_Init, - &Ppmd7_DecodeSymbol + &Ppmd7_DecodeSymbol, + &Ppmd7z_RangeEnc_Init, + &Ppmd7z_RangeEnc_FlushData, + &Ppmd7_EncodeSymbol }; diff --git a/libarchive/archive_ppmd7_private.h b/libarchive/archive_ppmd7_private.h index e9915f34a..3a6b9eb41 100644 --- a/libarchive/archive_ppmd7_private.h +++ b/libarchive/archive_ppmd7_private.h @@ -82,14 +82,14 @@ typedef struct /* ---------- Encode ---------- */ -/*typedef struct +typedef struct { UInt64 Low; UInt32 Range; Byte Cache; UInt64 CacheSize; IByteOut *Stream; -} CPpmd7z_RangeEnc;*/ +} CPpmd7z_RangeEnc; typedef struct { @@ -109,10 +109,10 @@ typedef struct int (*Ppmd7_DecodeSymbol)(CPpmd7 *p, IPpmd7_RangeDec *rc); /* Encode Functions */ - /*void (*Ppmd7z_RangeEnc_Init)(CPpmd7z_RangeEnc *p); + void (*Ppmd7z_RangeEnc_Init)(CPpmd7z_RangeEnc *p); void (*Ppmd7z_RangeEnc_FlushData)(CPpmd7z_RangeEnc *p); - void (*Ppmd7_EncodeSymbol)(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol);*/ + void (*Ppmd7_EncodeSymbol)(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol); } IPpmd7; extern const IPpmd7 __archive_ppmd7_functions; diff --git a/libarchive/archive_ppmd_private.h b/libarchive/archive_ppmd_private.h index 0975f6940..266676878 100644 --- a/libarchive/archive_ppmd_private.h +++ b/libarchive/archive_ppmd_private.h @@ -65,7 +65,7 @@ typedef struct typedef struct { - struct archive_read *a; + struct archive_write *a; void (*Write)(void *p, Byte b); } IByteOut; diff --git a/libarchive/archive_write_set_format_7zip.c b/libarchive/archive_write_set_format_7zip.c index 3aad75a47..f405d1f92 100644 --- a/libarchive/archive_write_set_format_7zip.c +++ b/libarchive/archive_write_set_format_7zip.c @@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$"); #include "archive_endian.h" #include "archive_entry.h" #include "archive_entry_locale.h" +#include "archive_ppmd7_private.h" #include "archive_private.h" #include "archive_rb.h" #include "archive_string.h" @@ -60,6 +61,7 @@ __FBSDID("$FreeBSD$"); #define _7Z_LZMA2 0x21 #define _7Z_DEFLATE 0x040108 #define _7Z_BZIP2 0x040202 +#define _7Z_PPMD 0x030401 /* * 7-Zip header property IDs. @@ -97,11 +99,11 @@ enum la_zaction { * Universal zstream. */ struct la_zstream { - const unsigned char *next_in; + const uint8_t *next_in; size_t avail_in; uint64_t total_in; - unsigned char *next_out; + uint8_t *next_out; size_t avail_out; uint64_t total_out; @@ -117,6 +119,20 @@ struct la_zstream { struct la_zstream *lastrm); }; +#define PPMD7_DEFAULT_ORDER 6 +#define PPMD7_DEFAULT_MEM_SIZE (1 << 24) + +struct ppmd_stream { + int stat; + CPpmd7 ppmd7_context; + CPpmd7z_RangeEnc range_enc; + IByteOut byteout; + uint8_t *buff; + uint8_t *buff_ptr; + uint8_t *buff_end; + size_t buff_bytes; +}; + struct coder { unsigned codec; size_t prop_size; @@ -248,6 +264,11 @@ static int compression_code_lzma(struct archive *, struct la_zstream *, enum la_zaction); static int compression_end_lzma(struct archive *, struct la_zstream *); #endif +static int compression_init_encoder_ppmd(struct archive *, + struct la_zstream *, unsigned, uint32_t); +static int compression_code_ppmd(struct archive *, + struct la_zstream *, enum la_zaction); +static int compression_end_ppmd(struct archive *, struct la_zstream *); static int _7z_compression_init_encoder(struct archive_write *, unsigned, int); static int compression_code(struct archive *, @@ -357,6 +378,10 @@ _7z_options(struct archive_write *a, const char *key, const char *value) #else name = "lzma2"; #endif + else if (strcmp(value, "ppmd") == 0 || + strcmp(value, "PPMD") == 0 || + strcmp(value, "PPMd") == 0) + zip->opt_compression = _7Z_PPMD; else { archive_set_error(&(a->archive), ARCHIVE_ERRNO_MISC, @@ -526,8 +551,8 @@ compress_out(struct archive_write *a, const void *buff, size_t s, if ((zip->crc32flg & PRECODE_CRC32) && s) zip->precode_crc32 = crc32(zip->precode_crc32, buff, s); zip->stream.next_in = (const unsigned char *)buff; + zip->stream.avail_in = s; do { - zip->stream.avail_in = s; /* Compress file data. */ r = compression_code(&(a->archive), &(zip->stream), run); if (r != ARCHIVE_OK && r != ARCHIVE_EOF) @@ -2021,6 +2046,161 @@ compression_init_encoder_lzma2(struct archive *a, } #endif +/* + * PPMd compression. + */ +static void * +ppmd_alloc(void *p, size_t size) +{ + (void)p; + return malloc(size); +} +static void +ppmd_free(void *p, void *address) +{ + (void)p; + free(address); +} +static ISzAlloc g_szalloc = { ppmd_alloc, ppmd_free }; +static void +ppmd_write(void *p, Byte b) +{ + struct archive_write *a = ((IByteOut *)p)->a; + struct _7zip *zip = (struct _7zip *)(a->format_data); + struct la_zstream *lastrm = &(zip->stream); + struct ppmd_stream *strm; + + if (lastrm->avail_out) { + *lastrm->next_out++ = b; + lastrm->avail_out--; + lastrm->total_out++; + return; + } + strm = (struct ppmd_stream *)lastrm->real_stream; + if (strm->buff_ptr < strm->buff_end) { + *strm->buff_ptr++ = b; + strm->buff_bytes++; + } +} + +static int +compression_init_encoder_ppmd(struct archive *a, + struct la_zstream *lastrm, unsigned maxOrder, uint32_t msize) +{ + struct ppmd_stream *strm; + uint8_t *props; + int r; + + if (lastrm->valid) + compression_end(a, lastrm); + strm = calloc(1, sizeof(*strm)); + if (strm == NULL) { + archive_set_error(a, ENOMEM, + "Can't allocate memory for PPMd"); + return (ARCHIVE_FATAL); + } + strm->buff = malloc(32); + if (strm->buff == NULL) { + free(strm); + archive_set_error(a, ENOMEM, + "Can't allocate memory for PPMd"); + return (ARCHIVE_FATAL); + } + strm->buff_ptr = strm->buff; + strm->buff_end = strm->buff + 32; + + props = malloc(1+4); + if (props == NULL) { + free(strm->buff); + free(strm); + archive_set_error(a, ENOMEM, + "Coludn't allocate memory for PPMd"); + return (ARCHIVE_FATAL); + } + props[0] = maxOrder; + archive_le32enc(props+1, msize); + __archive_ppmd7_functions.Ppmd7_Construct(&strm->ppmd7_context); + r = __archive_ppmd7_functions.Ppmd7_Alloc( + &strm->ppmd7_context, msize, &g_szalloc); + if (r == 0) { + free(strm->buff); + free(strm); + free(props); + archive_set_error(a, ENOMEM, + "Coludn't allocate memory for PPMd"); + return (ARCHIVE_FATAL); + } + __archive_ppmd7_functions.Ppmd7_Init(&(strm->ppmd7_context), maxOrder); + strm->byteout.a = (struct archive_write *)a; + strm->byteout.Write = ppmd_write; + strm->range_enc.Stream = &(strm->byteout); + __archive_ppmd7_functions.Ppmd7z_RangeEnc_Init(&(strm->range_enc)); + strm->stat = 0; + + lastrm->real_stream = strm; + lastrm->valid = 1; + lastrm->code = compression_code_ppmd; + lastrm->end = compression_end_ppmd; + lastrm->prop_size = 5; + lastrm->props = props; + return (ARCHIVE_OK); +} + +static int +compression_code_ppmd(struct archive *a, + struct la_zstream *lastrm, enum la_zaction action) +{ + struct ppmd_stream *strm; + + strm = (struct ppmd_stream *)lastrm->real_stream; + + /* Copy encoded data if there are remaining bytes from previous call. */ + if (strm->buff_bytes) { + uint8_t *p = strm->buff_ptr - strm->buff_bytes; + while (lastrm->avail_out && strm->buff_bytes) { + *lastrm->next_out++ = *p++; + lastrm->avail_out--; + lastrm->total_out++; + strm->buff_bytes--; + } + if (strm->buff_bytes) + return (ARCHIVE_OK); + if (strm->stat == 1) + return (ARCHIVE_EOF); + strm->buff_ptr = strm->buff; + } + while (lastrm->avail_in && lastrm->avail_out) { + __archive_ppmd7_functions.Ppmd7_EncodeSymbol( + &(strm->ppmd7_context), &(strm->range_enc), + *lastrm->next_in++); + lastrm->avail_in--; + lastrm->total_in++; + } + if (lastrm->avail_in == 0 && action == ARCHIVE_Z_FINISH) { + __archive_ppmd7_functions.Ppmd7z_RangeEnc_FlushData( + &(strm->range_enc)); + strm->stat = 1; + /* Return EOF if there are no remaining bytes. */ + if (strm->buff_bytes == 0) + return (ARCHIVE_EOF); + } + return (ARCHIVE_OK); +} + +static int +compression_end_ppmd(struct archive *a, struct la_zstream *lastrm) +{ + struct ppmd_stream *strm; + + strm = (struct ppmd_stream *)lastrm->real_stream; + __archive_ppmd7_functions.Ppmd7_Free(&strm->ppmd7_context, &g_szalloc); + free(strm->buff); + free(strm); + lastrm->real_stream = NULL; + lastrm->valid = 0; + return (ARCHIVE_OK); +} + static int _7z_compression_init_encoder(struct archive_write *a, unsigned compression, int compression_level) @@ -2050,6 +2230,11 @@ _7z_compression_init_encoder(struct archive_write *a, unsigned compression, &(a->archive), &(zip->stream), compression_level); break; + case _7Z_PPMD: + r = compression_init_encoder_ppmd( + &(a->archive), &(zip->stream), + PPMD7_DEFAULT_ORDER, PPMD7_DEFAULT_MEM_SIZE); + break; case _7Z_COPY: default: r = compression_init_encoder_copy( diff --git a/libarchive/test/test_write_format_7zip.c b/libarchive/test/test_write_format_7zip.c index 28043af5d..5015af42b 100644 --- a/libarchive/test/test_write_format_7zip.c +++ b/libarchive/test/test_write_format_7zip.c @@ -597,6 +597,8 @@ DEFINE_TEST(test_write_format_7zip) test_basic("lzma1"); /* Test that making a 7-Zip archive file with lzma2 compression. */ test_basic("lzma2"); + /* Test that making a 7-Zip archive file with PPMd compression. */ + test_basic("ppmd"); /* Test that making an empty 7-Zip archive file. */ test_empty_archive(); /* Test that write an empty file. */