]> git.ipfire.org Git - thirdparty/libarchive.git/commitdiff
Add support for PPMd to 7-Zip writer.
authorMichihiro NAKAJIMA <ggcueroad@gmail.com>
Mon, 19 Dec 2011 23:30:59 +0000 (18:30 -0500)
committerMichihiro NAKAJIMA <ggcueroad@gmail.com>
Mon, 19 Dec 2011 23:30:59 +0000 (18:30 -0500)
SVN-Revision: 3950

libarchive/archive_ppmd7.c
libarchive/archive_ppmd7_private.h
libarchive/archive_ppmd_private.h
libarchive/archive_write_set_format_7zip.c
libarchive/test/test_write_format_7zip.c

index 8aa492e3e995fca20b8c389eb36c826fc109d746..b2e8c3a3492238c187a8e68c83c90f98e24030e0 100644 (file)
@@ -965,6 +965,188 @@ static int Ppmd7_DecodeSymbol(CPpmd7 *p, IPpmd7_RangeDec *rc)
   }
 }
 
+/* ---------- Encode ---------- Ppmd7Enc.c */
+
+#define kTopValue (1 << 24)
+
+static void Ppmd7z_RangeEnc_Init(CPpmd7z_RangeEnc *p)
+{
+  p->Low = 0;
+  p->Range = 0xFFFFFFFF;
+  p->Cache = 0;
+  p->CacheSize = 1;
+}
+
+static void RangeEnc_ShiftLow(CPpmd7z_RangeEnc *p)
+{
+  if ((UInt32)p->Low < (UInt32)0xFF000000 || (unsigned)(p->Low >> 32) != 0)
+  {
+    Byte temp = p->Cache;
+    do
+    {
+      p->Stream->Write(p->Stream, (Byte)(temp + (Byte)(p->Low >> 32)));
+      temp = 0xFF;
+    }
+    while(--p->CacheSize != 0);
+    p->Cache = (Byte)((UInt32)p->Low >> 24);
+  }
+  p->CacheSize++;
+  p->Low = (UInt32)p->Low << 8;
+}
+
+static void RangeEnc_Encode(CPpmd7z_RangeEnc *p, UInt32 start, UInt32 size, UInt32 total)
+{
+  p->Low += start * (p->Range /= total);
+  p->Range *= size;
+  while (p->Range < kTopValue)
+  {
+    p->Range <<= 8;
+    RangeEnc_ShiftLow(p);
+  }
+}
+
+static void RangeEnc_EncodeBit_0(CPpmd7z_RangeEnc *p, UInt32 size0)
+{
+  p->Range = (p->Range >> 14) * size0;
+  while (p->Range < kTopValue)
+  {
+    p->Range <<= 8;
+    RangeEnc_ShiftLow(p);
+  }
+}
+
+static void RangeEnc_EncodeBit_1(CPpmd7z_RangeEnc *p, UInt32 size0)
+{
+  UInt32 newBound = (p->Range >> 14) * size0;
+  p->Low += newBound;
+  p->Range -= newBound;
+  while (p->Range < kTopValue)
+  {
+    p->Range <<= 8;
+    RangeEnc_ShiftLow(p);
+  }
+}
+
+static void Ppmd7z_RangeEnc_FlushData(CPpmd7z_RangeEnc *p)
+{
+  unsigned i;
+  for (i = 0; i < 5; i++)
+    RangeEnc_ShiftLow(p);
+}
+
+
+#define MASK(sym) ((signed char *)charMask)[sym]
+
+static void Ppmd7_EncodeSymbol(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol)
+{
+  size_t charMask[256 / sizeof(size_t)];
+  if (p->MinContext->NumStats != 1)
+  {
+    CPpmd_State *s = Ppmd7_GetStats(p, p->MinContext);
+    UInt32 sum;
+    unsigned i;
+    if (s->Symbol == symbol)
+    {
+      RangeEnc_Encode(rc, 0, s->Freq, p->MinContext->SummFreq);
+      p->FoundState = s;
+      Ppmd7_Update1_0(p);
+      return;
+    }
+    p->PrevSuccess = 0;
+    sum = s->Freq;
+    i = p->MinContext->NumStats - 1;
+    do
+    {
+      if ((++s)->Symbol == symbol)
+      {
+        RangeEnc_Encode(rc, sum, s->Freq, p->MinContext->SummFreq);
+        p->FoundState = s;
+        Ppmd7_Update1(p);
+        return;
+      }
+      sum += s->Freq;
+    }
+    while (--i);
+    
+    p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol];
+    PPMD_SetAllBitsIn256Bytes(charMask);
+    MASK(s->Symbol) = 0;
+    i = p->MinContext->NumStats - 1;
+    do { MASK((--s)->Symbol) = 0; } while (--i);
+    RangeEnc_Encode(rc, sum, p->MinContext->SummFreq - sum, p->MinContext->SummFreq);
+  }
+  else
+  {
+    UInt16 *prob = Ppmd7_GetBinSumm(p);
+    CPpmd_State *s = Ppmd7Context_OneState(p->MinContext);
+    if (s->Symbol == symbol)
+    {
+      RangeEnc_EncodeBit_0(rc, *prob);
+      *prob = (UInt16)PPMD_UPDATE_PROB_0(*prob);
+      p->FoundState = s;
+      Ppmd7_UpdateBin(p);
+      return;
+    }
+    else
+    {
+      RangeEnc_EncodeBit_1(rc, *prob);
+      *prob = (UInt16)PPMD_UPDATE_PROB_1(*prob);
+      p->InitEsc = PPMD7_kExpEscape[*prob >> 10];
+      PPMD_SetAllBitsIn256Bytes(charMask);
+      MASK(s->Symbol) = 0;
+      p->PrevSuccess = 0;
+    }
+  }
+  for (;;)
+  {
+    UInt32 escFreq;
+    CPpmd_See *see;
+    CPpmd_State *s;
+    UInt32 sum;
+    unsigned i, numMasked = p->MinContext->NumStats;
+    do
+    {
+      p->OrderFall++;
+      if (!p->MinContext->Suffix)
+        return; /* EndMarker (symbol = -1) */
+      p->MinContext = Ppmd7_GetContext(p, p->MinContext->Suffix);
+    }
+    while (p->MinContext->NumStats == numMasked);
+    
+    see = Ppmd7_MakeEscFreq(p, numMasked, &escFreq);
+    s = Ppmd7_GetStats(p, p->MinContext);
+    sum = 0;
+    i = p->MinContext->NumStats;
+    do
+    {
+      int cur = s->Symbol;
+      if (cur == symbol)
+      {
+        UInt32 low = sum;
+        CPpmd_State *s1 = s;
+        do
+        {
+          sum += (s->Freq & (int)(MASK(s->Symbol)));
+          s++;
+        }
+        while (--i);
+        RangeEnc_Encode(rc, low, s1->Freq, sum + escFreq);
+        Ppmd_See_Update(see);
+        p->FoundState = s1;
+        Ppmd7_Update2(p);
+        return;
+      }
+      sum += (s->Freq & (int)(MASK(cur)));
+      MASK(cur) = 0;
+      s++;
+    }
+    while (--i);
+    
+    RangeEnc_Encode(rc, sum, escFreq, sum + escFreq);
+    see->Summ = (UInt16)(see->Summ + sum + escFreq);
+  }
+}
+
 const IPpmd7 __archive_ppmd7_functions =
 {
   &Ppmd7_Construct,
@@ -975,5 +1157,8 @@ const IPpmd7 __archive_ppmd7_functions =
   &PpmdRAR_RangeDec_CreateVTable,
   &Ppmd7z_RangeDec_Init,
   &PpmdRAR_RangeDec_Init,
-  &Ppmd7_DecodeSymbol
+  &Ppmd7_DecodeSymbol,
+  &Ppmd7z_RangeEnc_Init,
+  &Ppmd7z_RangeEnc_FlushData,
+  &Ppmd7_EncodeSymbol
 };
index e9915f34aba66693a74a2e6415fc5dee7be35d90..3a6b9eb4190f79f0e35668d6a9fd08403514eec9 100644 (file)
@@ -82,14 +82,14 @@ typedef struct
 
 /* ---------- Encode ---------- */
 
-/*typedef struct
+typedef struct
 {
   UInt64 Low;
   UInt32 Range;
   Byte Cache;
   UInt64 CacheSize;
   IByteOut *Stream;
-} CPpmd7z_RangeEnc;*/
+} CPpmd7z_RangeEnc;
 
 typedef struct
 {
@@ -109,10 +109,10 @@ typedef struct
   int (*Ppmd7_DecodeSymbol)(CPpmd7 *p, IPpmd7_RangeDec *rc);
 
   /* Encode Functions */
-  /*void (*Ppmd7z_RangeEnc_Init)(CPpmd7z_RangeEnc *p);
+  void (*Ppmd7z_RangeEnc_Init)(CPpmd7z_RangeEnc *p);
   void (*Ppmd7z_RangeEnc_FlushData)(CPpmd7z_RangeEnc *p);
 
-  void (*Ppmd7_EncodeSymbol)(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol);*/
+  void (*Ppmd7_EncodeSymbol)(CPpmd7 *p, CPpmd7z_RangeEnc *rc, int symbol);
 } IPpmd7;
 
 extern const IPpmd7 __archive_ppmd7_functions;
index 0975f6940ab6f51c054f1c3091de1cea6605a524..26667687855740dead57d4d9dfb08c708c25b080 100644 (file)
@@ -65,7 +65,7 @@ typedef struct
 
 typedef struct
 {
-  struct archive_read *a;
+  struct archive_write *a;
   void (*Write)(void *p, Byte b);
 } IByteOut;
 
index 3aad75a47f64e07fd2f69f13306559805bcb0b91..f405d1f928834bf448c8e5916bad11f50e691548 100644 (file)
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include "archive_endian.h"
 #include "archive_entry.h"
 #include "archive_entry_locale.h"
+#include "archive_ppmd7_private.h"
 #include "archive_private.h"
 #include "archive_rb.h"
 #include "archive_string.h"
@@ -60,6 +61,7 @@ __FBSDID("$FreeBSD$");
 #define _7Z_LZMA2      0x21
 #define _7Z_DEFLATE    0x040108
 #define _7Z_BZIP2      0x040202
+#define _7Z_PPMD       0x030401
 
 /*
  * 7-Zip header property IDs.
@@ -97,11 +99,11 @@ enum la_zaction {
  * Universal zstream.
  */
 struct la_zstream {
-       const unsigned char     *next_in;
+       const uint8_t           *next_in;
        size_t                   avail_in;
        uint64_t                 total_in;
 
-       unsigned char           *next_out;
+       uint8_t                 *next_out;
        size_t                   avail_out;
        uint64_t                 total_out;
 
@@ -117,6 +119,20 @@ struct la_zstream {
                                    struct la_zstream *lastrm);
 };
 
+#define PPMD7_DEFAULT_ORDER    6
+#define PPMD7_DEFAULT_MEM_SIZE (1 << 24)
+
+struct ppmd_stream {
+       int                      stat;
+       CPpmd7                   ppmd7_context;
+       CPpmd7z_RangeEnc         range_enc;
+       IByteOut                 byteout;
+       uint8_t                 *buff;
+       uint8_t                 *buff_ptr;
+       uint8_t                 *buff_end;
+       size_t                   buff_bytes;
+};
+
 struct coder {
        unsigned                 codec;
        size_t                   prop_size;
@@ -248,6 +264,11 @@ static int compression_code_lzma(struct archive *,
                    struct la_zstream *, enum la_zaction);
 static int     compression_end_lzma(struct archive *, struct la_zstream *);
 #endif
+static int     compression_init_encoder_ppmd(struct archive *,
+                   struct la_zstream *, unsigned, uint32_t);
+static int     compression_code_ppmd(struct archive *,
+                   struct la_zstream *, enum la_zaction);
+static int     compression_end_ppmd(struct archive *, struct la_zstream *);
 static int     _7z_compression_init_encoder(struct archive_write *, unsigned,
                    int);
 static int     compression_code(struct archive *,
@@ -357,6 +378,10 @@ _7z_options(struct archive_write *a, const char *key, const char *value)
 #else
                        name = "lzma2";
 #endif
+               else if (strcmp(value, "ppmd") == 0 ||
+                   strcmp(value, "PPMD") == 0 ||
+                   strcmp(value, "PPMd") == 0)
+                       zip->opt_compression = _7Z_PPMD;
                else {
                        archive_set_error(&(a->archive),
                            ARCHIVE_ERRNO_MISC,
@@ -526,8 +551,8 @@ compress_out(struct archive_write *a, const void *buff, size_t s,
        if ((zip->crc32flg & PRECODE_CRC32) && s)
                zip->precode_crc32 = crc32(zip->precode_crc32, buff, s);
        zip->stream.next_in = (const unsigned char *)buff;
+       zip->stream.avail_in = s;
        do {
-               zip->stream.avail_in = s;
                /* Compress file data. */
                r = compression_code(&(a->archive), &(zip->stream), run);
                if (r != ARCHIVE_OK && r != ARCHIVE_EOF)
@@ -2021,6 +2046,161 @@ compression_init_encoder_lzma2(struct archive *a,
 }
 #endif
 
+/*
+ * PPMd compression.
+ */
+static void *
+ppmd_alloc(void *p, size_t size)
+{
+       (void)p;
+       return malloc(size);
+}
+static void
+ppmd_free(void *p, void *address)
+{
+       (void)p;
+       free(address);
+}
+static ISzAlloc g_szalloc = { ppmd_alloc, ppmd_free };
+static void
+ppmd_write(void *p, Byte b)
+{
+       struct archive_write *a = ((IByteOut *)p)->a;
+       struct _7zip *zip = (struct _7zip *)(a->format_data);
+       struct la_zstream *lastrm = &(zip->stream);
+       struct ppmd_stream *strm;
+
+       if (lastrm->avail_out) {
+               *lastrm->next_out++ = b;
+               lastrm->avail_out--;
+               lastrm->total_out++;
+               return;
+       }
+       strm = (struct ppmd_stream *)lastrm->real_stream;
+       if (strm->buff_ptr < strm->buff_end) {
+               *strm->buff_ptr++ = b;
+               strm->buff_bytes++;
+       }
+}
+
+static int
+compression_init_encoder_ppmd(struct archive *a,
+    struct la_zstream *lastrm, unsigned maxOrder, uint32_t msize)
+{
+       struct ppmd_stream *strm;
+       uint8_t *props;
+       int r;
+
+       if (lastrm->valid)
+               compression_end(a, lastrm);
+       strm = calloc(1, sizeof(*strm));
+       if (strm == NULL) {
+               archive_set_error(a, ENOMEM,
+                   "Can't allocate memory for PPMd");
+               return (ARCHIVE_FATAL);
+       }
+       strm->buff = malloc(32);
+       if (strm->buff == NULL) {
+               free(strm);
+               archive_set_error(a, ENOMEM,
+                   "Can't allocate memory for PPMd");
+               return (ARCHIVE_FATAL);
+       }
+       strm->buff_ptr = strm->buff;
+       strm->buff_end = strm->buff + 32;
+
+       props = malloc(1+4);
+       if (props == NULL) {
+               free(strm->buff);
+               free(strm);
+               archive_set_error(a, ENOMEM,
+                   "Coludn't allocate memory for PPMd");
+               return (ARCHIVE_FATAL);
+       }
+       props[0] = maxOrder;
+       archive_le32enc(props+1, msize);
+       __archive_ppmd7_functions.Ppmd7_Construct(&strm->ppmd7_context);
+       r = __archive_ppmd7_functions.Ppmd7_Alloc(
+               &strm->ppmd7_context, msize, &g_szalloc);
+       if (r == 0) {
+               free(strm->buff);
+               free(strm);
+               free(props);
+               archive_set_error(a, ENOMEM,
+                   "Coludn't allocate memory for PPMd");
+               return (ARCHIVE_FATAL);
+       }
+       __archive_ppmd7_functions.Ppmd7_Init(&(strm->ppmd7_context), maxOrder);
+       strm->byteout.a = (struct archive_write *)a;
+       strm->byteout.Write = ppmd_write;
+       strm->range_enc.Stream = &(strm->byteout);
+       __archive_ppmd7_functions.Ppmd7z_RangeEnc_Init(&(strm->range_enc));
+       strm->stat = 0;
+
+       lastrm->real_stream = strm;
+       lastrm->valid = 1;
+       lastrm->code = compression_code_ppmd;
+       lastrm->end = compression_end_ppmd;
+       lastrm->prop_size = 5;
+       lastrm->props = props;
+       return (ARCHIVE_OK);
+}
+
+static int
+compression_code_ppmd(struct archive *a,
+    struct la_zstream *lastrm, enum la_zaction action)
+{
+       struct ppmd_stream *strm;
+
+       strm = (struct ppmd_stream *)lastrm->real_stream;
+
+       /* Copy encoded data if there are remaining bytes from previous call. */
+       if (strm->buff_bytes) {
+               uint8_t *p = strm->buff_ptr - strm->buff_bytes;
+               while (lastrm->avail_out && strm->buff_bytes) {
+                       *lastrm->next_out++ = *p++;
+                       lastrm->avail_out--;
+                       lastrm->total_out++;
+                       strm->buff_bytes--;
+               }
+               if (strm->buff_bytes)
+                       return (ARCHIVE_OK);
+               if (strm->stat == 1)
+                       return (ARCHIVE_EOF);
+               strm->buff_ptr = strm->buff;
+       }
+       while (lastrm->avail_in && lastrm->avail_out) {
+               __archive_ppmd7_functions.Ppmd7_EncodeSymbol(
+                       &(strm->ppmd7_context), &(strm->range_enc),
+                       *lastrm->next_in++);
+               lastrm->avail_in--;
+               lastrm->total_in++;
+       }
+       if (lastrm->avail_in == 0 && action == ARCHIVE_Z_FINISH) {
+               __archive_ppmd7_functions.Ppmd7z_RangeEnc_FlushData(
+                       &(strm->range_enc));
+               strm->stat = 1;
+               /* Return EOF if there are no remaining bytes. */
+               if (strm->buff_bytes == 0)
+                       return (ARCHIVE_EOF);
+       }
+       return (ARCHIVE_OK);
+}
+
+static int
+compression_end_ppmd(struct archive *a, struct la_zstream *lastrm)
+{
+       struct ppmd_stream *strm;
+
+       strm = (struct ppmd_stream *)lastrm->real_stream;
+       __archive_ppmd7_functions.Ppmd7_Free(&strm->ppmd7_context, &g_szalloc);
+       free(strm->buff);
+       free(strm);
+       lastrm->real_stream = NULL;
+       lastrm->valid = 0;
+       return (ARCHIVE_OK);
+}
+
 static int
 _7z_compression_init_encoder(struct archive_write *a, unsigned compression,
     int compression_level)
@@ -2050,6 +2230,11 @@ _7z_compression_init_encoder(struct archive_write *a, unsigned compression,
                    &(a->archive), &(zip->stream),
                    compression_level);
                break;
+       case _7Z_PPMD:
+               r = compression_init_encoder_ppmd(
+                   &(a->archive), &(zip->stream),
+                   PPMD7_DEFAULT_ORDER, PPMD7_DEFAULT_MEM_SIZE);
+               break;
        case _7Z_COPY:
        default:
                r = compression_init_encoder_copy(
index 28043af5d66fa418ddec3acc1a60c9a98dccd39d..5015af42bb05fe96b54644ca099bce13a20a9e70 100644 (file)
@@ -597,6 +597,8 @@ DEFINE_TEST(test_write_format_7zip)
        test_basic("lzma1");
        /* Test that making a 7-Zip archive file with lzma2 compression. */
        test_basic("lzma2");
+       /* Test that making a 7-Zip archive file with PPMd compression. */
+       test_basic("ppmd");
        /* Test that making an empty 7-Zip archive file. */
        test_empty_archive();
        /* Test that write an empty file. */