Alloc page compression memory instead of using the stack

musl has a default stack size of just 128K, which is not
enough for our compression data.

diff --git a/src/repo_write.c b/src/repo_write.c
index 5de1bd6f87fe3f676be1e1256b3f45db21a17023..3d14872debbbdf6125027d74634bedc07ec7d471 100644 (file)
--- a/src/repo_write.c
+++ b/src/repo_write.c
@@ -164,7 +164,7 @@ write_blob(Repodata *data, void *blob, int len)
 }
 
 static void
-write_compressed_blob(Repodata *data, void *blob, int len)
+write_compressed_blob(PageCompressor *comp, Repodata *data, void *blob, int len)
 {
   unsigned char cpage[65536];
   if (data->error)
@@ -1007,7 +1007,7 @@ traverse_dirs(Dirpool *dp, Id *dirmap, Id n, Id dir, Id *used)
 }
 
 static void
-write_compressed_page(Repodata *data, unsigned char *page, int len)
+write_compressed_page(PageCompressor *comp, Repodata *data, unsigned char *page, int len)
 {
   int clen;
   unsigned char cpage[REPOPAGE_BLOBSIZE];
@@ -1071,7 +1071,7 @@ repo_write_stdkeyfilter(Repo *repo, Repokey *key, void *kfdata)
 }
 
 static int
-write_compressed_extdata(Repodata *target, struct extdata *xd, unsigned char *vpage, int lpage)
+write_compressed_extdata(PageCompressor *comp, Repodata *target, struct extdata *xd, unsigned char *vpage, int lpage)
 {
   unsigned char *dp = xd->buf;
   int l = xd->len;
@@ -1086,7 +1086,7 @@ write_compressed_extdata(Repodata *target, struct extdata *xd, unsigned char *vp
       l -= ll;
       if (lpage == REPOPAGE_BLOBSIZE)
        {
-         write_compressed_page(target, vpage, lpage);
+         write_compressed_page(comp, target, vpage, lpage);
          lpage = 0;
        }
     }
@@ -2145,7 +2145,11 @@ for (i = 1; i < target.nkeys; i++)
        }
       write_id(&target, cnt);
       if (cnt)
-        write_compressed_blob(&target, xd->buf, xd->len);
+       {
+         PageCompressor *comp = pagecompressor_create();
+         write_compressed_blob(comp, &target, xd->buf, xd->len);
+         pagecompressor_free(comp);
+       }
       solv_free(xd->buf);
     }
 
@@ -2170,13 +2174,14 @@ for (i = 1; i < target.nkeys; i++)
       /* have vertical data, write it in pages */
       unsigned char vpage[REPOPAGE_BLOBSIZE];
       int lpage = 0;
+      PageCompressor *comp = pagecompressor_create();
 
       write_u32(&target, REPOPAGE_BLOBSIZE);
       if (!cbdata.filelistmode)
        {
          for (i = 1; i < target.nkeys; i++)
            if (cbdata.extdata[i].len)
-             lpage = write_compressed_extdata(&target, cbdata.extdata + i, vpage, lpage);
+             lpage = write_compressed_extdata(comp, &target, cbdata.extdata + i, vpage, lpage);
        }
       else
        {
@@ -2207,15 +2212,16 @@ for (i = 1; i < target.nkeys; i++)
                }
              if (xd->len > 1024 * 1024)
                {
-                 lpage = write_compressed_extdata(&target, xd, vpage, lpage);
+                 lpage = write_compressed_extdata(comp, &target, xd, vpage, lpage);
                  xd->len = 0;
                }
            }
          if (xd->len)
-           lpage = write_compressed_extdata(&target, xd, vpage, lpage);
+           lpage = write_compressed_extdata(comp, &target, xd, vpage, lpage);
        }
       if (lpage)
-       write_compressed_page(&target, vpage, lpage);
+       write_compressed_page(comp, &target, vpage, lpage);
+      pagecompressor_free(comp);
     }
 
   for (i = 1; i < target.nkeys; i++)

diff --git a/src/repopage.c b/src/repopage.c
index 77c7fcc7bd2064da93cad5b80d4ad851b5286837..a9a1c074b31259cbd4beccc8dee468e243cf35de 100644 (file)
--- a/src/repopage.c
+++ b/src/repopage.c
@@ -41,6 +41,7 @@
 
 
 
+/* max size of a compression block */
 #define BLOCK_SIZE (65536*1)
 #if BLOCK_SIZE <= 65536
 typedef uint16_t Ref;
@@ -48,6 +49,9 @@ typedef uint16_t Ref;
 typedef uint32_t Ref;
 #endif
 
+/* size of the compression hash table */
+#define HS (65536)
+
 /*
    The format is tailored for fast decompression (i.e. only byte based),
    and skewed to ASCII content (highest bit often not set):
@@ -91,20 +95,25 @@ typedef uint32_t Ref;
      (with the current block size this can't happen)
    L >= 4096+18, so reduce to 4095+18                  : encode as f
 */
+/*
+  compress_buf used to contain:
+    Ref htab[HS];
+    Ref hnext[BLOCK_SIZE];
+  But this uses too much space on the stack, so we now take pointers
+  to allocated space.
+*/
 
 
 static unsigned int
-compress_buf(const unsigned char *in, unsigned int in_len,
+compress_buf(Ref *htab, Ref *hnext,
+             const unsigned char *in, unsigned int in_len,
              unsigned char *out, unsigned int out_len)
 {
   unsigned int oo = 0;         /* out-offset */
   unsigned int io = 0;         /* in-offset */
-#define HS (65536)
-  Ref htab[HS];
-  Ref hnext[BLOCK_SIZE];
   unsigned int litofs = 0;
-  memset(htab, -1, sizeof (htab));
-  memset(hnext, -1, sizeof (hnext));
+  memset(htab, -1, HS * sizeof(*htab));
+  memset(hnext, -1, BLOCK_SIZE * sizeof(*hnext));
   if (in_len > BLOCK_SIZE)
     return 0;                  /* Hey! */
   while (io + 2 < in_len)
@@ -792,10 +801,38 @@ repopagestore_load_page_range(Repopagestore *store, unsigned int pstart, unsigne
   return store->blob_store + best * REPOPAGE_BLOBSIZE;
 }
 
+struct s_PageCompressor {
+  Ref htab[HS];
+  Ref hnext[BLOCK_SIZE];
+};
+
+PageCompressor *
+pagecompressor_create()
+{
+  PageCompressor *comp = solv_malloc(sizeof(PageCompressor));
+  return comp;
+}
+
+PageCompressor *
+pagecompressor_free(PageCompressor *comp)
+{
+  solv_free(comp);
+  return 0;
+}
+
+unsigned int
+pagecompressor_compress(PageCompressor *comp, unsigned char *page, unsigned int len, unsigned char *cpage, unsigned int max)
+{
+  return compress_buf(comp->htab, comp->hnext, page, len, cpage, max);
+}
+
+/* obsolete, to be deleted */
 unsigned int
 repopagestore_compress_page(unsigned char *page, unsigned int len, unsigned char *cpage, unsigned int max)
 {
-  return compress_buf(page, len, cpage, max);
+  Ref htab[HS];
+  Ref hnext[BLOCK_SIZE];
+  return compress_buf(htab, hnext, page, len, cpage, max);
 }
 
 unsigned int
@@ -933,6 +970,8 @@ repopagestore_disable_paging(Repopagestore *store)
 static void
 transfer_file(FILE * from, FILE * to, int compress)
 {
+  Ref htab[HS];
+  Ref hnext[BLOCK_SIZE];
   unsigned char inb[BLOCK_SIZE];
   unsigned char outb[BLOCK_SIZE];
   while (!feof (from) && !ferror (from))
@@ -944,7 +983,7 @@ transfer_file(FILE * from, FILE * to, int compress)
          if (in_len)
            {
              unsigned char *b = outb;
-             out_len = compress_buf(inb, in_len, outb, sizeof (outb));
+             out_len = compress_buf(htab, hnext, inb, in_len, outb, sizeof (outb));
              if (!out_len)
                b = inb, out_len = in_len;
              if (fwrite(&out_len, sizeof (out_len), 1, to) != 1)
@@ -999,6 +1038,8 @@ benchmark(FILE * from)
 {
   unsigned char inb[BLOCK_SIZE];
   unsigned char outb[BLOCK_SIZE];
+  Ref htab[HS];
+  Ref hnext[BLOCK_SIZE];
   unsigned int in_len = fread(inb, 1, BLOCK_SIZE, from);
   unsigned int out_len;
   if (!in_len)
@@ -1045,7 +1086,7 @@ benchmark(FILE * from)
     {
       calib_loop *= 2;
       for (i = 0; i < calib_loop; i++)
-       compress_buf(inb, in_len, outb, sizeof(outb));
+       compress_buf(htab, hnext, inb, in_len, outb, sizeof(outb));
       per_loop += calib_loop;
     }
 
@@ -1055,13 +1096,13 @@ benchmark(FILE * from)
   start = clock();
   for (i = 0; i < 10; i++)
     for (j = 0; j < per_loop; j++)
-      compress_buf(inb, in_len, outb, sizeof(outb));
+      compress_buf(htab, hnext, inb, in_len, outb, sizeof(outb));
   end = clock();
   seconds = (end - start) / (float) CLOCKS_PER_SEC;
   fprintf(stderr, "%.2f seconds == %.2f MB/s\n", seconds,
           ((long long) in_len * per_loop * 10) / (1024 * 1024 * seconds));
 
-  out_len = compress_buf(inb, in_len, outb, sizeof(outb));
+  out_len = compress_buf(htab, hnext, inb, in_len, outb, sizeof(outb));
 
   calib_loop = 1;
   per_loop = 0;

diff --git a/src/repopage.h b/src/repopage.h
index 9fb84f0f841a2c7232207f94d67ff2183412880b..14f97549f79688414397495c60b17ef4d9b38bc7 100644 (file)
--- a/src/repopage.h
+++ b/src/repopage.h
@@ -41,9 +41,13 @@ typedef struct s_Repopagestore {
   unsigned int rr_counter;
 } Repopagestore;
 
-#ifdef __cplusplus
-extern "C" {
-#endif
+struct s_PageCompressor;
+typedef struct s_PageCompressor PageCompressor;
+
+PageCompressor *pagecompressor_create(void);
+PageCompressor *pagecompressor_free(PageCompressor *comp);
+unsigned int pagecompressor_compress(PageCompressor *comp, unsigned char *page, unsigned int len, unsigned char *cpage, unsigned int max);
+
 
 void repopagestore_init(Repopagestore *store);
 void repopagestore_free(Repopagestore *store);
@@ -61,8 +65,4 @@ int repopagestore_read_or_setup_pages(Repopagestore *store, FILE *fp, unsigned i
 
 void repopagestore_disable_paging(Repopagestore *store);
 
-#ifdef __cplusplus
-}
-#endif
-
 #endif /* LIBSOLV_REPOPAGE_H */