From: Nathan Moin Vaziri Date: Mon, 11 May 2026 04:15:47 +0000 (-0700) Subject: Cross-vary inflate and deflate benchmarks by input data type X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=dbb5a9a079ca1dd2acd8c674798f4cdae2b884e6;p=thirdparty%2Fzlib-ng.git Cross-vary inflate and deflate benchmarks by input data type inflate_bench and deflate_bench now register a separate benchmark per (variant, data_type) pair so the data type appears in the display name. inflate yields five benchmarks per size: inflate_bench/nocrc/{text,short_match,random,realistic_rgb,striped_rgb}/SIZE deflate yields five benchmarks per (variant, size, level) — across level/nocrc/filtered/huffman/rle/fixed — for 280 combinations total: deflate_bench/{level,nocrc,filtered,huffman,rle,fixed}/{text,...}/SIZE/LEVEL Both fixtures share a Dispatch() pattern with an empty SetUp; ->Name() controls the "/"-separated display. Inflate sizes its compressed buffer via deflateBound to handle incompressible inputs. All data generators move to test/test_data_p.h (renamed from compressible_data_p.h) behind a gen_test_data(test_data_type, bufsize) dispatcher. --- diff --git a/test/benchmarks/benchmark_compress.cc b/test/benchmarks/benchmark_compress.cc index f1aa263bf..25ed13bfe 100644 --- a/test/benchmarks/benchmark_compress.cc +++ b/test/benchmarks/benchmark_compress.cc @@ -15,7 +15,7 @@ extern "C" { # else # include "zlib-ng.h" # endif -# include "test/compressible_data_p.h" +# include "test/test_data_p.h" } #define MAX_SIZE (64 * 1024) @@ -35,11 +35,11 @@ public: // Initialize input buffer with highly compressible data, interspersed // with small amounts of random data and 3-byte matches. - inbuff = gen_compressible_data(MAX_SIZE); + inbuff = gen_test_data(TEST_DATA_TEXT, MAX_SIZE); if (inbuff == NULL) { free(outbuff); outbuff = NULL; - state.SkipWithError("gen_compressible_data() failed"); + state.SkipWithError("gen_test_data() failed"); return; } } diff --git a/test/benchmarks/benchmark_deflate.cc b/test/benchmarks/benchmark_deflate.cc index 0f6e5e693..ead6b0764 100644 --- a/test/benchmarks/benchmark_deflate.cc +++ b/test/benchmarks/benchmark_deflate.cc @@ -15,7 +15,7 @@ extern "C" { # else # include "zlib-ng.h" # endif -# include "test/compressible_data_p.h" +# include "test/test_data_p.h" } #define MAX_SIZE (1024 * 1024) @@ -28,7 +28,10 @@ private: z_uintmax_t outbuff_size = 0; public: - void SetUp(::benchmark::State& state) { + /* Real setup runs from Dispatch() so each variant can pass its data type. */ + void SetUp(::benchmark::State&) {} + + void DoSetUp(::benchmark::State& state, enum test_data_type data_type) { outbuff_size = PREFIX(deflateBound)(NULL, MAX_SIZE); outbuff = (uint8_t *)malloc(outbuff_size); if (outbuff == NULL) { @@ -36,16 +39,16 @@ public: return; } - inbuff = gen_compressible_data(MAX_SIZE); + inbuff = gen_test_data(data_type, MAX_SIZE); if (inbuff == NULL) { free(outbuff); outbuff = NULL; - state.SkipWithError("gen_compressible_data() failed"); + state.SkipWithError("gen_test_data() failed"); return; } } - void Bench(benchmark::State& state, int window_bits, int strategy = Z_DEFAULT_STRATEGY) { + void Bench(benchmark::State& state, int window_bits, int strategy) { int err; size_t size = (size_t)state.range(0); int level = (int)state.range(1); @@ -96,55 +99,53 @@ public: state.counters["ratio"] = benchmark::Counter(double(size) / double(strm.total_out)); } + void Dispatch(benchmark::State& state, enum test_data_type data_type, int window_bits, int strategy) { + DoSetUp(state, data_type); + if (state.skipped()) + return; + Bench(state, window_bits, strategy); + } + void TearDown(const ::benchmark::State&) { free(inbuff); free(outbuff); } }; -#define BENCHMARK_DEFLATE_ARGS \ +#define DEFLATE_ARGS \ ->Args({1024, 1})->Args({1024, 3})->Args({1024, 6})->Args({1024, 9}) \ ->Args({16384, 1})->Args({16384, 3})->Args({16384, 6})->Args({16384, 9}) \ ->Args({131072, 1})->Args({131072, 3})->Args({131072, 6})->Args({131072, 9}) \ ->Args({1048576, 1})->Args({1048576, 3})->Args({1048576, 6})->Args({1048576, 9}) -/* Parameterized deflate with zlib wrapping (includes adler32 checksum) */ -BENCHMARK_DEFINE_F(deflate_bench, deflate_level)(benchmark::State& state) { - Bench(state, MAX_WBITS); -} -BENCHMARK_REGISTER_F(deflate_bench, deflate_level) BENCHMARK_DEFLATE_ARGS; - -/* Parameterized raw deflate without checksum */ -BENCHMARK_DEFINE_F(deflate_bench, deflate_nocrc)(benchmark::State& state) { - Bench(state, -MAX_WBITS); -} -BENCHMARK_REGISTER_F(deflate_bench, deflate_nocrc) BENCHMARK_DEFLATE_ARGS; - /* Strategy benchmarks use fewer size/level combos to keep test count reasonable */ -#define BENCHMARK_DEFLATE_STRATEGY_ARGS \ +#define DEFLATE_STRATEGY_ARGS \ ->Args({1024, 1})->Args({1024, 6})->Args({1024, 9}) \ ->Args({1048576, 1})->Args({1048576, 6})->Args({1048576, 9}) -/* Parameterized deflate with filtered strategy */ -BENCHMARK_DEFINE_F(deflate_bench, deflate_filtered)(benchmark::State& state) { - Bench(state, MAX_WBITS, Z_FILTERED); -} -BENCHMARK_REGISTER_F(deflate_bench, deflate_filtered) BENCHMARK_DEFLATE_STRATEGY_ARGS; +#define DEFLATE_VARIANT(variant, data, wbits, strategy, dt, args_macro) \ + BENCHMARK_DEFINE_F(deflate_bench, variant##_##data)(benchmark::State& state) { \ + Dispatch(state, dt, wbits, strategy); \ + } \ + BENCHMARK_REGISTER_F(deflate_bench, variant##_##data) \ + ->Name("deflate_bench/" #variant "/" #data) args_macro -/* Parameterized deflate with Huffman-only strategy */ -BENCHMARK_DEFINE_F(deflate_bench, deflate_huffman)(benchmark::State& state) { - Bench(state, MAX_WBITS, Z_HUFFMAN_ONLY); -} -BENCHMARK_REGISTER_F(deflate_bench, deflate_huffman) BENCHMARK_DEFLATE_STRATEGY_ARGS; +#define DEFLATE_ALL_DATA(variant, wbits, strategy, args_macro) \ + DEFLATE_VARIANT(variant, text, wbits, strategy, TEST_DATA_TEXT, args_macro); \ + DEFLATE_VARIANT(variant, short_match, wbits, strategy, TEST_DATA_SHORT_MATCH, args_macro); \ + DEFLATE_VARIANT(variant, random, wbits, strategy, TEST_DATA_RANDOM, args_macro); \ + DEFLATE_VARIANT(variant, realistic_rgb, wbits, strategy, TEST_DATA_REALISTIC_RGB, args_macro); \ + DEFLATE_VARIANT(variant, striped_rgb, wbits, strategy, TEST_DATA_STRIPED_RGB, args_macro) +/* Parameterized deflate with zlib wrapping (includes adler32 checksum) */ +DEFLATE_ALL_DATA(level, MAX_WBITS, Z_DEFAULT_STRATEGY, DEFLATE_ARGS); +/* Parameterized raw deflate without checksum */ +DEFLATE_ALL_DATA(nocrc, -MAX_WBITS, Z_DEFAULT_STRATEGY, DEFLATE_ARGS); +/* Parameterized deflate with filtered strategy */ +DEFLATE_ALL_DATA(filtered, MAX_WBITS, Z_FILTERED, DEFLATE_STRATEGY_ARGS); +/* Parameterized deflate with Huffman-only strategy */ +DEFLATE_ALL_DATA(huffman, MAX_WBITS, Z_HUFFMAN_ONLY, DEFLATE_STRATEGY_ARGS); /* Parameterized deflate with RLE strategy */ -BENCHMARK_DEFINE_F(deflate_bench, deflate_rle)(benchmark::State& state) { - Bench(state, MAX_WBITS, Z_RLE); -} -BENCHMARK_REGISTER_F(deflate_bench, deflate_rle) BENCHMARK_DEFLATE_STRATEGY_ARGS; - +DEFLATE_ALL_DATA(rle, MAX_WBITS, Z_RLE, DEFLATE_STRATEGY_ARGS); /* Parameterized deflate with fixed Huffman codes */ -BENCHMARK_DEFINE_F(deflate_bench, deflate_fixed)(benchmark::State& state) { - Bench(state, MAX_WBITS, Z_FIXED); -} -BENCHMARK_REGISTER_F(deflate_bench, deflate_fixed) BENCHMARK_DEFLATE_STRATEGY_ARGS; +DEFLATE_ALL_DATA(fixed, MAX_WBITS, Z_FIXED, DEFLATE_STRATEGY_ARGS); diff --git a/test/benchmarks/benchmark_inflate.cc b/test/benchmarks/benchmark_inflate.cc index 2b974d9fc..e254e507f 100644 --- a/test/benchmarks/benchmark_inflate.cc +++ b/test/benchmarks/benchmark_inflate.cc @@ -15,36 +15,37 @@ extern "C" { # else # include "zlib-ng.h" # endif -# include "test/compressible_data_p.h" +# include "test/test_data_p.h" } #define MAX_SIZE (1024 * 1024) -#define NUM_TESTS 6 - class inflate_bench: public benchmark::Fixture { private: - uint8_t *inbuff; - uint8_t *outbuff; - uint8_t *compressed_buff[NUM_TESTS]; - z_uintmax_t compressed_sizes[NUM_TESTS]; - uint32_t sizes[NUM_TESTS] = {1, 64, 1024, 16384, 128*1024, 1024*1024}; + uint8_t *inbuff = nullptr; + uint8_t *outbuff = nullptr; + uint8_t *compressed_buff = nullptr; + z_uintmax_t compressed_size = 0; public: - void SetUp(::benchmark::State& state) { + /* Real setup runs from Run() so each variant can pass its data type + without the type needing to live in state.range. */ + void SetUp(::benchmark::State&) {} + + void DoSetUp(::benchmark::State& state, enum test_data_type data_type) { int err; + uint32_t size = (uint32_t)state.range(0); outbuff = (uint8_t *)malloc(MAX_SIZE + 16); if (outbuff == NULL) { state.SkipWithError("malloc failed"); return; } - // Initialize input buffer with highly compressible data, interspersed - // with small amounts of random data and 3-byte matches. - inbuff = gen_compressible_data(MAX_SIZE); + // Initialize input buffer with the selected type of test data + inbuff = gen_test_data(data_type, MAX_SIZE); if (inbuff == NULL) { free(outbuff); outbuff = NULL; - state.SkipWithError("gen_compressible_data() failed"); + state.SkipWithError("input data generator failed"); return; } @@ -64,37 +65,30 @@ public: return; } + // Compress the size being benchmarked + size_t buf_cap = (size_t)PREFIX(deflateBound)(&strm, size); + compressed_buff = (uint8_t *)malloc(buf_cap); + if (compressed_buff == NULL) { + state.SkipWithError("malloc failed"); + PREFIX(deflateEnd)(&strm); + return; + } - // Compress data into different buffers - for (int i = 0; i < NUM_TESTS; ++i) { - compressed_buff[i] = (uint8_t *)malloc(sizes[i] + 64); - if (compressed_buff[i] == NULL) { - state.SkipWithError("malloc failed"); - return; - } - - strm.avail_in = sizes[i]; // Size of the input buffer - strm.next_in = (z_const uint8_t *)inbuff; // Pointer to the input buffer - strm.next_out = compressed_buff[i]; // Pointer to the output buffer - strm.avail_out = sizes[i] + 64; // Maximum size of the output buffer - - err = PREFIX(deflate)(&strm, Z_FINISH); // Perform compression - if (err != Z_STREAM_END ) { - state.SkipWithError("deflate did not return Z_STREAM_END"); - PREFIX(deflateEnd)(&strm); - return; - } - - compressed_sizes[i] = strm.total_out; // Total compressed size + strm.avail_in = size; // Size of the input buffer + strm.next_in = (z_const uint8_t *)inbuff; // Pointer to the input buffer + strm.next_out = compressed_buff; // Pointer to the output buffer + strm.avail_out = (uint32_t)buf_cap; // Maximum size of the output buffer - err = PREFIX(deflateReset)(&strm); // Reset Deflate state - if (err != Z_OK) { - state.SkipWithError("deflateReset did not return Z_OK"); - return; - } + err = PREFIX(deflate)(&strm, Z_FINISH); // Perform compression + if (err != Z_STREAM_END) { + state.SkipWithError("deflate did not return Z_STREAM_END"); + PREFIX(deflateEnd)(&strm); + return; } - err = PREFIX(deflateEnd)(&strm); // Clean up the deflate stream + compressed_size = strm.total_out; // Total compressed size + + err = PREFIX(deflateEnd)(&strm); // Clean up the deflate stream if (err != Z_OK) { state.SkipWithError("deflateEnd did not return Z_OK"); return; @@ -103,8 +97,6 @@ public: void Bench(benchmark::State& state) { int err; - int index = 0; - while (sizes[index] != (uint32_t)state.range(0)) ++index; // Initialize the inflate stream PREFIX3(stream) strm; @@ -128,10 +120,10 @@ public: return; } - strm.avail_in = (uint32_t)compressed_sizes[index]; // Size of the input - strm.next_in = compressed_buff[index]; // Pointer to the compressed data - strm.avail_out = MAX_SIZE; // Max size for output - strm.next_out = outbuff; // Output buffer + strm.avail_in = (uint32_t)compressed_size; // Size of the input + strm.next_in = compressed_buff; // Pointer to the compressed data + strm.avail_out = MAX_SIZE; // Max size for output + strm.next_out = outbuff; // Output buffer // Perform decompression err = PREFIX(inflate)(&strm, Z_FINISH); @@ -150,18 +142,29 @@ public: } } + void Dispatch(benchmark::State& state, enum test_data_type data_type) { + DoSetUp(state, data_type); + if (state.skipped()) + return; + Bench(state); + } + void TearDown(const ::benchmark::State&) { free(inbuff); free(outbuff); - - for (int i = 0; i < NUM_TESTS; ++i) { - free(compressed_buff[i]); - } + free(compressed_buff); } }; -BENCHMARK_DEFINE_F(inflate_bench, inflate_nocrc)(benchmark::State& state) { - Bench(state); -} -BENCHMARK_REGISTER_F(inflate_bench, inflate_nocrc) - ->Arg(1)->Arg(64)->Arg(1024)->Arg(16<<10)->Arg(128<<10)->Arg(1024<<10); +#define INFLATE_SIZES_ARGS \ + ->Arg(1)->Arg(64)->Arg(1024)->Arg(16<<10)->Arg(128<<10)->Arg(1024<<10) + +#define INFLATE_VARIANT(name, dt) \ + BENCHMARK_DEFINE_F(inflate_bench, name)(benchmark::State& state) { Dispatch(state, dt); } \ + BENCHMARK_REGISTER_F(inflate_bench, name)->Name("inflate_bench/nocrc/" #name) INFLATE_SIZES_ARGS + +INFLATE_VARIANT(text, TEST_DATA_TEXT); +INFLATE_VARIANT(short_match, TEST_DATA_SHORT_MATCH); +INFLATE_VARIANT(random, TEST_DATA_RANDOM); +INFLATE_VARIANT(realistic_rgb, TEST_DATA_REALISTIC_RGB); +INFLATE_VARIANT(striped_rgb, TEST_DATA_STRIPED_RGB); diff --git a/test/benchmarks/benchmark_inflate_chunked.cc b/test/benchmarks/benchmark_inflate_chunked.cc index 8d42a1018..01fc5f041 100644 --- a/test/benchmarks/benchmark_inflate_chunked.cc +++ b/test/benchmarks/benchmark_inflate_chunked.cc @@ -15,7 +15,7 @@ extern "C" { # else # include "zlib-ng.h" # endif -# include "test/compressible_data_p.h" +# include "test/test_data_p.h" } #define TOTAL_SIZE (256 * 1024) @@ -36,11 +36,11 @@ public: return; } - inbuff = gen_compressible_data(TOTAL_SIZE); + inbuff = gen_test_data(TEST_DATA_TEXT, TOTAL_SIZE); if (inbuff == NULL) { free(outbuff); outbuff = NULL; - state.SkipWithError("gen_compressible_data() failed"); + state.SkipWithError("gen_test_data() failed"); return; } diff --git a/test/benchmarks/benchmark_png_decode.cc b/test/benchmarks/benchmark_png_decode.cc index f96650947..6639ddfb0 100644 --- a/test/benchmarks/benchmark_png_decode.cc +++ b/test/benchmarks/benchmark_png_decode.cc @@ -1,6 +1,7 @@ #include #include #include "benchmark_png_shared.h" +#include "test/test_data_p.h" #include class png_decode: public benchmark::Fixture { @@ -11,15 +12,9 @@ protected: uint8_t *output_img_buf = NULL; public: - /* Let's make the vanilla version have something extremely compressible */ - virtual void init_img(png_bytep img_bytes, size_t width, size_t height) { - init_compressible(img_bytes, width*height); - } - void SetUp(const ::benchmark::State&) { - output_img_buf = (uint8_t*)malloc(IMWIDTH * IMHEIGHT * 3); + output_img_buf = gen_test_data(TEST_DATA_STRIPED_RGB, IMWIDTH * IMHEIGHT * 3); assert(output_img_buf != NULL); - init_img(output_img_buf, IMWIDTH, IMHEIGHT); /* First we need to author the png bytes to be decoded */ for (int i = 0; i < 10; ++i) { diff --git a/test/benchmarks/benchmark_png_decode_small.cc b/test/benchmarks/benchmark_png_decode_small.cc index 6280d1431..466360539 100644 --- a/test/benchmarks/benchmark_png_decode_small.cc +++ b/test/benchmarks/benchmark_png_decode_small.cc @@ -3,6 +3,7 @@ #include #include "benchmark_png_shared.h" +#include "test/test_data_p.h" /* Decode PNGs through libpng at various image widths. libpng calls inflate() with * avail_out equal to one row (width * 3 bytes for RGB). Narrow images produce rows @@ -21,9 +22,8 @@ public: size_t num_pixels = (size_t)img_width * img_height; - output_buf = (uint8_t *)malloc(num_pixels * 3); + output_buf = gen_test_data(TEST_DATA_REALISTIC_RGB, num_pixels * 3); assert(output_buf != NULL); - init_realistic(output_buf, img_width, img_height); encoded = {NULL, 0, 0}; encode_png(output_buf, &encoded, 9, img_width, img_height); diff --git a/test/benchmarks/benchmark_png_encode.cc b/test/benchmarks/benchmark_png_encode.cc index d5e25cbc9..eb8640500 100644 --- a/test/benchmarks/benchmark_png_encode.cc +++ b/test/benchmarks/benchmark_png_encode.cc @@ -2,6 +2,7 @@ #include #include #include "benchmark_png_shared.h" +#include "test/test_data_p.h" #define IMWIDTH 1024 #define IMHEIGHT 1024 @@ -14,13 +15,8 @@ private: uint8_t *input_img_buf = NULL; public: - /* Let's make the vanilla version have something extremely compressible */ - virtual void init_img(png_bytep img_bytes, size_t width, size_t height) { - init_compressible(img_bytes, width * height); - } - void SetUp(const ::benchmark::State&) { - input_img_buf = (uint8_t*)malloc(IMWIDTH * IMHEIGHT * 3); + input_img_buf = gen_test_data(TEST_DATA_STRIPED_RGB, IMWIDTH * IMHEIGHT * 3); outpng.buf = (uint8_t*)malloc(IMWIDTH * IMHEIGHT * 3); /* Using malloc rather than zng_alloc so that we can call realloc. * IMWIDTH * IMHEIGHT is likely to be more than enough bytes, though, @@ -30,7 +26,6 @@ public: outpng.buf_rem = IMWIDTH * IMHEIGHT * 3; assert(input_img_buf != NULL); assert(outpng.buf != NULL); - init_img(input_img_buf, IMWIDTH, IMHEIGHT); } /* State in this circumstance will convey the compression level */ diff --git a/test/benchmarks/benchmark_png_shared.h b/test/benchmarks/benchmark_png_shared.h index ef60ce1bf..a6562f59b 100644 --- a/test/benchmarks/benchmark_png_shared.h +++ b/test/benchmarks/benchmark_png_shared.h @@ -48,55 +48,6 @@ static void png_write_cb(png_structp pngp, png_bytep data, png_size_t len) { dat->buf_rem -= len; } -/* Generate pixel data that resembles a real photograph: smooth gradients with per-pixel - * noise and occasional edges. Produces many short deflate matches and scattered literals */ -static void init_realistic(png_bytep buf, uint32_t width, uint32_t height) { - uint32_t seed = 0x12345678; - for (uint32_t y = 0; y < height; y++) { - for (uint32_t x = 0; x < width; x++) { - size_t idx = ((size_t)y * width + x) * 3; - /* Diagonal gradient as base color */ - uint8_t base_r = (uint8_t)((x + y) * 179 / (width + height)); - uint8_t base_g = (uint8_t)((x * 2 + y) * 131 / (width + height)); - uint8_t base_b = (uint8_t)(y * 241 / height); - /* Simple xorshift noise, +/- 15 levels */ - seed ^= seed << 13; - seed ^= seed >> 17; - seed ^= seed << 5; - int noise = (int)(seed & 0x1F) - 15; - buf[idx] = (uint8_t)MIN(MAX(base_r + noise, 0), 0xFF); - buf[idx + 1] = (uint8_t)MIN(MAX(base_g + (noise >> 1), 0), 0xFF); - buf[idx + 2] = (uint8_t)MIN(MAX(base_b - noise, 0), 0xFF); - } - } -} - -/* Generate a highly compressible RGB test image with solid R, G, and B stripes. */ -static void init_compressible(png_bytep buf, size_t num_pix) { - int32_t i = 0; - int32_t red_stop = num_pix / 3; - int32_t blue_stop = 2 * num_pix / 3; - int32_t green_stop = num_pix; - - for (int32_t x = 0; i < red_stop; x += 3, ++i) { - buf[x] = 255; - buf[x + 1] = 0; - buf[x + 2] = 0; - } - - for (int32_t x = 3 * i; i < blue_stop; x+= 3, ++i) { - buf[x] = 0; - buf[x + 1] = 255; - buf[x + 2] = 0; - } - - for (int32_t x = 3 * i; i < green_stop; x += 3, ++i) { - buf[x] = 0; - buf[x + 1] = 0; - buf[x + 2] = 255; - } -} - static inline void encode_png(png_bytep buf, png_dat *outpng, int32_t comp_level, uint32_t width, uint32_t height) { png_structp png = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL); diff --git a/test/benchmarks/benchmark_uncompress.cc b/test/benchmarks/benchmark_uncompress.cc index 6a82c05d0..d3a2f846f 100644 --- a/test/benchmarks/benchmark_uncompress.cc +++ b/test/benchmarks/benchmark_uncompress.cc @@ -15,7 +15,7 @@ extern "C" { # else # include "zlib-ng.h" # endif -# include "test/compressible_data_p.h" +# include "test/test_data_p.h" } #define MAX_SIZE (1024 * 1024) @@ -39,11 +39,11 @@ public: // Initialize input buffer with highly compressible data, interspersed // with small amounts of random data and 3-byte matches. - inbuff = gen_compressible_data(MAX_SIZE); + inbuff = gen_test_data(TEST_DATA_TEXT, MAX_SIZE); if (inbuff == NULL) { free(outbuff); outbuff = NULL; - state.SkipWithError("gen_compressible_data() failed"); + state.SkipWithError("gen_test_data() failed"); return; } diff --git a/test/compressible_data_p.h b/test/compressible_data_p.h deleted file mode 100644 index e345bb9c9..000000000 --- a/test/compressible_data_p.h +++ /dev/null @@ -1,54 +0,0 @@ -/* compressible_data_p.h -- generate compressible data - * Copyright (C) 2025 Hans Kristian Rosbach - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -#ifndef COMPRESSIBLE_DATA_P_H -#define COMPRESSIBLE_DATA_P_H - - -static inline size_t append_raw(uint8_t *dest, size_t size, const void *src, size_t len) { - if (len > size) len = size; - if (len == 0) return 0; - memcpy(dest, src, len); - return len; -} -static inline size_t append_str(uint8_t *dest, size_t size, const char *src) { - return append_raw(dest, size, src, strlen(src)); -} -static inline size_t append_uint8_t(uint8_t *dest, size_t size, uint8_t src) { - return append_raw(dest, size, &src, 1); -} - -// Alloc and initialize buffer with highly compressible data, -// interspersed with small amounts of random data and 3-byte matches. -static uint8_t *gen_compressible_data(size_t bufsize) { - const char teststr1[42] = "Hello hello World broken Test tast mello."; - const char teststr2[32] = "llollollollollo He Te me orld"; - const char teststr3[4] = "bro"; - int loops = 0; - - uint8_t *buffer = (uint8_t *)malloc(bufsize); - if (buffer == NULL) { - return NULL; - } - - for (size_t pos = 0; pos < bufsize; ) { - pos += append_str(buffer+pos, bufsize-pos, teststr1); - pos += append_uint8_t(buffer+pos, bufsize-pos, (uint8_t)(rand() & 0xFF)); - // Every so often, add a few other little bits to break the pattern - if (loops % 13 == 0) { - pos += append_str(buffer+pos, bufsize-pos, teststr3); - pos += append_uint8_t(buffer+pos, bufsize-pos, (uint8_t)(rand() & 0xFF)); - } - if (loops % 300 == 0) { // Only found once or twice per window - pos += append_str(buffer+pos, bufsize-pos, teststr2); - } - loops++; - } - if (bufsize > 0) { - buffer[bufsize-1] = 0; - } - return buffer; -} -#endif diff --git a/test/test_chunked.cc b/test/test_chunked.cc index ac8716ff4..1d8a309ed 100644 --- a/test/test_chunked.cc +++ b/test/test_chunked.cc @@ -14,7 +14,7 @@ #include -#include "compressible_data_p.h" +#include "test_data_p.h" struct chunked_params { size_t compr_size; @@ -37,7 +37,7 @@ protected: decompressed = (uint8_t *)calloc(1, p.uncompr_size); ASSERT_NE(decompressed, nullptr); - uncompr = gen_compressible_data(p.uncompr_size); + uncompr = gen_test_data(TEST_DATA_TEXT, p.uncompr_size); ASSERT_NE(uncompr, nullptr); } diff --git a/test/test_data_p.h b/test/test_data_p.h new file mode 100644 index 000000000..f63e3139f --- /dev/null +++ b/test/test_data_p.h @@ -0,0 +1,237 @@ +/* test_data_p.h -- shared input data generators for tests and benchmarks + * Copyright (C) 2025 Hans Kristian Rosbach + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#ifndef TEST_DATA_P_H +#define TEST_DATA_P_H + +static inline size_t append_raw(uint8_t *dest, size_t size, const void *src, size_t len) { + if (len > size) len = size; + if (len == 0) + return 0; + memcpy(dest, src, len); + return len; +} +static inline size_t append_str(uint8_t *dest, size_t size, const char *src) { + return append_raw(dest, size, src, strlen(src)); +} +static inline size_t append_uint8_t(uint8_t *dest, size_t size, uint8_t src) { + return append_raw(dest, size, &src, 1); +} + +/* English-like text: words drawn Zipf-style from a small vocabulary, with + occasional novel words mutated from vocabulary ones. Repeated words become + short-to-medium matches at text-like distances; novel words and word boundaries + leave scattered literals. */ +static inline uint8_t *gen_text_data(size_t bufsize) { + static const char letters[] = "etaoinshrdlucmfwypvbgk"; + uint8_t vocab[128][12]; + uint8_t vlen[128]; + uint32_t rng = 0x7e47da7a; + uint8_t *buf = (uint8_t *)malloc(bufsize); + if (buf == NULL) + return NULL; + + for (int w = 0; w < 128; w++) { + rng = rng * 1103515245u + 12345u; + vlen[w] = (uint8_t)(3 + ((rng >> 16) % 8)); + for (int c = 0; c < vlen[w]; c++) { + rng = rng * 1103515245u + 12345u; + vocab[w][c] = (uint8_t)letters[(rng >> 16) % (sizeof(letters) - 1)]; + } + } + + size_t pos = 0; + uint32_t words = 0; + while (pos < bufsize) { + /* AND of two 7-bit draws biases toward low ranks (Zipf-like) */ + rng = rng * 1103515245u + 12345u; + uint32_t w = ((rng >> 16) & 127) & ((rng >> 22) & 127); + uint8_t word[12]; + uint8_t len = vlen[w]; + memcpy(word, vocab[w], len); + rng = rng * 1103515245u + 12345u; + if (((rng >> 16) % 6) == 0) { + /* Novel word: mutate the tail into fresh literals */ + for (int c = len > 4 ? len - 4 : 1; c < len; c++) { + rng = rng * 1103515245u + 12345u; + word[c] = (uint8_t)letters[(rng >> 16) % (sizeof(letters) - 1)]; + } + } + pos += append_raw(buf + pos, bufsize - pos, word, len); + rng = rng * 1103515245u + 12345u; + words++; + if ((words % 12) == 0) + pos += append_str(buf + pos, bufsize - pos, ".\n"); + else if (((rng >> 16) % 16) == 0) + pos += append_str(buf + pos, bufsize - pos, ", "); + else + pos += append_uint8_t(buf + pos, bufsize - pos, ' '); + } + if (bufsize > 0) + buf[bufsize - 1] = 0; + return buf; +} + +/* A rotating pool of eight 3..8-byte random patterns emitted in random order. + Re-emitted patterns become short back-references at small distances that + frequently chain match-to-match; pool refreshes and separator bytes leave short + literal runs; ~1/16 of iterations emit a dist=1 RLE run. */ +static inline uint8_t *gen_short_match_data(size_t bufsize) { + uint8_t *buf = (uint8_t *)malloc(bufsize); + if (buf == NULL) + return NULL; + + uint32_t rng = 0xc001cafe; + size_t i = 0; + uint8_t pool[8][8]; + uint8_t plens[8]; + + for (int s = 0; s < 8; s++) { + rng = rng * 1103515245u + 12345u; + plens[s] = (uint8_t)(3 + ((rng >> 16) % 6)); + for (int j = 0; j < plens[s]; j++) { + rng = rng * 1103515245u + 12345u; + pool[s][j] = (uint8_t)(rng >> 24); + } + } + + while (i < bufsize) { + rng = rng * 1103515245u + 12345u; + uint32_t r = (rng >> 16) & 0xF; + uint32_t slot = (rng >> 20) & 7; + if (r == 0) { + /* RLE run: one byte repeated, matched at dist=1 */ + rng = rng * 1103515245u + 12345u; + uint8_t b = (uint8_t)(rng >> 24); + size_t run = 6 + ((rng >> 16) % 18); + for (size_t j = 0; j < run && i < bufsize; j++) + buf[i++] = b; + } else if (r <= 2) { + /* Refresh a pool slot with a fresh pattern and emit it: literals */ + rng = rng * 1103515245u + 12345u; + plens[slot] = (uint8_t)(3 + ((rng >> 16) % 6)); + for (int j = 0; j < plens[slot]; j++) { + rng = rng * 1103515245u + 12345u; + pool[slot][j] = (uint8_t)(rng >> 24); + if (i < bufsize) + buf[i++] = pool[slot][j]; + } + } else if (r == 3) { + /* Separator literal */ + rng = rng * 1103515245u + 12345u; + buf[i++] = (uint8_t)(rng >> 24); + } else { + /* Re-emit a pool pattern: a short match, often chaining */ + for (int j = 0; j < plens[slot] && i < bufsize; j++) + buf[i++] = pool[slot][j]; + } + } + return buf; +} + +static inline uint8_t clamp_uint8_t(int v) { + return (uint8_t)(v < 0 ? 0 : (v > 0xFF ? 0xFF : v)); +} + +/* Pseudorandom incompressible bytes. Forces deflate into stored blocks, exercising + the inflate literal-byte path with no chunk copies. */ +static inline uint8_t *gen_random_data(size_t bufsize) { + uint8_t *buf = (uint8_t *)malloc(bufsize); + if (buf == NULL) + return NULL; + uint32_t rng = 0xdeadbeef; + for (size_t i = 0; i < bufsize; i++) { + rng = rng * 1103515245u + 12345u; + buf[i] = (uint8_t)(rng >> 24); + } + return buf; +} + +/* RGB photo-like pixels at a fixed row width: smooth gradients with per-pixel + noise. Yields short matches at dist=3 (RGB stride) and longer inter-row matches; + deflate emits scattered literals between them. */ +static inline uint8_t *gen_realistic_rgb_data(size_t bufsize) { + uint8_t *buf = (uint8_t *)malloc(bufsize); + if (buf == NULL) + return NULL; + + size_t pixels = bufsize / 3; + uint32_t width = (uint32_t)(pixels >= 256 ? 256 : pixels); + uint32_t height = (uint32_t)(width > 0 ? pixels / width : 0); + if (height == 0) { + memset(buf, 0, bufsize); + return buf; + } + + uint32_t seed = 0x12345678; + for (uint32_t y = 0; y < height; y++) { + for (uint32_t x = 0; x < width; x++) { + size_t idx = ((size_t)y * width + x) * 3; + /* Diagonal gradient as base color */ + uint8_t base_r = (uint8_t)((x + y) * 179 / (width + height)); + uint8_t base_g = (uint8_t)((x * 2 + y) * 131 / (width + height)); + uint8_t base_b = (uint8_t)(y * 241 / height); + /* Simple xorshift noise, +/- 15 levels */ + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + int noise = (int)(seed & 0x1F) - 15; + buf[idx] = clamp_uint8_t(base_r + noise); + buf[idx + 1] = clamp_uint8_t(base_g + (noise >> 1)); + buf[idx + 2] = clamp_uint8_t(base_b - noise); + } + } + size_t filled = (size_t)width * height * 3; + if (filled < bufsize) memset(buf + filled, 0, bufsize - filled); + return buf; +} + +/* RGB pixels arranged as three solid R/G/B stripes. Yields long matches at dist=3 + within each stripe and large back-references across the stripe boundaries. */ +static inline uint8_t *gen_striped_rgb_data(size_t bufsize) { + uint8_t *buf = (uint8_t *)malloc(bufsize); + if (buf == NULL) + return NULL; + + size_t pixels = bufsize / 3; + size_t red_stop = pixels / 3; + size_t blue_stop = 2 * pixels / 3; + size_t i = 0; + + for (size_t x = 0; i < red_stop; x += 3, ++i) { + buf[x] = 255; buf[x + 1] = 0; buf[x + 2] = 0; + } + for (size_t x = 3 * i; i < blue_stop; x += 3, ++i) { + buf[x] = 0; buf[x + 1] = 255; buf[x + 2] = 0; + } + for (size_t x = 3 * i; i < pixels; x += 3, ++i) { + buf[x] = 0; buf[x + 1] = 0; buf[x + 2] = 255; + } + size_t filled = pixels * 3; + if (filled < bufsize) memset(buf + filled, 0, bufsize - filled); + return buf; +} + +/* Each variant targets a distinct shape of deflate stream. */ +enum test_data_type { + TEST_DATA_TEXT = 0, /* mixed literals + short/medium matches */ + TEST_DATA_SHORT_MATCH, /* many short back-references */ + TEST_DATA_RANDOM, /* incompressible, deflate uses stored blocks */ + TEST_DATA_REALISTIC_RGB, /* RGB photo, short matches at dist=3 */ + TEST_DATA_STRIPED_RGB, /* solid R/G/B stripes, long dist=3 matches */ +}; + +static inline uint8_t *gen_test_data(enum test_data_type data_type, size_t bufsize) { + switch (data_type) { + case TEST_DATA_TEXT: return gen_text_data(bufsize); + case TEST_DATA_SHORT_MATCH: return gen_short_match_data(bufsize); + case TEST_DATA_RANDOM: return gen_random_data(bufsize); + case TEST_DATA_REALISTIC_RGB: return gen_realistic_rgb_data(bufsize); + case TEST_DATA_STRIPED_RGB: return gen_striped_rgb_data(bufsize); + } + return NULL; +} + +#endif diff --git a/test/test_deflate.cc b/test/test_deflate.cc index 4927c821e..7c84d93fb 100644 --- a/test/test_deflate.cc +++ b/test/test_deflate.cc @@ -14,7 +14,7 @@ #include -#include "compressible_data_p.h" +#include "test_data_p.h" #include "test_shared.h" #define MAX_SIZE (1024 * 1024) @@ -22,7 +22,7 @@ class deflate_variant : public testing::TestWithParam> { public: static void SetUpTestSuite() { - inbuf = gen_compressible_data(MAX_SIZE); + inbuf = gen_test_data(TEST_DATA_TEXT, MAX_SIZE); ASSERT_TRUE(inbuf != NULL); } static void TearDownTestSuite() {