From 725c5e4e381966e2349712ed5f673c157fcbe27b Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 13 May 2021 16:16:47 -0700 Subject: [PATCH] [fuzz] Add determinism fuzzing to simple & dictionary round trip Compress the input twice in the `simple_round_trip` and `dictionary_round_trip` fuzzers with exactly the same parameters, but reusing the context. Then ensure that the compressed output is identical. --- tests/fuzz/dictionary_round_trip.c | 34 ++++++++++++++++++++++++++++-- tests/fuzz/fuzz_data_producer.c | 6 ++++++ tests/fuzz/fuzz_data_producer.h | 3 +++ tests/fuzz/simple_round_trip.c | 26 ++++++++++++++++++++--- 4 files changed, 64 insertions(+), 5 deletions(-) diff --git a/tests/fuzz/dictionary_round_trip.c b/tests/fuzz/dictionary_round_trip.c index 7bff4bd6c..0b20e8d67 100644 --- a/tests/fuzz/dictionary_round_trip.c +++ b/tests/fuzz/dictionary_round_trip.c @@ -42,8 +42,23 @@ static size_t roundTripTest(void *result, size_t resultCapacity, src, srcSize, dict.buff, dict.size, cLevel); + FUZZ_ZASSERT(cSize); + // Compress a second time and check for determinism + { + size_t const cSize0 = cSize; + XXH64_hash_t const hash0 = XXH64(compressed, cSize, 0); + cSize = ZSTD_compress_usingDict(cctx, + compressed, compressedCapacity, + src, srcSize, + dict.buff, dict.size, + cLevel); + FUZZ_ASSERT(cSize == cSize0); + FUZZ_ASSERT(XXH64(compressed, cSize, 0) == hash0); + } } else { + size_t remainingBytes; dictContentType = FUZZ_dataProducer_uint32Range(producer, 0, 2); + remainingBytes = FUZZ_dataProducer_remainingBytes(producer); FUZZ_setRandomParameters(cctx, srcSize, producer); /* Disable checksum so we can use sizes smaller than compress bound. */ FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0)); @@ -51,14 +66,29 @@ static size_t roundTripTest(void *result, size_t resultCapacity, FUZZ_ZASSERT(ZSTD_CCtx_refPrefix_advanced( cctx, dict.buff, dict.size, dictContentType)); - else + else FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced( cctx, dict.buff, dict.size, (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), dictContentType)); cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); + FUZZ_ZASSERT(cSize); + // Compress a second time and check for determinism + { + size_t const cSize0 = cSize; + XXH64_hash_t const hash0 = XXH64(compressed, cSize, 0); + FUZZ_dataProducer_rollBack(producer, remainingBytes); + FUZZ_setRandomParameters(cctx, srcSize, producer); + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0)); + if (refPrefix) + FUZZ_ZASSERT(ZSTD_CCtx_refPrefix_advanced( + cctx, dict.buff, dict.size, + dictContentType)); + cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); + FUZZ_ASSERT(cSize == cSize0); + FUZZ_ASSERT(XXH64(compressed, cSize, 0) == hash0); + } } - FUZZ_ZASSERT(cSize); if (refPrefix) FUZZ_ZASSERT(ZSTD_DCtx_refPrefix_advanced( dctx, dict.buff, dict.size, diff --git a/tests/fuzz/fuzz_data_producer.c b/tests/fuzz/fuzz_data_producer.c index beb0155cf..eae8ee4b6 100644 --- a/tests/fuzz/fuzz_data_producer.c +++ b/tests/fuzz/fuzz_data_producer.c @@ -66,6 +66,12 @@ size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){ return producer->size; } +void FUZZ_dataProducer_rollBack(FUZZ_dataProducer_t *producer, size_t remainingBytes) +{ + FUZZ_ASSERT(remainingBytes >= producer->size); + producer->size = remainingBytes; +} + int FUZZ_dataProducer_empty(FUZZ_dataProducer_t *producer) { return producer->size == 0; } diff --git a/tests/fuzz/fuzz_data_producer.h b/tests/fuzz/fuzz_data_producer.h index 045aaff83..62771a9f8 100644 --- a/tests/fuzz/fuzz_data_producer.h +++ b/tests/fuzz/fuzz_data_producer.h @@ -49,6 +49,9 @@ int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer, /* Returns the size of the remaining bytes of data in the producer */ size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer); +/* Rolls back the data producer state to have remainingBytes remaining */ +void FUZZ_dataProducer_rollBack(FUZZ_dataProducer_t *producer, size_t remainingBytes); + /* Returns true if the data producer is out of bytes */ int FUZZ_dataProducer_empty(FUZZ_dataProducer_t *producer); diff --git a/tests/fuzz/simple_round_trip.c b/tests/fuzz/simple_round_trip.c index c9fac26c5..9da986bc9 100644 --- a/tests/fuzz/simple_round_trip.c +++ b/tests/fuzz/simple_round_trip.c @@ -35,16 +35,36 @@ static size_t roundTripTest(void *result, size_t resultCapacity, size_t dSize; int targetCBlockSize = 0; if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) { + size_t const remainingBytes = FUZZ_dataProducer_remainingBytes(producer); FUZZ_setRandomParameters(cctx, srcSize, producer); cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); + FUZZ_ZASSERT(cSize); FUZZ_ZASSERT(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetCBlockSize, &targetCBlockSize)); + // Compress a second time and check for determinism + { + size_t const cSize0 = cSize; + XXH64_hash_t const hash0 = XXH64(compressed, cSize, 0); + FUZZ_dataProducer_rollBack(producer, remainingBytes); + FUZZ_setRandomParameters(cctx, srcSize, producer); + cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); + FUZZ_ASSERT(cSize == cSize0); + FUZZ_ASSERT(XXH64(compressed, cSize, 0) == hash0); + } } else { - int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); - + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); cSize = ZSTD_compressCCtx( cctx, compressed, compressedCapacity, src, srcSize, cLevel); + FUZZ_ZASSERT(cSize); + // Compress a second time and check for determinism + { + size_t const cSize0 = cSize; + XXH64_hash_t const hash0 = XXH64(compressed, cSize, 0); + cSize = ZSTD_compressCCtx( + cctx, compressed, compressedCapacity, src, srcSize, cLevel); + FUZZ_ASSERT(cSize == cSize0); + FUZZ_ASSERT(XXH64(compressed, cSize, 0) == hash0); + } } - FUZZ_ZASSERT(cSize); dSize = ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize); FUZZ_ZASSERT(dSize); /* When superblock is enabled make sure we don't expand the block more than expected. -- 2.47.2