[pzstd] Fix and test 32 bit support

diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile
index d71cf5b340c38320641a2f3d5e7d36ca66b04333..40fce267eb7933b6358da8fc6259d82531f27d5b 100644 (file)
--- a/contrib/pzstd/Makefile
+++ b/contrib/pzstd/Makefile
@@ -30,7 +30,7 @@ else
 EXT =
 endif
 
-.PHONY: default all test clean
+.PHONY: default all test clean test32 googletest googletest32
 
 default: pzstd
 
@@ -41,7 +41,6 @@ libzstd.a: $(ZSTD_FILES)
        $(MAKE) -C $(ZSTDDIR) libzstd
        @cp $(ZSTDDIR)/libzstd.a .
 
-
 Pzstd.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h
        $(CXX) $(FLAGS) -c Pzstd.cpp -o $@
 
@@ -54,23 +53,58 @@ Options.o: Options.h Options.cpp
 main.o: main.cpp *.h utils/*.h
        $(CXX) $(FLAGS) -c main.cpp -o $@
 
-pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a 
+pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a
        $(CXX) $(FLAGS) $^ -o $@$(EXT) -lpthread
 
+libzstd32.a: $(ZSTD_FILES)
+       $(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS="-m32"
+       @cp $(ZSTDDIR)/libzstd.a libzstd32.a
+
+Pzstd32.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h
+       $(CXX) -m32 $(FLAGS) -c Pzstd.cpp -o $@
+
+SkippableFrame32.o: SkippableFrame.h SkippableFrame.cpp utils/*.h
+       $(CXX) -m32 $(FLAGS) -c SkippableFrame.cpp -o $@
+
+Options32.o: Options.h Options.cpp
+       $(CXX) -m32 $(FLAGS) -c Options.cpp -o $@
+
+main32.o: main.cpp *.h utils/*.h
+       $(CXX) -m32 $(FLAGS) -c main.cpp -o $@
+
+pzstd32: Pzstd32.o SkippableFrame32.o Options32.o main32.o libzstd32.a
+       $(CXX) -m32 $(FLAGS) $^ -o $@$(EXT) -lpthread
+
 googletest:
+       @$(RM) -rf googletest
        @git clone https://github.com/google/googletest
        @mkdir -p googletest/build
        @cd googletest/build && cmake .. && make
 
+googletest32:
+       @$(RM) -rf googletest
+       @git clone https://github.com/google/googletest
+       @mkdir -p googletest/build
+       @cd googletest/build && cmake .. -DCMAKE_CXX_FLAGS=-m32 && make
+
 test: libzstd.a Pzstd.o Options.o SkippableFrame.o
        $(MAKE) -C utils/test clean
        $(MAKE) -C utils/test test
        $(MAKE) -C test clean
        $(MAKE) -C test test
 
+test32:
+       $(MAKE) clean
+       $(MAKE) pzstd MOREFLAGS="-m32"
+       $(MAKE) -C utils/test clean
+       $(MAKE) -C utils/test test MOREFLAGS="-m32"
+       $(MAKE) -C test clean
+       $(MAKE) -C test test MOREFLAGS="-m32"
+
+
 clean:
        $(MAKE) -C $(ZSTDDIR) clean
        $(MAKE) -C utils/test clean
        $(MAKE) -C test clean
-       @$(RM) -rf libzstd.a *.o pzstd$(EXT)
+       @$(RM) -rf libzstd.a *.o pzstd$(EXT) pzstd32$(EXT)
        @echo Cleaning completed

diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp
index 978eb9968b4aabbd71dac1dbe36d039fe66945dc..bf42fe81ee5517f18d7ac01271c22bfe400ac0b2 100644 (file)
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@@ -333,10 +333,9 @@ static size_t calculateStep(
   size_t step = size_t{1} << (params.cParams.windowLog + 2);
   // If file size is known, see if a smaller step will spread work more evenly
   if (size != 0) {
-    const std::uintmax_t newStep = size / std::uintmax_t{numThreads};
-    if (newStep != 0 &&
-        newStep <= std::uintmax_t{std::numeric_limits<size_t>::max()}) {
-      step = std::min(step, size_t{newStep});
+    const std::uintmax_t newStep = size / numThreads;
+    if (newStep != 0 && newStep <= std::numeric_limits<size_t>::max()) {
+      step = std::min(step, static_cast<size_t>(newStep));
     }
   }
   return step;

diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp
index 4075229ab5edf89930c1ff44e00112f96aad54a5..b8e0dbd2d604d59babb987adb8822641165a02b1 100644 (file)
--- a/contrib/pzstd/test/PzstdTest.cpp
+++ b/contrib/pzstd/test/PzstdTest.cpp
@@ -40,8 +40,6 @@ TEST(Pzstd, SmallSizes) {
       for (unsigned numThreads = 1; numThreads <= 4; numThreads *= 2) {
         for (unsigned level = 1; level <= 8; level *= 8) {
           auto errorGuard = makeScopeGuard([&] {
-            guard.dismiss();
-            std::fprintf(stderr, "file: %s\n", inputFile.c_str());
             std::fprintf(stderr, "pzstd headers: %u\n", headers);
             std::fprintf(stderr, "# threads: %u\n", numThreads);
             std::fprintf(stderr, "compression level: %u\n", level);
@@ -79,8 +77,6 @@ TEST(Pzstd, LargeSizes) {
       for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) {
         for (unsigned level = 1; level <= 4; level *= 2) {
           auto errorGuard = makeScopeGuard([&] {
-            guard.dismiss();
-            std::fprintf(stderr, "file: %s\n", inputFile.c_str());
             std::fprintf(stderr, "pzstd headers: %u\n", headers);
             std::fprintf(stderr, "# threads: %u\n", numThreads);
             std::fprintf(stderr, "compression level: %u\n", level);
@@ -98,6 +94,34 @@ TEST(Pzstd, LargeSizes) {
   }
 }
 
+TEST(Pzstd, ExtremelyLargeSize) {
+  unsigned seed = std::random_device{}();
+  std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed);
+  std::mt19937 gen(seed);
+
+  std::string inputFile = std::tmpnam(nullptr);
+  auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+
+  {
+    // Write 4GB + 64 MB
+    constexpr size_t kLength = 1 << 26;
+    std::unique_ptr<uint8_t[]> buf(new uint8_t[kLength]);
+    auto fd = std::fopen(inputFile.c_str(), "wb");
+    auto closeGuard = makeScopeGuard([&] { std::fclose(fd); });
+    for (size_t i = 0; i < (1 << 6) + 1; ++i) {
+      RDG_genBuffer(buf.get(), kLength, 0.5, 0.0, gen());
+      auto written = std::fwrite(buf.get(), 1, kLength, fd);
+      ASSERT_EQ(written, kLength);
+    }
+  }
+
+  Options options;
+  options.overwrite = true;
+  options.inputFiles = {inputFile};
+  options.compressionLevel = 1;
+  ASSERT_TRUE(roundTrip(options));
+}
+
 TEST(Pzstd, ExtremelyCompressible) {
   std::string inputFile = std::tmpnam(nullptr);
   auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });