Fix zstd-dll build missing dependencies (#3496)

* Fixes zstd-dll build (https://github.com/facebook/zstd/issues/3492):
- Adds pool.o and threading.o dependency to the zstd-dll target
- Moves custom allocation functions into header to avoid needing to add dependency on common.o
- Adds test target for zstd-dll
- Adds github workflow that buildis zstd-dll

diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml
index 655394f8435ca4648a007b6155febd6281d4667e..092c933c762e0ea3de8e262d26b634a2a4ce6bb6 100644 (file)
--- a/.github/workflows/dev-short-tests.yml
+++ b/.github/workflows/dev-short-tests.yml
@@ -55,6 +55,14 @@ jobs:
       run: |
         make c89build V=1
 
+  build-zstd-dll:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: build zstd bin against a dynamic lib (debuglevel for more dependencies)
+      run: |
+        make -C lib lib-mt-release 
+        DEBUGLEVEL=2 make -C programs zstd-dll
 
   gcc-7-libzstd:
     runs-on: ubuntu-latest
@@ -339,7 +347,6 @@ jobs:
         make -j -C programs allVariants MOREFLAGS=-O0
         ./tests/test-variants.sh
 
-
   qemu-consistency:
     name: QEMU ${{ matrix.name }}
     runs-on: ubuntu-20.04

diff --git a/contrib/linux-kernel/zstd_common_module.c b/contrib/linux-kernel/zstd_common_module.c
index 2fead39eb7439f0e9673f7427204f087895d1541..466828e3575256a269e5ef7d4bd0cb7283b50a8e 100644 (file)
--- a/contrib/linux-kernel/zstd_common_module.c
+++ b/contrib/linux-kernel/zstd_common_module.c
@@ -24,9 +24,6 @@ EXPORT_SYMBOL_GPL(HUF_readStats_wksp);
 EXPORT_SYMBOL_GPL(ZSTD_isError);
 EXPORT_SYMBOL_GPL(ZSTD_getErrorName);
 EXPORT_SYMBOL_GPL(ZSTD_getErrorCode);
-EXPORT_SYMBOL_GPL(ZSTD_customMalloc);
-EXPORT_SYMBOL_GPL(ZSTD_customCalloc);
-EXPORT_SYMBOL_GPL(ZSTD_customFree);
 
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION("Zstd Common");

diff --git a/lib/common/allocations.h b/lib/common/allocations.h
new file mode 100644 (file)
index 0000000..a3153c4
--- /dev/null
+++ b/lib/common/allocations.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* This file provides custom allocation primitives
+ */
+
+#define ZSTD_DEPS_NEED_MALLOC
+#include "zstd_deps.h"   /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
+
+#include "mem.h" /* MEM_STATIC */
+#define ZSTD_STATIC_LINKING_ONLY
+#include "../zstd.h" /* ZSTD_customMem */
+
+#ifndef ZSTD_ALLOCATIONS_H
+#define ZSTD_ALLOCATIONS_H
+
+/* custom memory allocation functions */
+
+MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
+{
+    if (customMem.customAlloc)
+        return customMem.customAlloc(customMem.opaque, size);
+    return ZSTD_malloc(size);
+}
+
+MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
+{
+    if (customMem.customAlloc) {
+        /* calloc implemented as malloc+memset;
+         * not as efficient as calloc, but next best guess for custom malloc */
+        void* const ptr = customMem.customAlloc(customMem.opaque, size);
+        ZSTD_memset(ptr, 0, size);
+        return ptr;
+    }
+    return ZSTD_calloc(1, size);
+}
+
+MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
+{
+    if (ptr!=NULL) {
+        if (customMem.customFree)
+            customMem.customFree(customMem.opaque, ptr);
+        else
+            ZSTD_free(ptr);
+    }
+}
+
+#endif /* ZSTD_ALLOCATIONS_H */

diff --git a/lib/common/pool.c b/lib/common/pool.c
index f3d9d08547c0847f5de40b60bd65ab6de9058512..d5ca5a7808afc2f9a2a9cf81a0c1aafce00b864e 100644 (file)
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -10,9 +10,9 @@
 
 
 /* ======   Dependencies   ======= */
+#include "../common/allocations.h"  /* ZSTD_customCalloc, ZSTD_customFree */
 #include "zstd_deps.h" /* size_t */
 #include "debug.h"     /* assert */
-#include "zstd_internal.h"  /* ZSTD_customCalloc, ZSTD_customFree */
 #include "pool.h"
 
 /* ======   Compiler specifics   ====== */

diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index 3208552475da290cdee579fe0f5ec218bbcd05fd..3f04c22abf6cc331c63e7ea926a456b74b205e3e 100644 (file)
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -14,7 +14,6 @@
 *  Dependencies
 ***************************************/
 #define ZSTD_DEPS_NEED_MALLOC
-#include "zstd_deps.h"   /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
 #include "error_private.h"
 #include "zstd_internal.h"
 
@@ -47,37 +46,3 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
 /*! ZSTD_getErrorString() :
  *  provides error code string from enum */
 const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
-
-
-
-/*=**************************************************************
-*  Custom allocator
-****************************************************************/
-void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
-{
-    if (customMem.customAlloc)
-        return customMem.customAlloc(customMem.opaque, size);
-    return ZSTD_malloc(size);
-}
-
-void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
-{
-    if (customMem.customAlloc) {
-        /* calloc implemented as malloc+memset;
-         * not as efficient as calloc, but next best guess for custom malloc */
-        void* const ptr = customMem.customAlloc(customMem.opaque, size);
-        ZSTD_memset(ptr, 0, size);
-        return ptr;
-    }
-    return ZSTD_calloc(1, size);
-}
-
-void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
-{
-    if (ptr!=NULL) {
-        if (customMem.customFree)
-            customMem.customFree(customMem.opaque, ptr);
-        else
-            ZSTD_free(ptr);
-    }
-}

diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 37836dc703f8b09de79d8dbaa69c39ec311e02c2..1f942f27bf090c94b4a22bc2b814a86d599d5a65 100644 (file)
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -350,11 +350,6 @@ typedef struct {
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);   /* compress & dictBuilder */
 int ZSTD_seqToCodes(const seqStore_t* seqStorePtr);   /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
 
-/* custom memory allocation functions */
-void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem);
-void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem);
-void ZSTD_customFree(void* ptr, ZSTD_customMem customMem);
-
 
 /* ZSTD_invalidateRepCodes() :
  * ensures next compression will not use repcodes from previous block.

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index b55f684cd710064d0e139278f5b3b2ecaab68f5f..173ce2421a1b73b8a64a71e28c3d06dcc02ae39b 100644 (file)
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -11,6 +11,7 @@
 /*-*************************************
 *  Dependencies
 ***************************************/
+#include "../common/allocations.h"  /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
 #include "../common/zstd_deps.h"  /* INT_MAX, ZSTD_memset, ZSTD_memcpy */
 #include "../common/mem.h"
 #include "hist.h"           /* HIST_countFast_wksp */

diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h
index 97676693b5eed10b306e1eaa05f3bd3744e95aa6..9113cdbd4fbcfe5acd2a4494a45e62830e3c7bd1 100644 (file)
--- a/lib/compress/zstd_cwksp.h
+++ b/lib/compress/zstd_cwksp.h
@@ -14,6 +14,7 @@
 /*-*************************************
 *  Dependencies
 ***************************************/
+#include "../common/allocations.h"  /* ZSTD_customMalloc, ZSTD_customFree */
 #include "../common/zstd_internal.h"
 
 #if defined (__cplusplus)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 7a2c71720a123cec0bac72463dccca3cbfbef4ef..bf2de4a79b54d977c03a13a6dec203abd85aae99 100644 (file)
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -20,6 +20,7 @@
 
 
 /* ======   Dependencies   ====== */
+#include "../common/allocations.h"  /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
 #include "../common/zstd_deps.h"   /* ZSTD_memcpy, ZSTD_memset, INT_MAX, UINT_MAX */
 #include "../common/mem.h"         /* MEM_STATIC */
 #include "../common/pool.h"        /* threadpool */

diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c
index ad5c34a7fc08ed4099b3dd554374b90851bd7ce2..309ec0d03642a1372aa9130eed51bc56d9d67ab5 100644 (file)
--- a/lib/decompress/zstd_ddict.c
+++ b/lib/decompress/zstd_ddict.c
@@ -14,6 +14,7 @@
 /*-*******************************************************
 *  Dependencies
 *********************************************************/
+#include "../common/allocations.h"  /* ZSTD_customMalloc, ZSTD_customFree */
 #include "../common/zstd_deps.h"   /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
 #include "../common/cpu.h"         /* bmi2 */
 #include "../common/mem.h"         /* low level memory routines */

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index d487966ccf1c2c21e442cbb00539ec120fc8c118..3f3ca57c068e2a19cf550d40d772a277c17111a7 100644 (file)
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -55,6 +55,7 @@
 /*-*******************************************************
 *  Dependencies
 *********************************************************/
+#include "../common/allocations.h"  /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
 #include "../common/zstd_deps.h"   /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
 #include "../common/mem.h"         /* low level memory routines */
 #define FSE_STATIC_LINKING_ONLY

diff --git a/programs/Makefile b/programs/Makefile
index fcff41dc1b3e0f845cd890bffa9128b1a29952e5..8507abef3f079765c2af970e4067d2039665eba5 100644 (file)
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -225,7 +225,7 @@ zstd-noxz : zstd
 .PHONY: zstd-dll
 zstd-dll : LDFLAGS+= -L$(LIBZSTD)
 zstd-dll : LDLIBS += -lzstd
-zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c
+zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c pool.c threading.c
 zstd-dll : zstd
 
 

diff --git a/tests/Makefile b/tests/Makefile
index 3eed19ea80c088b03167fb6d2094b3082ef26a71..778c7d675976d1744175ccc0e6fce2ca5ac698a2 100644 (file)
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -95,7 +95,7 @@ allnothread: fullbench fuzzer paramgrill datagen decodecorpus
 dll: fuzzer-dll zstreamtest-dll
 
 .PHONY: zstd zstd32 zstd-nolegacy  # only external makefile knows how to build or update them
-zstd zstd32 zstd-nolegacy:
+zstd zstd32 zstd-nolegacy zstd-dll:
        $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
 
 .PHONY: libzstd
@@ -328,13 +328,17 @@ test-all: test test32 test-decodecorpus-cli
 test-zstd: ZSTD = $(PRGDIR)/zstd
 test-zstd: zstd
 
+.PHONY: test-zstd-dll
+test-zstd-dll: ZSTD = $(PRGDIR)/zstd
+test-zstd-dll: zstd-dll
+
 test-zstd32: ZSTD = $(PRGDIR)/zstd32
 test-zstd32: zstd32
 
 test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy
 test-zstd-nolegacy: zstd-nolegacy
 
-test-zstd test-zstd32 test-zstd-nolegacy: datagen
+test-zstd test-zstd32 test-zstd-nolegacy test-zstd-dll: datagen
        file $(ZSTD)
        EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)