--- /dev/null
+#!/usr/bin/env python3
+# ################################################################
+# Copyright (c) 2020-2020, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ##########################################################################
+
+import argparse
+import contextlib
+import os
+import re
+import shutil
+import sys
+from typing import Optional
+
+INCLUDED_SUBDIRS = ["common", "compress", "decompress"]
+
+SKIPPED_FILES = [
+ "common/zstd_deps.h",
+ "common/pool.c",
+ "common/pool.h",
+ "common/threading.c",
+ "common/threading.h",
+ "compress/zstdmt_compress.h",
+ "compress/zstdmt_compress.c",
+]
+
+XXHASH_FILES = [
+ "common/xxhash.c",
+ "common/xxhash.h",
+]
+
+
+class FileLines(object):
+ def __init__(self, filename):
+ self.filename = filename
+ with open(self.filename, "r") as f:
+ self.lines = f.readlines()
+
+ def write(self):
+ with open(self.filename, "w") as f:
+ f.write("".join(self.lines))
+
+
+class Freestanding(object):
+ def __init__(
+ self,zstd_deps: str, source_lib: str, output_lib: str,
+ external_xxhash: bool, rewritten_includes: [(str, str)],
+ defs: [(str, Optional[str])], undefs: [str], excludes: [str]
+ ):
+ self._zstd_deps = zstd_deps
+ self._src_lib = source_lib
+ self._dst_lib = output_lib
+ self._external_xxhash = external_xxhash
+ self._rewritten_includes = rewritten_includes
+ self._defs = defs
+ self._undefs = undefs
+ self._excludes = excludes
+
+ def _dst_lib_file_paths(self):
+ """
+ Yields all the file paths in the dst_lib.
+ """
+ for root, dirname, filenames in os.walk(self._dst_lib):
+ for filename in filenames:
+ filepath = os.path.join(root, filename)
+ yield filepath
+
+ def _log(self, *args, **kwargs):
+ print(*args, **kwargs)
+
+ def _copy_file(self, lib_path):
+ if not (lib_path.endswith(".c") or lib_path.endswith(".h")):
+ return
+ if lib_path in SKIPPED_FILES:
+ self._log(f"\tSkipping file: {lib_path}")
+ return
+ if self._external_xxhash and lib_path in XXHASH_FILES:
+ self._log(f"\tSkipping xxhash file: {lib_path}")
+ return
+
+ src_path = os.path.join(self._src_lib, lib_path)
+ dst_path = os.path.join(self._dst_lib, lib_path)
+ self._log(f"\tCopying: {src_path} -> {dst_path}")
+ shutil.copyfile(src_path, dst_path)
+
+ def _copy_source_lib(self):
+ self._log("Copying source library into output library")
+
+ assert os.path.exists(self._src_lib)
+ os.makedirs(self._dst_lib, exist_ok=True)
+ self._copy_file("zstd.h")
+ for subdir in INCLUDED_SUBDIRS:
+ src_dir = os.path.join(self._src_lib, subdir)
+ dst_dir = os.path.join(self._dst_lib, subdir)
+
+ assert os.path.exists(src_dir)
+ os.makedirs(dst_dir, exist_ok=True)
+
+ for filename in os.listdir(src_dir):
+ lib_path = os.path.join(subdir, filename)
+ self._copy_file(lib_path)
+
+ def _copy_zstd_deps(self):
+ dst_zstd_deps = os.path.join(self._dst_lib, "common", "zstd_deps.h")
+ self._log(f"Copying zstd_deps: {self._zstd_deps} -> {dst_zstd_deps}")
+ shutil.copyfile(self._zstd_deps, dst_zstd_deps)
+
+ def _hardwire_preprocessor(self, name: str, value: Optional[str] = None, undef=False):
+ """
+ If value=None then hardwire that it is defined, but not what the value is.
+ If undef=True then value must be None.
+ If value='' then the macro is defined to '' exactly.
+ """
+ assert not (undef and value is not None)
+ for filepath in self._dst_lib_file_paths():
+ file = FileLines(filepath)
+
+ def _hardwire_defines(self):
+ self._log("Hardwiring defined macros")
+ for (name, value) in self._defs:
+ self._log(f"\tHardwiring: #define {name} {value}")
+ self._hardwire_preprocessor(name, value=value)
+ self._log("Hardwiring undefined macros")
+ for name in self._undefs:
+ self._log(f"\tHardwiring: #undef {name}")
+ self._hardwire_preprocessor(name, undef=True)
+
+ def _remove_excludes(self):
+ self._log("Removing excluded sections")
+ for exclude in self._excludes:
+ self._log(f"\tRemoving excluded sections for: {exclude}")
+ begin_re = re.compile(f"BEGIN {exclude}")
+ end_re = re.compile(f"END {exclude}")
+ for filepath in self._dst_lib_file_paths():
+ file = FileLines(filepath)
+ outlines = []
+ skipped = []
+ emit = True
+ for line in file.lines:
+ if emit and begin_re.search(line) is not None:
+ assert end_re.search(line) is None
+ emit = False
+ if emit:
+ outlines.append(line)
+ else:
+ skipped.append(line)
+ if end_re.search(line) is not None:
+ assert begin_re.search(line) is None
+ self._log(f"\t\tRemoving excluded section: {exclude}")
+ for s in skipped:
+ self._log(f"\t\t\t- {s}")
+ emit = True
+ skipped = []
+ if not emit:
+ raise RuntimeError("Excluded section unfinished!")
+ file.lines = outlines
+ file.write()
+
+ def _rewrite_include(self, original, rewritten):
+ self._log(f"\tRewriting include: {original} -> {rewritten}")
+ regex = re.compile(f"\\s*#\\s*include\\s*(?P<include>{original})")
+ for filepath in self._dst_lib_file_paths():
+ file = FileLines(filepath)
+ for i, line in enumerate(file.lines):
+ match = regex.match(line)
+ if match is None:
+ continue
+ s = match.start('include')
+ e = match.end('include')
+ file.lines[i] = line[:s] + rewritten + line[e:]
+ file.write()
+
+ def _rewrite_includes(self):
+ self._log("Rewriting includes")
+ for original, rewritten in self._rewritten_includes:
+ self._rewrite_include(original, rewritten)
+
+ def go(self):
+ self._copy_source_lib()
+ self._copy_zstd_deps()
+ self._hardwire_defines()
+ self._remove_excludes()
+ self._rewrite_includes()
+
+
+def parse_defines(defines: [str]) -> [(str, Optional[str])]:
+ output = []
+ for define in defines:
+ parsed = define.split('=')
+ if len(parsed) == 1:
+ output.append((parsed[0], None))
+ elif len(parsed) == 2:
+ output.append((parsed[0], parsed[1]))
+ else:
+ raise RuntimeError(f"Bad define: {define}")
+ return output
+
+
+def parse_rewritten_includes(rewritten_includes: [str]) -> [(str, str)]:
+ output = []
+ for rewritten_include in rewritten_includes:
+ parsed = rewritten_include.split('=')
+ if len(parsed) == 2:
+ output.append((parsed[0], parsed[1]))
+ else:
+ raise RuntimeError(f"Bad rewritten include: {rewritten_include}")
+ return output
+
+
+
+def main(name, args):
+ parser = argparse.ArgumentParser(prog=name)
+ parser.add_argument("--zstd-deps", default="zstd_deps.h", help="Zstd dependencies file")
+ parser.add_argument("--source-lib", default="../../lib", help="Location of the zstd library")
+ parser.add_argument("--output-lib", default="./freestanding_lib", help="Where to output the freestanding zstd library")
+ parser.add_argument("--xxhash", default=None, help="Alternate external xxhash include e.g. --xxhash='<xxhash.h>'. If set xxhash is not included.")
+ parser.add_argument("--rewrite-include", default=[], dest="rewritten_includes", action="append", help="Rewrite an include REGEX=NEW (e.g. '<stddef\\.h>=<linux/types.h>')")
+ parser.add_argument("-D", "--define", default=[], dest="defs", action="append", help="Pre-define this macro (can be passed multiple times)")
+ parser.add_argument("-U", "--undefine", default=[], dest="undefs", action="append", help="Pre-undefine this macro (can be passed mutliple times)")
+ parser.add_argument("-E", "--exclude", default=[], dest="excludes", action="append", help="Exclude all lines between 'BEGIN <EXCLUDE>' and 'END <EXCLUDE>'")
+ args = parser.parse_args(args)
+
+ # Always remove threading
+ if "ZSTD_MULTITHREAD" not in args.undefs:
+ args.undefs.append("ZSTD_MULTITHREAD")
+
+ args.defs = parse_defines(args.defs)
+ for name, _ in args.defs:
+ if name in args.undefs:
+ raise RuntimeError(f"{name} is both defined and undefined!")
+
+ args.rewritten_includes = parse_rewritten_includes(args.rewritten_includes)
+
+ external_xxhash = False
+ if args.xxhash is not None:
+ external_xxhash = True
+ args.rewritten_includes.append(('"(\\.\\./common/)?xxhash.h"', args.xxhash))
+
+ print(args.zstd_deps)
+ print(args.output_lib)
+ print(args.source_lib)
+ print(args.xxhash)
+ print(args.rewritten_includes)
+ print(args.defs)
+ print(args.undefs)
+
+ Freestanding(
+ args.zstd_deps,
+ args.source_lib,
+ args.output_lib,
+ external_xxhash,
+ args.rewritten_includes,
+ args.defs,
+ args.undefs,
+ args.excludes
+ ).go()
+
+if __name__ == "__main__":
+ main(sys.argv[0], sys.argv[1:])
--- /dev/null
+/*
+ * Copyright (c) 2016-2020, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+#ifndef __x86_64__
+# error This test only works on x86-64
+#endif
+
+#include "./libzstd/zstd.h"
+
+void* ZSTD_malloc(size_t size) {
+ (void)size;
+ return NULL;
+}
+void ZSTD_free(void* ptr) {
+ (void)ptr;
+}
+void* ZSTD_calloc(size_t num, size_t size) {
+ (void)num;
+ (void)size;
+ return NULL;
+}
+
+/* The standard requires these three functions be present.
+ * The compiler is free to insert calls to these functions.
+ */
+void* memmove(void* destination, const void* source, size_t num) {
+ char* const d = (char*)destination;
+ char const* const s = (char const*)source;
+ if (s < d) {
+ for (size_t i = num; i-- > 0;) {
+ d[i] = s[i];
+ }
+ } else {
+ for (size_t i = 0; i < num; ++i) {
+ d[i] = s[i];
+ }
+ }
+ return destination;
+}
+void* memcpy(void* destination, const void* source, size_t num) {
+ return memmove(destination, source, num);
+}
+void* memset(void* destination, int value, size_t num) {
+ char* const d = (char*)destination;
+ for (size_t i = 0; i < num; ++i) {
+ d[i] = value;
+ }
+ return destination;
+}
+
+void* ZSTD_memmove(void* destination, const void* source, size_t num) {
+ return memmove(destination, source, num);
+}
+void* ZSTD_memcpy(void* destination, const void* source, size_t num) {
+ return memmove(destination, source, num);
+}
+void* ZSTD_memset(void* destination, int value, size_t num) {
+ return memset(destination, value, num);
+}
+
+int main(void) {
+ char dst[100];
+ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+ ZSTD_freeCCtx(cctx);
+ if (cctx != NULL) {
+ return 1;
+ }
+ if (!ZSTD_isError(ZSTD_compress(dst, sizeof(dst), NULL, 0, 1))) {
+ return 2;
+ }
+ return 0;
+}
+
+static inline long syscall1(long syscall, long arg1) {
+ long ret;
+ __asm__ volatile
+ (
+ "syscall"
+ : "=a" (ret)
+ : "a" (syscall), "D" (arg1)
+ : "rcx", "r11", "memory"
+ );
+ return ret;
+}
+
+static inline void exit(int status) {
+ syscall1(60, status);
+}
+
+void _start(void) {
+ int const ret = main();
+ exit(ret);
+}
--- /dev/null
+/*
+ * Copyright (c) 2016-2020, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* Need:
+ * NULL
+ * ZSTD_memcpy()
+ * ZSTD_memset()
+ * ZSTD_memmove()
+ * BYTE
+ * S16
+ * U16
+ * U32
+ * U64
+ * size_t
+ * ptrdiff_t
+ * INT_MAX
+ * ...
+ */
+#ifndef ZSTD_DEPS_COMMON
+#define ZSTD_DEPS_COMMON
+
+#include <stddef.h>
+
+typedef unsigned char BYTE;
+typedef unsigned short U16;
+typedef signed short S16;
+typedef unsigned int U32;
+typedef signed int S32;
+typedef unsigned long long U64;
+typedef signed long long S64;
+
+#ifndef INT_MAX
+# define INT_MAX ((int)((1u << 31) - 1))
+#endif
+
+#if defined(__GNUC__) && __GNUC__ >= 4
+#define ZSTD_memcpy(d,s,n) __builtin_memcpy((d),(s),(n))
+#define ZSTD_memmove(d,s,n) __builtin_memmove((d),(s),(n))
+#define ZSTD_memset(d,s,n) __builtin_memset((d),(s),(n))
+#else
+void* ZSTD_memcpy(void* destination, const void* source, size_t num);
+void* ZSTD_memmove(void* destination, const void* source, size_t num);
+void* ZSTD_memset(void* destination, int value, size_t num);
+#endif
+
+#endif /* ZSTD_DEPS_COMMON */
+
+/* Need:
+ * ZSTD_malloc()
+ * ZSTD_free()
+ * ZSTD_calloc()
+ */
+#ifdef ZSTD_DEPS_NEED_MALLOC
+#ifndef ZSTD_DEPS_MALLOC
+#define ZSTD_DEPS_MALLOC
+
+void* ZSTD_malloc(size_t size);
+void ZSTD_free(void* ptr);
+void* ZSTD_calloc(size_t num, size_t size);
+
+#endif /* ZSTD_DEPS_MALLOC */
+#endif /* ZSTD_DEPS_NEED_MALLOC */
+
+/* Need:
+ * assert()
+ */
+#ifdef ZSTD_DEPS_NEED_ASSERT
+#ifndef ZSTD_DEPS_ASSERT
+#define ZSTD_DEPS_ASSERT
+
+#define assert(x) ((void)0)
+
+#endif /* ZSTD_DEPS_ASSERT */
+#endif /* ZSTD_DEPS_NEED_ASSERT */
+
+/* Need:
+ * ZSTD_DEBUG_PRINT()
+ */
+#ifdef ZSTD_DEPS_NEED_IO
+#ifndef ZSTD_DEPS_IO
+#define ZSTD_DEPS_IO
+
+#define ZSTD_DEBUG_PRINT(...)
+
+#endif /* ZSTD_DEPS_IO */
+#endif /* ZSTD_DEPS_NEED_IO */
+
+/* Only requested when <stdint.h> is known to be present.
+ * Need:
+ * intptr_t
+ */
+#ifdef ZSTD_DEPS_NEED_STDINT
+#ifndef ZSTD_DEPS_STDINT
+#define ZSTD_DEPS_STDINT
+
+#define intptr_t size_t
+
+#endif /* ZSTD_DEPS_STDINT */
+#endif /* ZSTD_DEPS_NEED_STDINT */
projects / thirdparty / zstd.git / commitdiff
