From: Yann Collet Date: Fri, 16 Oct 2020 07:01:41 +0000 (-0700) Subject: faster rebuild of zstd X-Git-Tag: v1.4.7~48^2~7 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=80cee8d3fee5997d819c6cb24bfdae1a13c141a2;p=thirdparty%2Fzstd.git faster rebuild of zstd Building the zstd CLI costs time. Some part of it is incompressible, leading to substantial iteration delay when testing code modifications. That's mainly because all source files from the library must be rebuilt from source every time. The main reason we don't build the CLI from library object files is that we can't just build the object directly in the lib/ directory (which they would by default) since they use different compilation flags. Specifically, the CLI enables multithreading, while the library doesn't (by default). This is solved in this commit, by generating the object files locally. Now, the CLI and the library can employ different sets of flags, without tripping over each other. All library object files are generated directly into programs/ dir. This works because no 2 source files have the same name. Now, modifying a file doesn't require to recompile the entire lib, just the modified files. The recipe is also compatible with `-j` parallel build, leading to large build time reductions on multi-core systems. --- diff --git a/programs/Makefile b/programs/Makefile index 6cb4a57c2..4899c69c3 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -15,7 +15,7 @@ # zstd-decompress : decompressor-only version of zstd # ########################################################################## -ZSTDDIR = ../lib +ZSTDDIR := ../lib # Version numbers LIBVER_SRC := $(ZSTDDIR)/zstd.h @@ -43,41 +43,48 @@ else ALIGN_LOOP = endif -CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_ +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ ifeq ($(OS),Windows_NT) # MinGW assumed -CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif -CFLAGS ?= -O3 +CFLAGS ?= -O3 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef -Wpointer-arith \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wredundant-decls -Wmissing-prototypes -Wc++-compat -CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +ZSTDLIB_COMMON := $(ZSTDDIR)/common +ZSTDLIB_COMPRESS := $(ZSTDDIR)/compress +ZSTDLIB_DECOMPRESS := $(ZSTDDIR)/decompress +ZDICT_DIR := $(ZSTDDIR)/dictBuilder +ZSTDLEGACY_DIR := $(ZSTDDIR)/legacy -ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c -ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c -ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c -ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) -ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c -ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o +VPATH := $(ZSTDLIB_COMMON):$(ZSTDLIB_COMPRESS):$(ZSTDLIB_DECOMPRESS):$(ZDICT_DIR):$(ZSTDLEGACY_DIR) + +ZSTDLIB_COMMON_C := $(wildcard $(ZSTDLIB_COMMON)/*.c) +ZSTDLIB_COMPRESS_C := $(wildcard $(ZSTDLIB_COMPRESS)/*.c) +ZSTDLIB_DECOMPRESS_C := $(wildcard $(ZSTDLIB_DECOMPRESS)/*.c) +ZSTDLIB_CORE_SRC := $(ZSTDLIB_DECOMPRESS_C) $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) +ZDICT_SRC := $(wildcard $(ZDICT_DIR)/*.c) ZSTD_LEGACY_SUPPORT ?= 5 -ZSTDLEGACY_FILES := +ZSTDLEGACY_SRC := ifneq ($(ZSTD_LEGACY_SUPPORT), 0) ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) - ZSTDLEGACY_FILES += $(shell ls $(ZSTDDIR)/legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') + ZSTDLEGACY_SRC += $(notdir $(shell ls $(ZSTDLEGACY_DIR)/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')) endif -else endif # Sort files in alphabetical order for reproducible builds -ZSTDLIB_FILES := $(sort $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES))) +ZSTDLIB_FULL_SRC = $(sort $(ZSTDLIB_CORE_SRC) $(ZSTDLEGACY_SRC) $(ZDICT_SRC)) +ZSTDLIB_LOCAL_SRC = $(notdir $(ZSTDLIB_FULL_SRC)) +ZSTDLIB_LOCAL_OBJ = $(ZSTDLIB_LOCAL_SRC:.c=.o) ZSTD_CLI_FILES := $(wildcard *.c) -ZSTD_CLI_OBJ := $(patsubst %.c,%.o,$(ZSTD_CLI_FILES)) +ZSTD_CLI_OBJ := $(ZSTD_CLI_FILES:.c=.o) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -164,15 +171,13 @@ all: zstd .PHONY: allVariants allVariants: zstd zstd-compress zstd-decompress zstd-small zstd-nolegacy zstd-dictBuilder -$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) - zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP) zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD) zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) ifneq (,$(filter Windows%,$(OS))) zstd : $(RES_FILE) endif -zstd : $(ZSTDLIB_FILES) $(ZSTD_CLI_OBJ) +zstd : $(ZSTDLIB_LOCAL_OBJ) $(ZSTD_CLI_OBJ) @echo "$(THREAD_MSG)" @echo "$(ZLIB_MSG)" @echo "$(LZMA_MSG)" @@ -190,12 +195,12 @@ zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) ifneq (,$(filter Windows%,$(OS))) zstd32 : $(RES32_FILE) endif -zstd32 : $(ZSTDLIB_FILES) $(ZSTD_CLI_FILES) +zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_FILES) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) ## zstd-nolegacy: same scope as zstd, with just support of legacy formats removed zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD) -zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) $(ZSTD_CLI_OBJ) +zstd-nolegacy : $(ZSTDLIB_CORE_SRC) $(ZDICT_SRC) $(ZSTD_CLI_OBJ) $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) zstd-nomt : THREAD_CPP := @@ -222,11 +227,15 @@ zstd-noxz : zstd # It's unclear at this stage if this is a scenario that must be supported .PHONY: zstd-dll zstd-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd -zstd-dll : ZSTDLIB_FILES = +zstd-dll : ZSTDLIB_FULL_SRC = zstd-dll : $(ZSTD_CLI_OBJ) $(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS) +ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o + +$(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) + ## zstd-pgo: zstd executable optimized with PGO. zstd-pgo : $(MAKE) clean @@ -243,18 +252,18 @@ zstd-pgo : ## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format. zstd-small: CFLAGS = -Os -s -zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c util.c timefn.c fileio.c +zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT) -zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c timefn.c fileio.c +zstd-decompress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_DECOMPRESS_C) zstdcli.c util.c timefn.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT) -zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c timefn.c fileio.c +zstd-compress: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) zstdcli.c util.c timefn.c fileio.c $(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT) ## zstd-dictBuilder: executable supporting dictionary creation and compression (only) zstd-dictBuilder: CPPFLAGS += -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -zstd-dictBuilder: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(ZDICT_FILES) zstdcli.c util.c timefn.c fileio.c dibio.c +zstd-dictBuilder: $(ZSTDLIB_COMMON_C) $(ZSTDLIB_COMPRESS_C) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c dibio.c $(CC) $(FLAGS) $^ -o $@$(EXT) zstdmt: zstd