From efeefe0759fc93399e2874146fe57a5b65c99765 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 25 Feb 2024 10:35:07 +0100 Subject: [PATCH] build: Improve handling of hiredis and zstd dependencies Introduced a new CMake variable called DEPS with the following semantics: - AUTO (the default): Use dependencies from the local system if available. Otherwise: Use bundled dependencies if available. Otherwise: Download dependencies from the internet (dependencies will then be linked statically). - DOWNLOAD: Use bundled dependencies if available. Otherwise: Download recommended versions from the internet (dependencies will then be linked statically). - LOCAL: Use dependencies from the local system if available. Otherwise: Use bundled dependencies if available. The old {HIREDIS,ZSTD}_FROM_INTERNET and OFFLINE variables have been removed in favor of DEPS. Also streamlined the CMake code for hiredis and zstd. --- .clang-format | 7 ++- .github/workflows/build.yaml | 66 +++++++++++--------- CMakeLists.txt | 14 +++++ ci/build-macos-binary | 13 ++-- cmake/DefaultBuildType.cmake | 2 +- cmake/Dependencies.cmake | 28 ++++----- cmake/FindHiredis.cmake | 76 +++++++++++++++++++++++ cmake/FindZstd.cmake | 71 +++++++++++++++++++++ cmake/Findhiredis.cmake | 117 ----------------------------------- cmake/Findzstd.cmake | 100 ------------------------------ cmake/Utils.cmake | 49 +++++++++++++++ doc/INSTALL.md | 83 +++++++++++++------------ misc/test-all-systems | 4 +- src/ccache/CMakeLists.txt | 8 +-- src/ccache/util/zstd.cpp | 9 +-- 15 files changed, 323 insertions(+), 324 deletions(-) create mode 100644 cmake/FindHiredis.cmake create mode 100644 cmake/FindZstd.cmake delete mode 100644 cmake/Findhiredis.cmake delete mode 100644 cmake/Findzstd.cmake diff --git a/.clang-format b/.clang-format index d18311d43..bc2b62430 100644 --- a/.clang-format +++ b/.clang-format @@ -30,12 +30,15 @@ IncludeCategories: # Ccache headers: - Regex: '^$' + Priority: 3 # System headers: - Regex: '\.h.*>$' - Priority: 3 + Priority: 4 # C++ headers: - Regex: '^<' - Priority: 4 + Priority: 5 IndentPPDirectives: AfterHash KeepEmptyLinesAtTheStartOfBlocks: false PointerAlignment: Left diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5b162335c..b718bcd8b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -99,22 +99,23 @@ jobs: run: | sudo apt-get update - packages=" + cmake_params=(-D CMAKE_BUILD_TYPE=CI) + packages=( elfutils libhiredis-dev libzstd-dev ninja-build - pkg-config python3 redis-server redis-tools - " + ) # Install ld.gold (binutils) and ld.lld (lld) on different runs. if [ "${{ matrix.os }}" = "ubuntu-22.04" ]; then - sudo apt-get install -y $packages binutils + packages+=(binutils) else - sudo apt-get install -y $packages lld + packages+=(lld) fi + sudo apt-get install -y "${packages[@]}" if [ "${{ matrix.compiler }}" = "gcc" ]; then echo "CC=gcc-${{ matrix.version }}" >> $GITHUB_ENV @@ -135,11 +136,20 @@ jobs: sudo apt-get install -y clang-${{ matrix.version }} g++-multilib lld-${{ matrix.version }} fi + case "${{ matrix.os }}" in + ubuntu-*) + cmake_params+=(-D DEPS=LOCAL) + ;; + *) + ;; + esac + echo "CMAKE_PARAMS=${cmake_params[*]}" >> $GITHUB_ENV + - name: Install dependencies on macOS if: ${{ runner.os == 'macOS' }} run: | HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 \ - brew install ninja pkg-config hiredis redis + brew install ninja hiredis redis if [ "${{ matrix.compiler }}" = "gcc" ]; then brew install gcc@${{ matrix.version }} @@ -156,8 +166,6 @@ jobs: - name: Build and test run: ci/build - env: - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI - name: Collect testdir from failed tests if: failure() @@ -274,7 +282,7 @@ jobs: BUILDDIR: . CCACHE_LOC: . CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Debug - apt_get: elfutils libzstd-dev pkg-config libhiredis-dev + apt_get: elfutils libhiredis-dev libzstd-dev - name: Linux GCC 32-bit os: ubuntu-20.04 @@ -283,26 +291,26 @@ jobs: CFLAGS: -m32 -g -O2 CXXFLAGS: -m32 -g -O2 LDFLAGS: -m32 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -D DEPS=AUTO apt_get: elfutils gcc-multilib g++-multilib lib32stdc++-10-dev - name: Linux GCC CUDA os: ubuntu-20.04 CC: gcc CXX: g++ - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -D DEPS=LOCAL CUDA: 11.7.0-1 - apt_get: elfutils libzstd-dev + apt_get: elfutils libzstd-dev libhiredis-dev - name: Linux MinGW 32-bit os: ubuntu-20.04 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -D CMAKE_TOOLCHAIN_FILE=../toolchains/i686-w64-mingw32-posix.cmake -D ZSTD_FROM_INTERNET=ON -D HIREDIS_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -D CMAKE_TOOLCHAIN_FILE=../toolchains/i686-w64-mingw32-posix.cmake RUN_TESTS: none apt_get: elfutils mingw-w64 - name: Linux MinGW 64-bit os: ubuntu-20.04 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -D CMAKE_TOOLCHAIN_FILE=../toolchains/x86_64-w64-mingw32-posix.cmake -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -D CMAKE_TOOLCHAIN_FILE=../toolchains/x86_64-w64-mingw32-posix.cmake RUN_TESTS: unittest-in-wine apt_get: elfutils mingw-w64 wine @@ -312,7 +320,7 @@ jobs: CC: cl CXX: cl CMAKE_GENERATOR: Ninja - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI TEST_CC: clang -target i686-pc-windows-msvc - name: Windows VS2019 32-bit MSBuild @@ -322,7 +330,7 @@ jobs: CC: cl CXX: cl CMAKE_GENERATOR: Visual Studio 16 2019 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON -A Win32 + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -A Win32 TEST_CC: clang -target i686-pc-windows-msvc - name: Windows VS2019 64-bit Ninja @@ -331,7 +339,7 @@ jobs: CC: cl CXX: cl CMAKE_GENERATOR: Ninja - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI TEST_CC: clang -target x86_64-pc-windows-msvc - name: Windows VS2019 64-bit MSBuild @@ -341,7 +349,7 @@ jobs: CC: cl CXX: cl CMAKE_GENERATOR: Visual Studio 16 2019 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON -A x64 + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -A x64 TEST_CC: clang -target x86_64-pc-windows-msvc - name: Windows VS2022 32-bit Ninja @@ -351,7 +359,7 @@ jobs: CC: cl CXX: cl CMAKE_GENERATOR: Ninja - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI TEST_CC: clang -target i686-pc-windows-msvc - name: Windows VS2022 32-bit MSBuild @@ -361,7 +369,7 @@ jobs: CC: cl CXX: cl CMAKE_GENERATOR: Visual Studio 17 2022 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON -A Win32 + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -A Win32 TEST_CC: clang -target i686-pc-windows-msvc - name: Windows VS2022 64-bit Ninja @@ -371,7 +379,7 @@ jobs: CC: cl CXX: cl CMAKE_GENERATOR: Ninja - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI TEST_CC: clang -target x86_64-pc-windows-msvc - name: Windows VS2022 64-bit MSBuild @@ -381,7 +389,7 @@ jobs: CC: cl CXX: cl CMAKE_GENERATOR: Visual Studio 17 2022 - CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON -DHIREDIS_FROM_INTERNET=ON -A x64 + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -A x64 TEST_CC: clang -target x86_64-pc-windows-msvc - name: Clang address & UB sanitizer @@ -390,7 +398,7 @@ jobs: CXX: clang++ CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DENABLE_SANITIZER_ADDRESS=ON -DENABLE_SANITIZER_UNDEFINED_BEHAVIOR=ON ASAN_OPTIONS: detect_leaks=0 - apt_get: elfutils libzstd-dev pkg-config libhiredis-dev + apt_get: elfutils libzstd-dev libhiredis-dev - name: Clang static analyzer os: ubuntu-20.04 @@ -398,7 +406,7 @@ jobs: CXX: clang++ CMAKE_PREFIX: scan-build RUN_TESTS: none - apt_get: libzstd-dev pkg-config libhiredis-dev + apt_get: libzstd-dev libhiredis-dev - name: Linux binary os: ubuntu-20.04 @@ -406,26 +414,26 @@ jobs: CXX: g++ SPECIAL: build-and-verify-package CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=Release - apt_get: elfutils libzstd-dev pkg-config libhiredis-dev ninja-build + apt_get: elfutils libzstd-dev libhiredis-dev ninja-build - name: Source package os: ubuntu-20.04 CC: gcc CXX: g++ SPECIAL: build-and-verify-source-package - apt_get: elfutils libzstd-dev pkg-config libhiredis-dev ninja-build asciidoctor + apt_get: elfutils libzstd-dev libhiredis-dev ninja-build asciidoctor - name: HTML documentation os: ubuntu-20.04 EXTRA_CMAKE_BUILD_FLAGS: --target doc-html RUN_TESTS: none - apt_get: libzstd-dev pkg-config libhiredis-dev asciidoctor + apt_get: libzstd-dev libhiredis-dev asciidoctor - name: Manual page os: ubuntu-20.04 EXTRA_CMAKE_BUILD_FLAGS: --target doc-man-page RUN_TESTS: none - apt_get: libzstd-dev pkg-config libhiredis-dev asciidoctor + apt_get: libzstd-dev libhiredis-dev asciidoctor - name: Clang-Tidy os: ubuntu-20.04 @@ -433,7 +441,7 @@ jobs: CXX: clang++-12 RUN_TESTS: none CMAKE_PARAMS: -DENABLE_CLANG_TIDY=ON -DCLANGTIDY=/usr/bin/clang-tidy-12 - apt_get: libzstd-dev pkg-config libhiredis-dev clang-12 clang-tidy-12 + apt_get: libzstd-dev libhiredis-dev clang-12 clang-tidy-12 steps: - name: Get source diff --git a/CMakeLists.txt b/CMakeLists.txt index 4088f8f30..410d70ed6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -165,3 +165,17 @@ add_custom_target( COMMENT "Checking code formatting" USES_TERMINAL WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + +# +# Configuration summary +# + +message(STATUS) +message(STATUS "Configuration summary:") +message(STATUS " Storage backends:") +message(STATUS " file ON") +message(STATUS " http ON") +message(STATUS " redis ${REDIS_STORAGE_BACKEND}") +message(STATUS " Dependencies:") +print_dependency_summary(" ") +message(STATUS) diff --git a/ci/build-macos-binary b/ci/build-macos-binary index f2e880404..e076ef614 100755 --- a/ci/build-macos-binary +++ b/ci/build-macos-binary @@ -6,13 +6,12 @@ build() { mkdir -p "build_$1" cd "build_$1" cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DZSTD_FROM_INTERNET=ON \ - -DHIREDIS_FROM_INTERNET=ON \ - -DCMAKE_OSX_DEPLOYMENT_TARGET="10.15" \ - -DCMAKE_OSX_ARCHITECTURES="$1" \ - -DCMAKE_SYSTEM_PROCESSOR="$1" \ - -DENABLE_TESTING=OFF \ + -D CMAKE_BUILD_TYPE=Release \ + -D CMAKE_OSX_DEPLOYMENT_TARGET="10.15" \ + -D CMAKE_OSX_ARCHITECTURES="$1" \ + -D CMAKE_SYSTEM_PROCESSOR="$1" \ + -D DEPS=DOWNLOAD \ + -D ENABLE_TESTING=OFF \ .. cmake --build . cd .. diff --git a/cmake/DefaultBuildType.cmake b/cmake/DefaultBuildType.cmake index 630ecfb8c..96991c68f 100644 --- a/cmake/DefaultBuildType.cmake +++ b/cmake/DefaultBuildType.cmake @@ -17,7 +17,7 @@ else() endif() message( STATUS - "Setting CMAKE_BUILD_TYPE to ${CMAKE_BUILD_TYPE} as none was specified." + "Setting CMAKE_BUILD_TYPE to ${CMAKE_BUILD_TYPE} as none was specified" ) # Set the possible values of build type for CMake UIs. diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 372405272..d61a81668 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -1,26 +1,26 @@ +include(FindPackageHandleStandardArgs) + if(POLICY CMP0135) # Set timestamps on extracted files to time of extraction. cmake_policy(SET CMP0135 NEW) endif() -set(ZSTD_FROM_INTERNET AUTO CACHE STRING "Download and use libzstd from the Internet") -set_property(CACHE ZSTD_FROM_INTERNET PROPERTY STRINGS AUTO ON OFF) - -set(HIREDIS_FROM_INTERNET AUTO CACHE STRING "Download and use libhiredis from the Internet") -set_property(CACHE HIREDIS_FROM_INTERNET PROPERTY STRINGS AUTO ON OFF) +if(NOT DEPS AND FETCHCONTENT_FULLY_DISCONNECTED) + message(STATUS "Setting DEPS=LOCAL as FETCHCONTENT_FULLY_DISCONNECTED is set") +endif() -option( - OFFLINE "Do not download anything from the internet" - "$(FETCHCONTENT_FULLY_DISCONNECTED}" -) +# Hint about usage of the previously supported OFFLINE variable. if(OFFLINE) - set(FETCHCONTENT_FULLY_DISCONNECTED ON) - set(ZSTD_FROM_INTERNET OFF) - set(HIREDIS_FROM_INTERNET OFF) + message(FATAL_ERROR "Please use -D DEPS=LOCAL instead of -D OFFLINE=ON") endif() -find_package(zstd 1.1.2 MODULE REQUIRED) +# How to locate/retrieve dependencies. See the Dependencies section in +# doc/INSTALL.md. +set(DEPS AUTO CACHE STRING "How to retrieve third party dependencies") +set_property(CACHE DEPS PROPERTY STRINGS AUTO DOWNLOAD LOCAL) + +find_package(Zstd 1.3.4 MODULE REQUIRED) if(REDIS_STORAGE_BACKEND) - find_package(hiredis 0.13.3 MODULE REQUIRED) + find_package(Hiredis 0.13.3 MODULE REQUIRED) endif() diff --git a/cmake/FindHiredis.cmake b/cmake/FindHiredis.cmake new file mode 100644 index 000000000..d7a8c8140 --- /dev/null +++ b/cmake/FindHiredis.cmake @@ -0,0 +1,76 @@ +mark_as_advanced(HIREDIS_INCLUDE_DIR HIREDIS_LIBRARY) + +if(DEPS STREQUAL "DOWNLOAD" OR DEP_HIREDIS STREQUAL "DOWNLOAD") + message(STATUS "Downloading Hiredis as requested") + set(_download_hiredis TRUE) +else() + find_path(HIREDIS_INCLUDE_DIR hiredis/hiredis.h) + find_library(HIREDIS_LIBRARY hiredis) + if(HIREDIS_INCLUDE_DIR AND HIREDIS_LIBRARY) + file(READ "${HIREDIS_INCLUDE_DIR}/hiredis/hiredis.h" _hiredis_h) + string(REGEX MATCH "#define HIREDIS_MAJOR +([0-9]+).*#define HIREDIS_MINOR +([0-9]+).*#define HIREDIS_PATCH +([0-9]+)" _ "${_hiredis_h}") + set(_hiredis_version_string "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") + if(NOT "${CMAKE_MATCH_0}" STREQUAL "" AND "${_hiredis_version_string}" VERSION_GREATER_EQUAL "${Hiredis_FIND_VERSION}") + message(STATUS "Using system Hiredis (${HIREDIS_LIBRARY})") + set(_hiredis_origin "SYSTEM (${HIREDIS_LIBRARY})") + add_library(dep_hiredis UNKNOWN IMPORTED) + set_target_properties( + dep_hiredis + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${HIREDIS_INCLUDE_DIR}" + IMPORTED_LOCATION "${HIREDIS_LIBRARY}" + ) + endif() + endif() + if(NOT _hiredis_origin) + if(DEPS STREQUAL "AUTO") + message(STATUS "Downloading Hiredis from the internet since Hiredis>=${Hiredis_FIND_VERSION} was not found locally and DEPS=AUTO") + set(_download_hiredis TRUE) + else() + message(FATAL_ERROR "Could not find Hiredis>=${Hiredis_FIND_VERSION}") + endif() + endif() +endif() + +if(_download_hiredis) + set(_hiredis_origin DOWNLOADED) + set(_hiredis_version_string 1.2.0) + + include(FetchContent) + FetchContent_Declare( + Hiredis + URL "https://github.com/redis/hiredis/archive/refs/tags/v${_hiredis_version_string}.tar.gz" + URL_HASH SHA256=82ad632d31ee05da13b537c124f819eb88e18851d9cb0c30ae0552084811588c + ) + + # Intentionally not using hiredis's build system since it doesn't put headers + # in a hiredis subdirectory. + FetchContent_Populate(Hiredis) + set( + _hiredis_sources + "${hiredis_SOURCE_DIR}/alloc.c" + "${hiredis_SOURCE_DIR}/async.c" + "${hiredis_SOURCE_DIR}/dict.c" + "${hiredis_SOURCE_DIR}/hiredis.c" + "${hiredis_SOURCE_DIR}/net.c" + "${hiredis_SOURCE_DIR}/read.c" + "${hiredis_SOURCE_DIR}/sds.c" + "${hiredis_SOURCE_DIR}/sockcompat.c" + ) + add_library(dep_hiredis STATIC EXCLUDE_FROM_ALL "${_hiredis_sources}") + if(WIN32) + target_compile_definitions(dep_hiredis PRIVATE _CRT_SECURE_NO_WARNINGS) + endif() + make_directory("${hiredis_SOURCE_DIR}/include/hiredis") + file(GLOB _hiredis_headers "${hiredis_SOURCE_DIR}/*.h") + file(COPY ${_hiredis_headers} DESTINATION "${hiredis_SOURCE_DIR}/include/hiredis") + target_include_directories( + dep_hiredis SYSTEM INTERFACE "$" + ) +endif() + +if(WIN32) + target_link_libraries(dep_hiredis INTERFACE ws2_32) +endif() + +register_dependency(Hiredis "${_hiredis_origin}" "${_hiredis_version_string}") diff --git a/cmake/FindZstd.cmake b/cmake/FindZstd.cmake new file mode 100644 index 000000000..7630cd9dd --- /dev/null +++ b/cmake/FindZstd.cmake @@ -0,0 +1,71 @@ +mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY) + +if(DEPS STREQUAL "DOWNLOAD" OR DEP_ZSTD STREQUAL "DOWNLOAD") + message(STATUS "Downloading Zstd as requested") + set(_download_zstd TRUE) +else() + find_path(ZSTD_INCLUDE_DIR zstd.h) + find_library(ZSTD_LIBRARY zstd) + if(ZSTD_INCLUDE_DIR AND ZSTD_LIBRARY) + file(READ "${ZSTD_INCLUDE_DIR}/zstd.h" _zstd_h) + string(REGEX MATCH "#define ZSTD_VERSION_MAJOR +([0-9]+).*#define ZSTD_VERSION_MINOR +([0-9]+).*#define ZSTD_VERSION_RELEASE +([0-9]+)" _ "${_zstd_h}") + set(_zstd_version_string "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") + if(NOT "${CMAKE_MATCH_0}" STREQUAL "" AND "${_zstd_version_string}" VERSION_GREATER_EQUAL "${Zstd_FIND_VERSION}") + message(STATUS "Using system Zstd (${ZSTD_LIBRARY})") + set(_zstd_origin "SYSTEM (${ZSTD_LIBRARY})") + add_library(dep_zstd UNKNOWN IMPORTED) + set_target_properties( + dep_zstd + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}" + IMPORTED_LOCATION "${ZSTD_LIBRARY}" + ) + endif() + endif() + if(NOT _zstd_origin) + if(DEPS STREQUAL "AUTO") + message(STATUS "Downloading Zstd from the internet since Zstd>=${Zstd_FIND_VERSION} was not found locally and DEPS=AUTO") + set(_download_zstd TRUE) + else() + message(FATAL_ERROR "Could not find Zstd>=${Zstd_FIND_VERSION}") + endif() + endif() +endif() + +if(_download_zstd) + set(_zstd_version_string 1.5.5) + + if(XCODE) + # See https://github.com/facebook/zstd/pull/3665 + set(_zstd_patch PATCH_COMMAND sed -i .bak -e "s/^set_source_files_properties.*PROPERTIES.*LANGUAGE.*C/\\#&/ build/cmake/lib/CMakeLists.txt") + endif() + + set(ZSTD_BUILD_PROGRAMS OFF) + set(ZSTD_BUILD_SHARED OFF) + set(ZSTD_BUILD_STATIC ON) + set(ZSTD_BUILD_TESTS OFF) + + include(FetchContent) + FetchContent_Declare( + Zstd + URL "https://github.com/facebook/zstd/releases/download/v${_zstd_version_string}/zstd-${_zstd_version_string}.tar.gz" + URL_HASH SHA256=9c4396cc829cfae319a6e2615202e82aad41372073482fce286fac78646d3ee4 + SOURCE_SUBDIR build/cmake + ${_zstd_patch} + ) + FetchContent_MakeAvailable(Zstd) + + # Workaround until https://github.com/facebook/zstd/pull/3968 is included in a + # release: + set_target_properties(libzstd_static PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "$") + + unset(ZSTD_BUILD_PROGRAMS) + unset(ZSTD_BUILD_SHARED) + unset(ZSTD_BUILD_STATIC) + unset(ZSTD_BUILD_TESTS) + + set(_zstd_origin DOWNLOADED) + add_library(dep_zstd ALIAS libzstd_static) +endif() + +register_dependency(Zstd "${_zstd_origin}" "${_zstd_version_string}") diff --git a/cmake/Findhiredis.cmake b/cmake/Findhiredis.cmake deleted file mode 100644 index 9a72a309d..000000000 --- a/cmake/Findhiredis.cmake +++ /dev/null @@ -1,117 +0,0 @@ -if(hiredis_FOUND) - return() -endif() - -set(hiredis_FOUND FALSE) - -if(HIREDIS_FROM_INTERNET AND NOT HIREDIS_FROM_INTERNET STREQUAL "AUTO") - message(STATUS "Using hiredis from the Internet") - set(do_download TRUE) -else() - find_package(PkgConfig) - if(PKG_CONFIG_FOUND) - pkg_check_modules(HIREDIS hiredis>=${hiredis_FIND_VERSION}) - find_library(HIREDIS_LIBRARY ${HIREDIS_LIBRARIES} HINTS ${HIREDIS_LIBDIR}) - find_path(HIREDIS_INCLUDE_DIR hiredis/hiredis.h HINTS ${HIREDIS_PREFIX}/include) - if(HIREDIS_LIBRARY AND HIREDIS_INCLUDE_DIR) - message(STATUS "Using hiredis from ${HIREDIS_LIBRARY} via pkg-config") - set(hiredis_FOUND TRUE) - endif() - endif() - - if(NOT hiredis_FOUND) - find_library(HIREDIS_LIBRARY hiredis) - find_path(HIREDIS_INCLUDE_DIR hiredis/hiredis.h) - if(HIREDIS_LIBRARY AND HIREDIS_INCLUDE_DIR) - message(STATUS "Using hiredis from ${HIREDIS_LIBRARY}") - set(hiredis_FOUND TRUE) - endif() - endif() - - if(hiredis_FOUND) - mark_as_advanced(HIREDIS_INCLUDE_DIR HIREDIS_LIBRARY) - add_library(HIREDIS::HIREDIS UNKNOWN IMPORTED) - set_target_properties( - HIREDIS::HIREDIS - PROPERTIES - IMPORTED_LOCATION "${HIREDIS_LIBRARY}" - INTERFACE_COMPILE_OPTIONS "${HIREDIS_CFLAGS_OTHER}" - INTERFACE_INCLUDE_DIRECTORIES "${HIREDIS_INCLUDE_DIR}" - ) - if(WIN32 AND STATIC_LINK) - target_link_libraries(HIREDIS::HIREDIS INTERFACE ws2_32) - endif() - set(hiredis_FOUND TRUE) - set(target HIREDIS::HIREDIS) - elseif(HIREDIS_FROM_INTERNET STREQUAL "AUTO") - message(STATUS "*** WARNING ***: Using hiredis from the Internet because it was not found and HIREDIS_FROM_INTERNET is AUTO") - set(do_download TRUE) - else() - include(FindPackageHandleStandardArgs) - find_package_handle_standard_args( - hiredis - REQUIRED_VARS HIREDIS_LIBRARY HIREDIS_INCLUDE_DIR - ) - endif() -endif() - -if(do_download) - set(hiredis_version "1.2.0") - set(hiredis_dir ${CMAKE_BINARY_DIR}/hiredis-${hiredis_version}) - set(hiredis_build ${CMAKE_BINARY_DIR}/hiredis-build) - - include(FetchContent) - FetchContent_Declare( - hiredis - URL https://github.com/redis/hiredis/archive/refs/tags/v${hiredis_version}.tar.gz - SOURCE_DIR ${hiredis_dir} - BINARY_DIR ${hiredis_build} - ) - - FetchContent_GetProperties(hiredis) - if(NOT hiredis_POPULATED) - FetchContent_Populate(hiredis) - endif() - - set( - hiredis_sources - "${hiredis_dir}/alloc.c" - "${hiredis_dir}/async.c" - "${hiredis_dir}/dict.c" - "${hiredis_dir}/hiredis.c" - "${hiredis_dir}/net.c" - "${hiredis_dir}/read.c" - "${hiredis_dir}/sds.c" - "${hiredis_dir}/sockcompat.c" - ) - add_library(libhiredis_static STATIC EXCLUDE_FROM_ALL ${hiredis_sources}) - add_library(HIREDIS::HIREDIS ALIAS libhiredis_static) - - if(WIN32) - target_compile_definitions(libhiredis_static PRIVATE _CRT_SECURE_NO_WARNINGS) - endif() - - make_directory("${hiredis_dir}/include") - make_directory("${hiredis_dir}/include/hiredis") - file(GLOB hiredis_headers "${hiredis_dir}/*.h") - file(COPY ${hiredis_headers} DESTINATION "${hiredis_dir}/include/hiredis") - set_target_properties( - libhiredis_static - PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "$") - - set(hiredis_FOUND TRUE) - set(target libhiredis_static) -endif() - -if(WIN32 AND hiredis_FOUND) - target_link_libraries(${target} INTERFACE ws2_32) -endif() - -include(FeatureSummary) -set_package_properties( - hiredis - PROPERTIES - URL "https://github.com/redis/hiredis" - DESCRIPTION "Hiredis is a minimalistic C client library for the Redis database" -) diff --git a/cmake/Findzstd.cmake b/cmake/Findzstd.cmake deleted file mode 100644 index c409a0dfa..000000000 --- a/cmake/Findzstd.cmake +++ /dev/null @@ -1,100 +0,0 @@ -if(zstd_FOUND) - return() -endif() - -set(zstd_FOUND FALSE) - -if(ZSTD_FROM_INTERNET AND NOT ZSTD_FROM_INTERNET STREQUAL "AUTO") - message(STATUS "Using zstd from the Internet") - set(do_download TRUE) -else() - find_package(PkgConfig) - if(PKG_CONFIG_FOUND) - pkg_search_module(PC_ZSTD libzstd) - find_library(ZSTD_LIBRARY zstd HINTS ${PC_ZSTD_LIBDIR} ${PC_ZSTD_LIBRARY_DIRS}) - find_path(ZSTD_INCLUDE_DIR zstd.h HINTS ${PC_ZSTD_INCLUDEDIR} ${PC_ZSTD_INCLUDE_DIRS}) - if(ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR) - message(STATUS "Using zstd from ${ZSTD_LIBRARY} via pkg-config") - set(zstd_FOUND TRUE) - endif() - endif() - - if(NOT zstd_FOUND) - find_library(ZSTD_LIBRARY zstd) - find_path(ZSTD_INCLUDE_DIR zstd.h) - if(ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR) - message(STATUS "Using zstd from ${ZSTD_LIBRARY}") - set(zstd_FOUND TRUE) - endif() - endif() - - if(zstd_FOUND) - mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY) - add_library(ZSTD::ZSTD UNKNOWN IMPORTED) - set_target_properties( - ZSTD::ZSTD - PROPERTIES - IMPORTED_LOCATION "${ZSTD_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}" - ) - set(zstd_FOUND TRUE) - elseif(ZSTD_FROM_INTERNET STREQUAL "AUTO") - message(STATUS "*** WARNING ***: Using zstd from the Internet because it was not found and ZSTD_FROM_INTERNET is AUTO") - set(do_download TRUE) - else() - include(FindPackageHandleStandardArgs) - find_package_handle_standard_args( - zstd - REQUIRED_VARS ZSTD_LIBRARY ZSTD_INCLUDE_DIR - ) - endif() -endif() - -if(do_download) - # Although ${zstd_FIND_VERSION} was requested, let's download a newer version. - # Note: The directory structure has changed in 1.3.0; we only support 1.3.0 - # and newer. - set(zstd_version "1.5.5") - set(zstd_dir ${CMAKE_BINARY_DIR}/zstd-${zstd_version}) - set(zstd_build ${CMAKE_BINARY_DIR}/zstd-build) - - if(XCODE) - # See https://github.com/facebook/zstd/pull/3665 - set(zstd_patch PATCH_COMMAND sed -i .bak -e s/^set_source_files_properties.*PROPERTIES.*LANGUAGE.*C/\#&/ build/cmake/lib/CMakeLists.txt) - endif() - - include(FetchContent) - FetchContent_Declare( - zstd - URL https://github.com/facebook/zstd/releases/download/v${zstd_version}/zstd-${zstd_version}.tar.gz - URL_HASH SHA256=9c4396cc829cfae319a6e2615202e82aad41372073482fce286fac78646d3ee4 - SOURCE_DIR ${zstd_dir} - BINARY_DIR ${zstd_build} - ${zstd_patch} - ) - - FetchContent_GetProperties(zstd) - if(NOT zstd_POPULATED) - FetchContent_Populate(zstd) - endif() - - set(ZSTD_BUILD_SHARED OFF) - add_subdirectory("${zstd_dir}/build/cmake" "${zstd_build}" EXCLUDE_FROM_ALL) - - add_library(ZSTD::ZSTD ALIAS libzstd_static) - set_target_properties( - libzstd_static - PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "$" - ) - - set(zstd_FOUND TRUE) -endif() - -include(FeatureSummary) -set_package_properties( - zstd - PROPERTIES - URL "https://facebook.github.io/zstd" - DESCRIPTION "Zstandard - Fast real-time compression algorithm" -) diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake index 431b8d298..e89f9edee 100644 --- a/cmake/Utils.cmake +++ b/cmake/Utils.cmake @@ -31,3 +31,52 @@ endmacro() macro(add_compile_flag_if_supported varname flag) add_compile_flag_if_supported_ex("${varname}" "${flag}" "") endmacro() + +set(dependencies "" CACHE INTERNAL "") + +function(register_dependency name origin version) + list(APPEND dependencies "${name}:${origin}:${version}") + set(dependencies "${dependencies}" CACHE INTERNAL "") +endfunction() + +function(print_dependency_summary prefix) + list(SORT dependencies) + + list(LENGTH dependencies n_deps) + math(EXPR n_deps_minus_1 "${n_deps} - 1") + + set(max_name_length 0) + set(max_version_length 0) + foreach(entry IN LISTS dependencies) + string(REPLACE ":" ";" parts "${entry}") + list(GET parts 0 name) + list(GET parts 2 version) + + string(LENGTH "${name}" name_length) + if("${name_length}" GREATER "${max_name_length}") + set(max_name_length "${name_length}") + endif() + + string(LENGTH "${version}" version_length) + if("${version_length}" GREATER "${max_version_length}") + set(max_version_length "${version_length}") + endif() + endforeach() + + foreach(entry IN LISTS dependencies) + string(REPLACE ":" ";" parts "${entry}") + list(GET parts 0 name) + list(GET parts 1 origin) + list(GET parts 2 version) + + string(LENGTH "${name}" name_length) + math(EXPR pad_count "${max_name_length} - ${name_length}") + string(REPEAT " " "${pad_count}" name_pad) + + string(LENGTH "${version}" version_length) + math(EXPR pad_count "${max_version_length} - ${version_length}") + string(REPEAT " " "${pad_count}" version_pad) + + message(STATUS "${prefix}${name}${name_pad} ${version}${version_pad} ${origin}") + endforeach() +endfunction() diff --git a/doc/INSTALL.md b/doc/INSTALL.md index 05dd77d05..c42f2a688 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -1,8 +1,6 @@ -Ccache installation -=================== +# Ccache installation -Prerequisites -------------- +## Prerequisites To build ccache you need: @@ -11,59 +9,66 @@ To build ccache you need: languages](https://ccache.dev/platform-compiler-language-support.html) for details. - A C99 compiler. -- [libzstd](http://www.zstd.net). If you don't have libzstd installed and can't - or don't want to install it in a standard system location, it will be - automatically downloaded, built and linked statically as part of the build - process. To disable this, pass `-DOFFLINE=TRUE` or `-DZSTD_FROM_INTERNET=OFF` - to `cmake`, or pass `-DZSTD_FROM_INTERNET=ON` to force downloading. You can - also install zstd in a custom path and pass - `-DCMAKE_PREFIX_PATH=/some/custom/path` to `cmake`. - - To link libzstd statically (and you have a static libzstd available), pass - `-DSTATIC_LINK=ON` to `cmake`. This is the default on Windows. Alternatively, - use `-DZSTD_LIBRARY=/path/to/libzstd.a`. +- Various software libraries, see _Dependencies_ below. Optional: -- [hiredis](https://github.com/redis/hiredis) for the Redis storage backend. If - you don't have libhiredis installed and can't or don't want to install it in - a standard system location, it will be automatically downloaded, built and - linked statically as part of the build process. To disable this, pass - `-DOFFLINE=TRUE` or `-DHIREDIS_FROM_INTERNET=OFF` to `cmake`, or pass - `-DHIREDIS_FROM_INTERNET=ON` to force downloading. You can also install - hiredis in a custom path and pass `-DCMAKE_PREFIX_PATH=/some/custom/path` to - `cmake`. - - To link libhiredis statically (and you have a static libhiredis available), - pass `-DSTATIC_LINK=ON` to `cmake`. This is the default on Windows. - Alternatively, use `-DHIREDIS_LIBRARY=/path/to/libhiredis.a`. - GNU Bourne Again SHell (bash) for tests. - [Asciidoctor](https://asciidoctor.org) to build the HTML documentation. - [Python](https://www.python.org) to debug and run the performance test suite. -Reference configurations: +See also [CI configurations](../.github/workflows/build.yaml) for regularly +tested build setups, including cross-compilation and dependencies required in +Debian/Ubuntu environments. -- See [CI configurations](../.github/workflows/build.yaml) for a selection of - regularly tested build setups, including cross-compiling and explicit - dependencies required in Debian/Ubuntu environment. +## Software library dependencies -Installation ------------- +### How to locate or retrieve -Here is the typical way to build and install ccache: +The CMake variable `DEPS` can be set to select how software library dependencies +should be located or retrieved: + +- `AUTO` (the default): Use dependencies from the local system if available. + Otherwise: Use bundled dependencies if available. Otherwise: Download + dependencies from the internet (dependencies will then be linked statically). +- `DOWNLOAD`: Use bundled dependencies if available. Otherwise: Download + dependencies from the internet (dependencies will then be linked + statically). +- `LOCAL`: Use dependencies from the local system if available. Otherwise: Use + bundled dependencies if available. + +### Dependencies + +- [hiredis](https://github.com/redis/hiredis)[^1] (optional, disable with `-D + REDIS_STORAGE_BACKEND=OFF`) +- [Zstandard](https://github.com/facebook/zstd)[^1] + +[^1]: A downloaded version will be used if missing locally. + +### Tips + +- To make CMake search for libraries in a custom location, use `-D + CMAKE_PREFIX_PATH=/some/custom/path`. +- To link libraries statically, pass `-D STATIC_LINK=ON` to CMake (this is the + default on Windows). Alternatively, use `-D + EXAMPLE_LIBRARY=/path/to/libexample.a` to link statically with libexample. + +## Installation + +Here is a typical way to build and install ccache: ```bash mkdir build cd build -cmake -DCMAKE_BUILD_TYPE=Release .. +cmake -D CMAKE_BUILD_TYPE=Release .. make make install ``` -You can set the installation directory to e.g. `/usr` by adding -`-DCMAKE_INSTALL_PREFIX=/usr` to the `cmake` command. You can set the directory -where the system configuration file should be located to e.g. `/etc` by adding -`-DCMAKE_INSTALL_SYSCONFDIR=/etc`. +You can set the installation directory to e.g. `/usr` by adding `-D +CMAKE_INSTALL_PREFIX=/usr` to the CMake command. You can set the directory where +the system configuration file should be located to e.g. `/etc` by adding `-D +CMAKE_INSTALL_SYSCONFDIR=/etc`. There are two different ways to use ccache to cache a compilation: diff --git a/misc/test-all-systems b/misc/test-all-systems index 8f8f52be3..982cef528 100755 --- a/misc/test-all-systems +++ b/misc/test-all-systems @@ -37,8 +37,8 @@ build ubuntu-22.04 clang clang++ clang build ubuntu-22.04 gcc-12 g++-12 gcc build ubuntu-22.04 clang clang++ clang -build centos-7 gcc g++ gcc -DHIREDIS_FROM_INTERNET=ON -build centos-7 gcc g++ clang -DHIREDIS_FROM_INTERNET=ON +build centos-7 gcc g++ gcc +build centos-7 gcc g++ clang build fedora-36 gcc g++ gcc build fedora-36 clang clang++ clang diff --git a/src/ccache/CMakeLists.txt b/src/ccache/CMakeLists.txt index bfd6ae330..c0eea21a9 100644 --- a/src/ccache/CMakeLists.txt +++ b/src/ccache/CMakeLists.txt @@ -44,7 +44,8 @@ set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) target_link_libraries( ccache_framework - PRIVATE standard_settings standard_warnings ZSTD::ZSTD Threads::Threads third_party + PUBLIC third_party + PRIVATE dep_zstd standard_settings standard_warnings Threads::Threads ) get_filename_component(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) @@ -52,10 +53,7 @@ target_include_directories(ccache_framework PUBLIC ${CMAKE_BINARY_DIR} ${SRC_DIR if(REDIS_STORAGE_BACKEND) target_compile_definitions(ccache_framework PUBLIC -DHAVE_REDIS_STORAGE_BACKEND) - target_link_libraries( - ccache_framework - PUBLIC standard_settings standard_warnings HIREDIS::HIREDIS third_party - ) + target_link_libraries(ccache_framework PRIVATE dep_hiredis) endif() add_executable(test-lockfile test_lockfile.cpp) diff --git a/src/ccache/util/zstd.cpp b/src/ccache/util/zstd.cpp index 81fe6f1ee..1bc75e473 100644 --- a/src/ccache/util/zstd.cpp +++ b/src/ccache/util/zstd.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2022-2023 Joel Rosdahl and other contributors +// Copyright (C) 2022-2024 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -72,13 +72,6 @@ zstd_compress_bound(size_t input_size) std::tuple zstd_supported_compression_level(int8_t wanted_level) { - // libzstd 1.3.4 and newer support negative levels. However, the query - // function ZSTD_minCLevel did not appear until 1.3.6, so perform detection - // based on version instead. - if (ZSTD_versionNumber() < 10304 && wanted_level < 1) { - return {1, "minimum level supported by libzstd"}; - } - const int8_t level = static_cast(std::min(wanted_level, ZSTD_maxCLevel())); if (level != wanted_level) { -- 2.47.2