From: Luboš Luňák Date: Thu, 14 Nov 2019 19:17:22 +0000 (+0100) Subject: Allow -fmodules in direct depend mode (#482) X-Git-Tag: v4.0~711 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3eb2483f66fc482df3bf31616d35daedc5ac49cc;p=thirdparty%2Fccache.git Allow -fmodules in direct depend mode (#482) Clang creates the internal on-disk representation of modules in the background on demand as necessary. This means that ccache does not need to cache that, all that it needs to cache is the actual compilation result and the dependencies. Which in the case of modules includes also the module.modulemap files, and Clang outputs such dependencies correctly only in the depend mode, in the preprocessed output there is no way to find out the module dependencies. Therefore support -fmodules in direct depend mode by more or less ignoring it. Tested with source build of LLVM/Clang configured with -DLLVM_ENABLE_MODULES=On (and the build actually occassionally fails because of module problems, but that happens regardless of ccache). See also pull request #130 (and this reuses the tests from there). --- diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 6f05adeca..87d0cc642 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -673,6 +673,11 @@ still has to do _some_ preprocessing (like macros). *pch_defines*:: Be sloppy about **#define**s when precompiling a header file. See <<_precompiled_headers,PRECOMPILED HEADERS>> for more information. +*modules*:: + By default, ccache will not cache compilation if -fmodules is used, because + it cannot hash the state of compiler's internal representation of relevant + modules. This option allow caching in such case. + See <<_c_modules,C++ MODULES>> for more information. *system_headers*:: By default, ccache will also include all system headers in the manifest. With this option set, ccache will only include system headers in the hash @@ -859,6 +864,9 @@ The compiler was called for preprocessing, not compiling. Preconditions for using <<_precompiled_headers,precompiled headers>> were not fulfilled. +| can't use modules | +Preconditions for using <<_c_modules,C++ modules>> were not fulfilled. + | ccache internal error | Unexpected failure, e.g. due to problems reading/writing the cache. @@ -1194,6 +1202,23 @@ non-precompiled header file is not available). -- +C++ modules +----------- + +ccache has support for Clang's -fmodules option. In practice ccache only +additionally hashes module.modulemap files, it does not know how +Clang handles its cached binary form of modules, and so those are ignored. +This should not matter in practice, as long as everything else including +module.modulemap files is the same the cached result should work. Still, +you must set <> to *modules* to allow +caching. + +You must use both <<_the_direct_mode,direct mode>> and +<<_the_depend_mode,depend mode>>. When using +<<_the_preprocessor_mode,the preprocessor mode>> Clang does not provide +enough information to allow hashing of module.modulemap files. + + Sharing a cache --------------- @@ -1385,6 +1410,8 @@ problems and what may be done to increase the hit rate: HEADERS>> for how to remedy this. * If ``can't use precompiled header'' has been incremented, see <<_precompiled_headers,PRECOMPILED HEADERS>>. +* If ``can't use modules'' has been incremented, see + <<_c_modules,C++ MODULES>>. Corrupt object files diff --git a/src/Config.cpp b/src/Config.cpp index 1ca7f9e9a..7908283c0 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -266,6 +266,8 @@ parse_sloppiness(const std::string& value) result |= SLOPPY_CLANG_INDEX_STORE; } else if (token == "locale") { result |= SLOPPY_LOCALE; + } else if (token == "modules") { + result |= SLOPPY_MODULES; } else if (token != "") { throw Error(fmt::format("unknown sloppiness: \"{}\"", token)); } @@ -308,6 +310,9 @@ format_sloppiness(uint32_t sloppiness) if (sloppiness & SLOPPY_LOCALE) { result += "locale, "; } + if (sloppiness & SLOPPY_MODULES) { + result += "modules, "; + } if (!result.empty()) { // Strip last ", ". result.resize(result.size() - 2); diff --git a/src/ccache.cpp b/src/ccache.cpp index f46e4239b..25223f17f 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -2571,6 +2571,31 @@ cc_process_args(struct args* args, found_fpch_preprocess = true; } + // Modules are handled on demand as necessary in the background, + // so there is no need to cache them, they can be in practice ignored. + // All that is needed is to correctly depend also on module.modulemap files, + // and those are included only in depend mode (preprocessed output does not + // list them). Still, not including the modules themselves in the hash + // could possibly result in an object file that would be different + // from the actual compilation (even though it should be compatible), + // so require a sloppiness flag. + if (str_eq(argv[i], "-fmodules")) { + if (!g_config.depend_mode() || !g_config.direct_mode()) { + cc_log("Compiler option %s is unsupported without direct depend mode", + argv[i]); + stats_update(STATS_CANTUSEMODULES); + result = false; + goto out; + } else if (!(g_config.sloppiness() & SLOPPY_MODULES)) { + cc_log( + "You have to specify \"modules\" sloppiness when using" + " -fmodules to get hits"); + stats_update(STATS_CANTUSEMODULES); + result = false; + goto out; + } + } + // We must have -c. if (str_eq(argv[i], "-c")) { found_c_opt = true; diff --git a/src/ccache.hpp b/src/ccache.hpp index 302296b70..8243aebc7 100644 --- a/src/ccache.hpp +++ b/src/ccache.hpp @@ -77,6 +77,7 @@ enum stats { STATS_NUMCLEANUPS = 29, STATS_UNSUPPORTED_DIRECTIVE = 30, STATS_ZEROTIMESTAMP = 31, + STATS_CANTUSEMODULES = 32, STATS_END }; @@ -109,6 +110,8 @@ extern enum guessed_compiler guessed_compiler; #define SLOPPY_CLANG_INDEX_STORE (1U << 8) // Ignore locale settings. #define SLOPPY_LOCALE (1U << 9) +// Allow caching even if -fmodules is used. +#define SLOPPY_MODULES (1U << 10) #define str_eq(s1, s2) (strcmp((s1), (s2)) == 0) #define str_startswith(s, prefix) \ diff --git a/src/compopt.cpp b/src/compopt.cpp index ecc24da34..61a8888cf 100644 --- a/src/compopt.cpp +++ b/src/compopt.cpp @@ -92,7 +92,6 @@ static const struct compopt compopts[] = { {"-bind_at_load", AFFECTS_COMP}, {"-bundle", AFFECTS_COMP}, {"-ccbin", AFFECTS_CPP | TAKES_ARG}, // nvcc - {"-fmodules", TOO_HARD}, {"-fno-working-directory", AFFECTS_CPP}, {"-fplugin=libcc1plugin", TOO_HARD}, // interaction with GDB {"-frepo", TOO_HARD}, diff --git a/src/stats.cpp b/src/stats.cpp index 820dbcc82..07ae92803 100644 --- a/src/stats.cpp +++ b/src/stats.cpp @@ -108,6 +108,7 @@ static struct "can't use precompiled header", NULL, 0}, + {STATS_CANTUSEMODULES, "could_not_use_modules", "can't use modules", NULL, 0}, {STATS_COMPILER, "could_not_find_compiler", "couldn't find the compiler", diff --git a/test/run b/test/run index a85d300b3..6cd2f634e 100755 --- a/test/run +++ b/test/run @@ -214,6 +214,18 @@ expect_file_content() { fi } +expect_file_contains() { + local file="$1" + local string="$2" + + if [ ! -f "$file" ]; then + test_failed "$file not found" + fi + if ! grep -q "$string" "$file"; then + test_failed "File $file does not contain: $string. Actual content: $(cat $file)" + fi +} + expect_file_count() { local expected=$1 local pattern=$2 @@ -421,6 +433,7 @@ readonly readonly_direct cleanup pch +modules upgrade input_charset nvcc diff --git a/test/suites/modules.bash b/test/suites/modules.bash new file mode 100644 index 000000000..dcd02abd6 --- /dev/null +++ b/test/suites/modules.bash @@ -0,0 +1,81 @@ +SUITE_modules_PROBE() { + if ! $COMPILER_TYPE_CLANG; then + echo "-fmodules/-fcxx-modules not supported by compiler" + return + fi + } + +SUITE_modules_SETUP() { + unset CCACHE_NODIRECT + export CCACHE_DEPEND=1 + + cat <test1.h +#include +EOF + backdate test1.h + +cat <module.modulemap +module "Test1" { + header "test1.h" + export * +} +EOF + backdate module.modulemap + + cat <test1.cpp +#import "test1.h" +int main() { return 0; } +EOF +} + +SUITE_modules() { + # ------------------------------------------------------------------------- + TEST "preprocessor output" + $COMPILER -fmodules -fcxx-modules test1.cpp -E > test1.preprocessed.cpp + expect_file_contains "test1.preprocessed.cpp" "#pragma clang module import Test1" + + # ------------------------------------------------------------------------- + TEST "fall back to real compiler, no sloppiness" + + $CCACHE_COMPILE -fmodules -fcxx-modules -c test1.cpp -MD + expect_stat "can't use modules" 1 + + $CCACHE_COMPILE -fmodules -fcxx-modules -c test1.cpp -MD + expect_stat "can't use modules" 2 + + # ------------------------------------------------------------------------- + TEST "fall back to real compiler, no depend mode" + + unset CCACHE_DEPEND + + CCACHE_SLOPPINESS="$DEFAULT_SLOPPINESS modules" $CCACHE_COMPILE -fmodules -fcxx-modules -c test1.cpp -MD + expect_stat "can't use modules" 1 + + CCACHE_SLOPPINESS="$DEFAULT_SLOPPINESS modules" $CCACHE_COMPILE -fmodules -fcxx-modules -c test1.cpp -MD + expect_stat "can't use modules" 2 + + # ------------------------------------------------------------------------- + TEST "cache hit" + + CCACHE_SLOPPINESS="$DEFAULT_SLOPPINESS modules" $CCACHE_COMPILE -fmodules -fcxx-modules -c test1.cpp -MD + expect_stat 'cache hit (direct)' 0 + expect_stat 'cache miss' 1 + + CCACHE_SLOPPINESS="$DEFAULT_SLOPPINESS modules" $CCACHE_COMPILE -fmodules -fcxx-modules -c test1.cpp -MD + expect_stat 'cache hit (direct)' 1 + expect_stat 'cache miss' 1 + + # ------------------------------------------------------------------------- + TEST "cache miss" + + CCACHE_SLOPPINESS="$DEFAULT_SLOPPINESS modules" $CCACHE_COMPILE -MD -fmodules -fcxx-modules -c test1.cpp -MD + expect_stat 'cache miss' 1 + + cat <test1.h +#include +void f(); +EOF + + CCACHE_SLOPPINESS="$DEFAULT_SLOPPINESS modules" $CCACHE_COMPILE -MD -fmodules -fcxx-modules -c test1.cpp -MD + expect_stat 'cache miss' 2 +}