From: Orgad Shaneh Date: Wed, 5 Oct 2022 17:34:31 +0000 (+0300) Subject: feat: Support auto depend mode for MSVC without /showIncludes (#1158) X-Git-Tag: v4.7~28 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8b65880b5ad817156b58c58b5133aafc99b0a264;p=thirdparty%2Fccache.git feat: Support auto depend mode for MSVC without /showIncludes (#1158) Co-authored-by: Luboš Luňák --- diff --git a/src/ArgsInfo.hpp b/src/ArgsInfo.hpp index 92d089b52..ce96ceed2 100644 --- a/src/ArgsInfo.hpp +++ b/src/ArgsInfo.hpp @@ -75,6 +75,9 @@ struct ArgsInfo // Is the compiler being asked to output dependencies? bool generating_dependencies = false; + // Is the compiler being asked to output includes (MSVC -showIncludes)? + bool generating_includes = false; + // The dependency target in the dependency file (the object file unless // overridden via e.g. -MT or -MQ). std::optional dependency_target; diff --git a/src/Config.cpp b/src/Config.cpp index 520557e10..3cec222f6 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -83,6 +83,7 @@ enum class ConfigItem { log_file, max_files, max_size, + msvc_dep_prefix, namespace_, path, pch_external_checksum, @@ -138,6 +139,7 @@ const std::unordered_map k_config_key_table = {"log_file", {ConfigItem::log_file}}, {"max_files", {ConfigItem::max_files}}, {"max_size", {ConfigItem::max_size}}, + {"msvc_dep_prefix", {ConfigItem::msvc_dep_prefix}}, {"namespace", {ConfigItem::namespace_}}, {"path", {ConfigItem::path}}, {"pch_external_checksum", {ConfigItem::pch_external_checksum}}, @@ -187,6 +189,7 @@ const std::unordered_map k_env_variable_table = { {"LOGFILE", "log_file"}, {"MAXFILES", "max_files"}, {"MAXSIZE", "max_size"}, + {"MSVC_DEP_PREFIX", "msvc_dep_prefix"}, {"NAMESPACE", "namespace"}, {"PATH", "path"}, {"PCH_EXTSUM", "pch_external_checksum"}, @@ -759,6 +762,9 @@ Config::get_string_value(const std::string& key) const case ConfigItem::max_size: return format_cache_size(m_max_size); + case ConfigItem::msvc_dep_prefix: + return m_msvc_dep_prefix; + case ConfigItem::namespace_: return m_namespace; @@ -1007,6 +1013,10 @@ Config::set_item(const std::string& key, m_max_size = Util::parse_size(value); break; + case ConfigItem::msvc_dep_prefix: + m_msvc_dep_prefix = Util::expand_environment_variables(value); + break; + case ConfigItem::namespace_: m_namespace = Util::expand_environment_variables(value); break; diff --git a/src/Config.hpp b/src/Config.hpp index 7a01b35e4..48e1a8017 100644 --- a/src/Config.hpp +++ b/src/Config.hpp @@ -77,6 +77,7 @@ public: const std::string& log_file() const; uint64_t max_files() const; uint64_t max_size() const; + const std::string& msvc_dep_prefix() const; const std::string& path() const; bool pch_external_checksum() const; const std::string& prefix_command() const; @@ -185,6 +186,7 @@ private: std::string m_log_file; uint64_t m_max_files = 0; uint64_t m_max_size = 5ULL * 1000 * 1000 * 1000; + std::string m_msvc_dep_prefix; std::string m_path; bool m_pch_external_checksum = false; std::string m_prefix_command; @@ -385,6 +387,12 @@ Config::max_size() const return m_max_size; } +inline const std::string& +Config::msvc_dep_prefix() const +{ + return m_msvc_dep_prefix; +} + inline const std::string& Config::path() const { diff --git a/src/Context.hpp b/src/Context.hpp index b76248115..1b29342a7 100644 --- a/src/Context.hpp +++ b/src/Context.hpp @@ -118,6 +118,8 @@ public: std::unique_ptr mini_trace; #endif + bool auto_depend_mode = false; + // Register a temporary file to remove at program exit. void register_pending_tmp_file(const std::string& path); diff --git a/src/argprocessing.cpp b/src/argprocessing.cpp index 412787c51..2b0603f95 100644 --- a/src/argprocessing.cpp +++ b/src/argprocessing.cpp @@ -684,6 +684,12 @@ process_option_arg(const Context& ctx, return Statistic::none; } + if (args[i] == "-showIncludes") { + args_info.generating_includes = true; + state.dep_args.push_back(args[i]); + return Statistic::none; + } + if (args[i] == "-fprofile-arcs") { args_info.profile_arcs = true; state.common_args.push_back(args[i]); @@ -1493,6 +1499,13 @@ process_args(Context& ctx) } } + if (ctx.config.depend_mode() && !args_info.generating_includes + && ctx.config.compiler_type() == CompilerType::msvc) { + ctx.auto_depend_mode = true; + args_info.generating_includes = true; + args_info.depend_extra_args.push_back("-showIncludes"); + } + return { preprocessor_args, extra_args_to_hash, diff --git a/src/ccache.cpp b/src/ccache.cpp index 7884a009b..494200544 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -692,6 +693,36 @@ struct DoExecuteResult std::string stderr_data; }; +// Extract the used includes from -showIncludes output in stdout. Note that we +// cannot distinguish system headers from other includes here. +static std::optional +result_key_from_includes(Context& ctx, + Hash& hash, + const std::string& stdout_data) +{ + for (std::string_view token : core::ShowIncludesParser::tokenize( + stdout_data, ctx.config.msvc_dep_prefix())) { + const std::string path = Util::make_relative_path(ctx, token); + remember_include_file(ctx, path, hash, false, &hash); + } + + // Explicitly check the .pch file as it is not mentioned in the + // includes output. + if (!ctx.args_info.included_pch_file.empty()) { + std::string pch_path = + Util::make_relative_path(ctx, ctx.args_info.included_pch_file); + hash.hash(pch_path); + remember_include_file(ctx, pch_path, hash, false, nullptr); + } + + const bool debug_included = getenv("CCACHE_DEBUG_INCLUDED"); + if (debug_included) { + print_included_files(ctx, stdout); + } + + return hash.digest(); +} + // Execute the compiler/preprocessor, with logic to retry without requesting // colored diagnostics messages if that fails. static nonstd::expected @@ -1040,7 +1071,10 @@ to_cache(Context& ctx, // We can output stderr immediately instead of rerunning the compiler. Util::send_to_fd(ctx, result->stderr_data, STDERR_FILENO); - Util::send_to_fd(ctx, result->stdout_data, STDOUT_FILENO); + Util::send_to_fd(ctx, + core::ShowIncludesParser::strip_includes( + ctx, std::move(result->stdout_data)), + STDOUT_FILENO); auto failure = Failure(Statistic::compile_failed); failure.set_exit_code(result->exit_status); @@ -1049,7 +1083,14 @@ to_cache(Context& ctx, if (ctx.config.depend_mode()) { ASSERT(depend_mode_hash); - result_key = result_key_from_depfile(ctx, *depend_mode_hash); + if (ctx.args_info.generating_dependencies) { + result_key = result_key_from_depfile(ctx, *depend_mode_hash); + } else if (ctx.args_info.generating_includes) { + result_key = + result_key_from_includes(ctx, *depend_mode_hash, result->stdout_data); + } else { + ASSERT(false); + } if (!result_key) { return nonstd::make_unexpected(Statistic::internal_error); } @@ -1090,7 +1131,10 @@ to_cache(Context& ctx, // Everything OK. Util::send_to_fd(ctx, result->stderr_data, STDERR_FILENO); // Send stdout after stderr, it makes the output clearer with MSVC. - Util::send_to_fd(ctx, result->stdout_data, STDOUT_FILENO); + Util::send_to_fd(ctx, + core::ShowIncludesParser::strip_includes( + ctx, std::move(result->stdout_data)), + STDOUT_FILENO); return *result_key; } @@ -2293,12 +2337,14 @@ do_cache_compilation(Context& ctx, const char* const* argv) ctx.config.set_run_second_cpp(true); } - if (ctx.config.depend_mode() - && (!ctx.args_info.generating_dependencies - || ctx.args_info.output_dep == "/dev/null" - || !ctx.config.run_second_cpp())) { - LOG_RAW("Disabling depend mode"); - ctx.config.set_depend_mode(false); + if (ctx.config.depend_mode()) { + const bool deps = ctx.args_info.generating_dependencies + && ctx.args_info.output_dep != "/dev/null"; + const bool includes = ctx.args_info.generating_includes; + if (!ctx.config.run_second_cpp() || (!deps && !includes)) { + LOG_RAW("Disabling depend mode"); + ctx.config.set_depend_mode(false); + } } if (ctx.storage.has_remote_storage()) { diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index d2431be34..6932336f3 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -6,6 +6,7 @@ set( ResultExtractor.cpp ResultInspector.cpp ResultRetriever.cpp + ShowIncludesParser.cpp Statistics.cpp StatisticsCounters.cpp StatsLog.cpp diff --git a/src/core/ResultRetriever.cpp b/src/core/ResultRetriever.cpp index bdefc19fe..95925836b 100644 --- a/src/core/ResultRetriever.cpp +++ b/src/core/ResultRetriever.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -61,7 +62,10 @@ ResultRetriever::on_embedded_file(uint8_t file_number, data.size()); if (file_type == FileType::stdout_output) { - Util::send_to_fd(m_ctx, util::to_string_view(data), STDOUT_FILENO); + std::string str = util::to_string(util::to_string_view(data)); + Util::send_to_fd(m_ctx, + ShowIncludesParser::strip_includes(m_ctx, std::move(str)), + STDOUT_FILENO); } else if (file_type == FileType::stderr_output) { Util::send_to_fd(m_ctx, util::to_string_view(data), STDERR_FILENO); } else { diff --git a/src/core/ShowIncludesParser.cpp b/src/core/ShowIncludesParser.cpp new file mode 100644 index 000000000..318e82371 --- /dev/null +++ b/src/core/ShowIncludesParser.cpp @@ -0,0 +1,78 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ShowIncludesParser.hpp" + +#include +#include +#include + +namespace core::ShowIncludesParser { + +std::vector +tokenize(std::string_view file_content, std::string_view prefix) +{ + // -showIncludes output is written to stdout together with other messages. + // Every line of it is ' ', prefix is 'Note: including + // file:' in English but can be localized. + + if (prefix.empty()) { + prefix = "Note: including file:"; + } + + std::vector result; + // This will split at each \r or \n, but that simply means there will be empty + // "lines". + for (std::string_view line : Util::split_into_views(file_content, "\r\n")) { + if (util::starts_with(line, prefix)) { + size_t pos = prefix.size(); + while (pos < line.size() && isspace(line[pos])) { + ++pos; + } + std::string_view include = line.substr(pos); + if (!include.empty()) { + result.push_back(include); + } + } + } + return result; +} + +std::string +strip_includes(const Context& ctx, std::string&& stdout_data) +{ + using util::Tokenizer; + using Mode = Tokenizer::Mode; + using IncludeDelimiter = Tokenizer::IncludeDelimiter; + + if (stdout_data.empty() || !ctx.auto_depend_mode + || ctx.config.compiler_type() != CompilerType::msvc) { + return std::move(stdout_data); + } + + std::string new_stdout_text; + for (const auto line : Tokenizer( + stdout_data, "\n", Mode::include_empty, IncludeDelimiter::yes)) { + if (!util::starts_with(line, "Note: including file:")) { + new_stdout_text.append(line.data(), line.length()); + } + } + return new_stdout_text; +} + +} // namespace core::ShowIncludesParser diff --git a/src/core/ShowIncludesParser.hpp b/src/core/ShowIncludesParser.hpp new file mode 100644 index 000000000..ebc2284b3 --- /dev/null +++ b/src/core/ShowIncludesParser.hpp @@ -0,0 +1,33 @@ +// Copyright (C) 2022 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#pragma once + +#include +#include + +class Context; + +namespace core::ShowIncludesParser { + +std::vector tokenize(std::string_view file_content, + std::string_view prefix); + +std::string strip_includes(const Context& ctx, std::string&& stdout_data); + +} // namespace core::ShowIncludesParser diff --git a/unittest/test_Config.cpp b/unittest/test_Config.cpp index 0226101f0..f9b772aa3 100644 --- a/unittest/test_Config.cpp +++ b/unittest/test_Config.cpp @@ -64,6 +64,7 @@ TEST_CASE("Config: default values") CHECK(config.log_file().empty()); CHECK(config.max_files() == 0); CHECK(config.max_size() == static_cast(5) * 1000 * 1000 * 1000); + CHECK(config.msvc_dep_prefix().empty()); CHECK(config.path().empty()); CHECK_FALSE(config.pch_external_checksum()); CHECK(config.prefix_command().empty()); @@ -122,6 +123,7 @@ TEST_CASE("Config::update_from_file") "log_file = $USER${USER} \n" "max_files = 17\n" "max_size = 123M\n" + "msvc_dep_prefix = Note: including file:\n" "path = $USER.x\n" "pch_external_checksum = true\n" "prefix_command = x$USER\n" @@ -162,6 +164,7 @@ TEST_CASE("Config::update_from_file") CHECK(config.log_file() == FMT("{0}{0}", user)); CHECK(config.max_files() == 17); CHECK(config.max_size() == 123 * 1000 * 1000); + CHECK(config.msvc_dep_prefix() == "Note: including file:"); CHECK(config.path() == FMT("{}.x", user)); CHECK(config.pch_external_checksum()); CHECK(config.prefix_command() == FMT("x{}", user)); @@ -406,6 +409,7 @@ TEST_CASE("Config::visit_items") "log_file = lf\n" "max_files = 4711\n" "max_size = 98.7M\n" + "msvc_dep_prefix = mdp\n" "namespace = ns\n" "path = p\n" "pch_external_checksum = true\n" @@ -467,6 +471,7 @@ TEST_CASE("Config::visit_items") "(test.conf) log_file = lf", "(test.conf) max_files = 4711", "(test.conf) max_size = 98.7M", + "(test.conf) msvc_dep_prefix = mdp", "(test.conf) namespace = ns", "(test.conf) path = p", "(test.conf) pch_external_checksum = true",