feat: Support caching stdout for all compiler types

diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc
index 6ac6127a6296385f179629bc5dbf48bb45d32451..307707ffca983f505d3350e2d0c3a1c7c8454c74 100644 (file)
--- a/doc/MANUAL.adoc
+++ b/doc/MANUAL.adoc
@@ -1314,12 +1314,6 @@ compilation.
 The compiler's output file (typically an object file) was missing after
 compilation.
 
-| Compiler produced stdout |
-The compiler wrote data to standard output. This is something that compilers
-normally never do, so ccache is not designed to store such output in the cache.
-For compilers that may normally write data to standard output, as such MSVC,
-the output is cached properly and it is not considered to be an error.
-
 | Could not find the compiler |
 The compiler to execute could not be found.
 

diff --git a/src/ccache.cpp b/src/ccache.cpp
index 980209fda436a20bf2ba3b546c505d4eb8042920..de262781f3c2699ea530f1a17368028c400e7900 100644 (file)
--- a/src/ccache.cpp
+++ b/src/ccache.cpp
@@ -789,24 +789,18 @@ static bool
 write_result(Context& ctx,
              const std::string& result_path,
              const Stat& obj_stat,
-             const std::string& stdout_path,
-             const std::string& stderr_path)
+             const std::string& stdout_data,
+             const std::string& stderr_data)
 {
   Result::Writer result_writer(ctx, result_path);
 
-  const auto stdout_stat = Stat::stat(stdout_path, Stat::OnError::log);
-  const auto stderr_stat = Stat::stat(stderr_path, Stat::OnError::log);
-  if (!stdout_stat || !stderr_stat) {
-    return false;
-  }
-
-  if (stderr_stat.size() > 0) {
-    result_writer.write_file(Result::FileType::stderr_output, stderr_path);
+  if (!stderr_data.empty()) {
+    result_writer.write_data(Result::FileType::stderr_output, stderr_data);
   }
   // Write stdout only after stderr (better with MSVC), as ResultRetriever
   // will later print process them in the order they are read.
-  if (stdout_stat.size() > 0) {
-    result_writer.write_file(Result::FileType::stdout_output, stdout_path);
+  if (!stdout_data.empty()) {
+    result_writer.write_data(Result::FileType::stdout_output, stdout_data);
   }
   if (obj_stat) {
     result_writer.write_file(Result::FileType::object,
@@ -855,6 +849,32 @@ write_result(Context& ctx,
   return true;
 }
 
+static std::string
+rewrite_stdout_from_compiler(const Context& ctx, std::string&& stdout_data)
+{
+  // distcc-pump outputs lines like this:
+  //
+  //   __________Using # distcc servers in pump mode
+  //
+  // We don't want to cache those.
+  if (!stdout_data.empty()
+      && ctx.config.compiler_type() == CompilerType::pump) {
+    std::string new_stdout_text;
+    for (const auto line : util::Tokenizer(
+           stdout_data, "\n", util::Tokenizer::Mode::include_empty)) {
+      if (util::starts_with(line, "__________")) {
+        Util::send_to_fd(ctx, std::string(line), STDOUT_FILENO);
+      } else {
+        new_stdout_text.append(line.data(), line.length());
+        new_stdout_text.append("\n");
+      }
+    }
+    return new_stdout_text;
+  } else {
+    return std::move(stdout_data);
+  }
+}
+
 // Run the real compiler and put the result in cache. Returns the result key.
 static nonstd::expected<Digest, Failure>
 to_cache(Context& ctx,
@@ -941,20 +961,6 @@ to_cache(Context& ctx,
     return nonstd::make_unexpected(status.error());
   }
 
-  auto st = Stat::stat(tmp_stdout_path, Stat::OnError::log);
-  if (!st) {
-    // The stdout file was removed - cleanup in progress? Better bail out.
-    return nonstd::make_unexpected(Statistic::missing_cache_file);
-  }
-
-  // distcc-pump outputs lines like this:
-  // __________Using # distcc servers in pump mode
-  if (st.size() != 0 && ctx.config.compiler_type() != CompilerType::pump
-      && ctx.config.compiler_type() != CompilerType::cl) {
-    LOG_RAW("Compiler produced stdout");
-    return nonstd::make_unexpected(Statistic::compiler_produced_stdout);
-  }
-
   // Merge stderr from the preprocessor (if any) and stderr from
   // the real compiler into tmp_stderr.
   if (!ctx.cpp_stderr.empty()) {
@@ -963,12 +969,25 @@ to_cache(Context& ctx,
     Util::write_file(tmp_stderr_path, combined_stderr);
   }
 
+  std::string stdout_data;
+  std::string stderr_data;
+  try {
+    stdout_data = Util::read_file(tmp_stdout_path);
+    stderr_data = Util::read_file(tmp_stderr_path);
+  } catch (core::Error&) {
+    // The stdout or stderr file was removed - cleanup in progress? Better bail
+    // out.
+    return nonstd::make_unexpected(Statistic::missing_cache_file);
+  }
+
+  stdout_data = rewrite_stdout_from_compiler(ctx, std::move(stdout_data));
+
   if (status != 0) {
     LOG("Compiler gave exit status {}", *status);
 
     // We can output stderr immediately instead of rerunning the compiler.
-    Util::send_to_fd(ctx, Util::read_file(tmp_stderr_path), STDERR_FILENO);
-    Util::send_to_fd(ctx, Util::read_file(tmp_stdout_path), STDOUT_FILENO);
+    Util::send_to_fd(ctx, stderr_data, STDERR_FILENO);
+    Util::send_to_fd(ctx, stdout_data, STDOUT_FILENO);
 
     auto failure = Failure(Statistic::compile_failed);
     failure.set_exit_code(*status);
@@ -1008,8 +1027,7 @@ to_cache(Context& ctx,
   MTR_BEGIN("result", "result_put");
   const bool added = ctx.storage.put(
     *result_key, core::CacheEntryType::result, [&](const auto& path) {
-      return write_result(
-        ctx, path, obj_stat, tmp_stdout_path, tmp_stderr_path);
+      return write_result(ctx, path, obj_stat, stdout_data, stderr_data);
     });
   MTR_END("result", "result_put");
   if (!added) {
@@ -1017,9 +1035,9 @@ to_cache(Context& ctx,
   }
 
   // Everything OK.
-  Util::send_to_fd(ctx, Util::read_file(tmp_stderr_path), STDERR_FILENO);
+  Util::send_to_fd(ctx, stderr_data, STDERR_FILENO);
   // Send stdout after stderr, it makes the output clearer with MSVC.
-  Util::send_to_fd(ctx, Util::read_file(tmp_stdout_path), STDOUT_FILENO);
+  Util::send_to_fd(ctx, stdout_data, STDOUT_FILENO);
 
   return *result_key;
 }

diff --git a/test/suites/base.bash b/test/suites/base.bash
index fb9d0ee93d93a4ba5aa9c9c9252b4a0aa01eb09c..d082bd06270dab0ffd597520a3f23a35b91b338f 100644 (file)
--- a/test/suites/base.bash
+++ b/test/suites/base.bash
@@ -174,12 +174,6 @@ base_tests() {
     $CCACHE_COMPILE -M foo -c test1.c >/dev/null 2>&1
     expect_stat unsupported_compiler_option 1
 
-    # -------------------------------------------------------------------------
-    TEST "Compiler produced stdout"
-
-    $CCACHE echo foo -c test1.c >/dev/null
-    expect_stat compiler_produced_stdout 1
-
     # -------------------------------------------------------------------------
     TEST "Output to directory"
 
@@ -1213,24 +1207,34 @@ EOF
     expect_equal_content reference.d test.d
 
     # -------------------------------------------------------------------------
-    TEST "No stdout"
+    TEST "Caching stdout and stderr"
 
-    cat <<EOF >test.c
-// just something random
-int test() { return 1; }
+    cat >compiler.sh <<EOF
+#!/bin/sh
+if [ \$1 = -E ] || ! echo "\$*" | grep -q '\.i\$'; then
+    printf "cpp_err|" >&2
+fi
+if [ \$1 != -E ]; then
+    printf "cc_err|" >&2
+    printf "cc_out|"
+fi
+CCACHE_DISABLE=1 # If $COMPILER happens to be a ccache symlink...
+export CCACHE_DISABLE
+exec $COMPILER "\$@"
 EOF
+    chmod +x compiler.sh
 
-    stderr=$($CCACHE_COMPILE -c test.c >stdout 2>stderr)
+    $CCACHE ./compiler.sh -c test1.c >stdout 2>stderr
     expect_stat preprocessed_cache_hit 0
     expect_stat cache_miss 1
-    expect_content stdout ""
-    expect_content stderr ""
+    expect_content stdout "cc_out|"
+    expect_content stderr "cpp_err|cc_err|"
 
-    stderr=$($CCACHE_COMPILE -c test.c >stdout 2>stderr)
+    $CCACHE ./compiler.sh -c test1.c >stdout 2>stderr
     expect_stat preprocessed_cache_hit 1
     expect_stat cache_miss 1
-    expect_content stdout ""
-    expect_content stderr ""
+    expect_content stdout "cc_out|"
+    expect_content stderr "cpp_err|cc_err|"
 
     # -------------------------------------------------------------------------
     TEST "--zero-stats"