Add functionality to debug the hashing

diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc
index 3bdfa9f687b997f10f0f4799d6622d69336f8d2a..53e5c11a31223cdf9e2bed9dfdd80f1f965b8e2d 100644 (file)
--- a/doc/MANUAL.adoc
+++ b/doc/MANUAL.adoc
@@ -352,8 +352,10 @@ WRAPPERS>>.
 
 *debug* (*CCACHE_DEBUG* or *CCACHE_NODEBUG*, see <<_boolean_values,Boolean values>> above)::
 
-    If true, the debug mode will be used. The default is false. In debug mode,
-    the hash input and the log file will be stored next to the object file.
+    If true, the debug mode will be used. The default is false. In the debug mode,
+    the hash input and the debug log file will be stored next to the output files.
+    The input to the hashes is shown both in binary (hashc/hashd/hashp) as well as
+    in a text (input) format for easier reading. See <<_cache_debugging_,debugging>>.
 
 *direct_mode* (*CCACHE_DIRECT* or *CCACHE_NODIRECT*, see <<_boolean_values,Boolean values>> above)::
 
@@ -754,6 +756,26 @@ A source language e.g. specified with *-x* was unsupported by ccache.
 |==============================================================================
 
 
+Cache debugging
+---------------
+
+When investigating why you are not getting hits, and what information
+that ccache is actually hashing, you can enable the "debugging" mode.
+
+In this mode, ccache will add some additional output files next to the
+object files with results from the hashing and a copy of the debug log.
+
+There are several binary hash files created, for different partial input:
+
+* Common (*'c'*)
+* Direct mode (*'d'*)
+* Preprocessor mode (*'p'*)
+
+The hash is the checksum of the common + direct or common + preprocessor.
+
+There is also a human readable text file, for easier reading and diffing.
+
+
 How ccache works
 ----------------
 

diff --git a/src/ccache.c b/src/ccache.c
index 253a4a8aec0f5fe10ca12d4aba5e48b59d70639e..eaa9b308fbee86fa155a711089fd80162cb7454d 100644 (file)
--- a/src/ccache.c
+++ b/src/ccache.c
@@ -476,6 +476,25 @@ clean_up_internal_tempdir(void)
        closedir(dir);
 }
 
+static void
+debug_start(const char *path)
+{
+       char *hash_bin = format("%s%s", path, ".ccache-hashX");
+       char *hash_txt = format("%s%s", path, ".ccache-input");
+       hash_debug(hash_bin, hash_txt);
+       free(hash_bin);
+       free(hash_txt);
+}
+
+static void
+debug_end(bool hit)
+{
+       (void) hit;
+       char *path = format("%s%s", output_obj, ".ccache-log");
+       cc_copylog(path);
+       free(path);
+}
+
 static enum guessed_compiler
 guess_compiler(const char *path)
 {
@@ -894,7 +913,7 @@ process_preprocessed_file(struct mdfour *hash, const char *path, bool pump)
                                free(cwd);
                        }
                        if (should_hash_inc_path) {
-                               hash_string(hash, inc_path);
+                               hash_buffer(hash, inc_path, strlen(inc_path));
                        }
 
                        remember_include_file(inc_path, hash, system);
@@ -2051,6 +2070,10 @@ from_cache(enum fromcache_call_mode mode, bool put_object_in_manifest)
                break;
        }
 
+       if (conf->debug) {
+               debug_end(true);
+       }
+
        // And exit with the right status code.
        x_exit(0);
 }
@@ -3360,12 +3383,18 @@ ccache(int argc, char *argv[])
 
        cc_log("Object file: %s", output_obj);
 
+       if (conf->debug) {
+               debug_start(output_obj);
+       }
+
        struct mdfour common_hash;
        hash_start(&common_hash);
+       mdfour_identify(&common_hash, 'c');
        calculate_common_hash(preprocessor_args, &common_hash);
 
        // Try to find the hash using the manifest.
        struct mdfour direct_hash = common_hash;
+       mdfour_identify(&direct_hash, 'd');
        bool put_object_in_manifest = false;
        struct file_hash *object_hash = NULL;
        struct file_hash *object_hash_from_manifest = NULL;
@@ -3396,6 +3425,7 @@ ccache(int argc, char *argv[])
 
        // Find the hash using the preprocessed output. Also updates included_files.
        struct mdfour cpp_hash = common_hash;
+       mdfour_identify(&cpp_hash, 'p');
        object_hash = calculate_object_hash(preprocessor_args, &cpp_hash, 0);
        if (!object_hash) {
                fatal("internal error: object hash from cpp returned NULL");
@@ -3437,6 +3467,10 @@ ccache(int argc, char *argv[])
        // Run real compiler, sending output to cache.
        to_cache(compiler_args);
 
+       if (conf->debug) {
+               debug_end(false);
+       }
+
        x_exit(0);
 }
 

diff --git a/src/ccache.h b/src/ccache.h
index 6b6f25da51e2dc8b5177553968dd9e79eb4f6e8c..2c320d83cd504b73ad3857adb3e3bc6327632765 100644 (file)
--- a/src/ccache.h
+++ b/src/ccache.h
@@ -118,6 +118,7 @@ bool args_equal(struct args *args1, struct args *args2);
 // ----------------------------------------------------------------------------
 // hash.c
 
+void hash_debug(const char *bin, const char *txt);
 void hash_start(struct mdfour *md);
 void hash_buffer(struct mdfour *md, const void *s, size_t len);
 char *hash_result(struct mdfour *md);

diff --git a/src/hash.c b/src/hash.c
index 3b462ca9eaf7ad22c0af43a8dd577691a945e481..5a5bd27eb7da877fd74dc82b2182feb2913b74b9 100644 (file)
--- a/src/hash.c
+++ b/src/hash.c
@@ -18,6 +18,60 @@
 #include "ccache.h"
 
 #define HASH_DELIMITER "\000cCaChE"
+#define HASH_DEBUG_DELIMITER "### "
+
+// binary input, for hashing
+char *debug_hash_bin;
+
+// text input, for debugging
+char *debug_hash_txt;
+
+void hash_debug(const char *bin, const char *txt)
+{
+       static char *hash_types = "cdp"; // common, direct, cpp
+       if (bin) {
+               debug_hash_bin = x_strdup(bin);
+               assert(debug_hash_bin[strlen(debug_hash_bin)-1] == 'X');
+               for (char *p = hash_types; *p != '\0'; p++) {
+                       debug_hash_bin[strlen(debug_hash_bin)-1] = *p;
+                       x_try_unlink(debug_hash_bin);
+               }
+       }
+       if (txt) {
+               debug_hash_txt = x_strdup(txt);
+               x_try_unlink(debug_hash_txt);
+       }
+}
+
+static void
+hash_binary_buffer(struct mdfour *md, const void *s, size_t len)
+{
+       mdfour_update(md, (unsigned char *)s, len);
+       if (!md->identifier || len == 0) {
+               return;
+       }
+       if (debug_hash_bin) {
+               // log to different files, for the different hash types
+               debug_hash_bin[strlen(debug_hash_bin)-1] = md->identifier;
+
+               FILE *f = fopen(debug_hash_bin, "a");
+               fwrite(s, 1, len, f);
+               fclose(f);
+       }
+}
+
+static void
+hash_debug_buffer(struct mdfour *md, const void *s, size_t len)
+{
+       if (!md->identifier || len == 0) {
+               return;
+       }
+       if (debug_hash_txt) {
+               FILE *f = fopen(debug_hash_txt, "a");
+               fwrite(s, 1, len, f);
+               fclose(f);
+       }
+}
 
 void
 hash_start(struct mdfour *md)
@@ -28,7 +82,8 @@ hash_start(struct mdfour *md)
 void
 hash_buffer(struct mdfour *md, const void *s, size_t len)
 {
-       mdfour_update(md, (unsigned char *)s, len);
+       hash_binary_buffer(md, s, len);
+       hash_debug_buffer(md, s, len);
 }
 
 // Return the hash result as a hex string. Caller frees.
@@ -45,7 +100,7 @@ hash_result(struct mdfour *md)
 void
 hash_result_as_bytes(struct mdfour *md, unsigned char *out)
 {
-       hash_buffer(md, NULL, 0);
+       mdfour_update(md, NULL, 0);
        mdfour_result(md, out);
 }
 
@@ -70,8 +125,11 @@ hash_equal(struct mdfour *md1, struct mdfour *md2)
 void
 hash_delimiter(struct mdfour *md, const char *type)
 {
-       hash_buffer(md, HASH_DELIMITER, sizeof(HASH_DELIMITER));
-       hash_buffer(md, type, strlen(type) + 1); // Include NUL.
+       hash_binary_buffer(md, HASH_DELIMITER, sizeof(HASH_DELIMITER));
+       hash_binary_buffer(md, type, strlen(type) + 1); // Include NUL.
+       hash_debug_buffer(md, HASH_DEBUG_DELIMITER, strlen(HASH_DEBUG_DELIMITER));
+       hash_debug_buffer(md, type, strlen(type));
+       hash_debug_buffer(md, "\n", 1);
 }
 
 void
@@ -83,13 +141,19 @@ hash_string(struct mdfour *md, const char *s)
 void
 hash_string_length(struct mdfour *md, const char *s, int length)
 {
-       hash_buffer(md, s, length);
+       hash_binary_buffer(md, s, length);
+       hash_debug_buffer(md, s, length);
+       hash_debug_buffer(md, "\n", 1);
 }
 
 void
 hash_int(struct mdfour *md, int x)
 {
-       hash_buffer(md, (char *)&x, sizeof(x));
+       hash_binary_buffer(md, (char *)&x, sizeof(x));
+       char buf[16];
+       snprintf(buf, sizeof(buf), "%d", x);
+       hash_debug_buffer(md, buf, strlen(buf));
+       hash_debug_buffer(md, "\n", 1);
 }
 
 // Add contents of an open file to the hash. Returns true on success, otherwise
@@ -105,7 +169,8 @@ hash_fd(struct mdfour *md, int fd)
                        break;
                }
                if (n > 0) {
-                       hash_buffer(md, buf, n);
+                       hash_binary_buffer(md, buf, n);
+                       hash_debug_buffer(md, buf, n);
                }
        }
        return n == 0;

diff --git a/src/hashutil.c b/src/hashutil.c
index 0086bf7fd2b668b67ca15d76b34ea91c67a54177..3f515b4df10a152f64c866cf0dff82a0dd78996f 100644 (file)
--- a/src/hashutil.c
+++ b/src/hashutil.c
@@ -101,7 +101,7 @@ hash_source_code_string(
        }
 
        // Hash the source string.
-       hash_buffer(hash, str, len);
+       hash_string_length(hash, str, len);
 
        if (result & HASH_SOURCE_CODE_FOUND_DATE) {
                // Make sure that the hash sum changes if the (potential) expansion of
@@ -110,9 +110,9 @@ hash_source_code_string(
                struct tm *now = localtime(&t);
                cc_log("Found __DATE__ in %s", path);
                hash_delimiter(hash, "date");
-               hash_buffer(hash, &now->tm_year, sizeof(now->tm_year));
-               hash_buffer(hash, &now->tm_mon, sizeof(now->tm_mon));
-               hash_buffer(hash, &now->tm_mday, sizeof(now->tm_mday));
+               hash_int(hash, now->tm_year);
+               hash_int(hash, now->tm_mon);
+               hash_int(hash, now->tm_mday);
        }
        if (result & HASH_SOURCE_CODE_FOUND_TIME) {
                // We don't know for sure that the program actually uses the __TIME__

diff --git a/src/mdfour.c b/src/mdfour.c
index f557a20fb8322124b1e0d06ecfa1579bc96af671..4315b1053fe356006afcd50ad67d6ebcb9294b5c 100644 (file)
--- a/src/mdfour.c
+++ b/src/mdfour.c
@@ -131,6 +131,12 @@ mdfour_begin(struct mdfour *md)
        md->totalN = 0;
        md->tail_len = 0;
        md->finalized = 0;
+       md->identifier = 0;
+}
+
+void mdfour_identify(struct mdfour *md, int identifier)
+{
+       md->identifier = identifier;
 }
 
 static
@@ -161,14 +167,6 @@ void mdfour_tail(const unsigned char *in, size_t n)
 void
 mdfour_update(struct mdfour *md, const unsigned char *in, size_t n)
 {
-#ifdef CCACHE_DEBUG_HASH
-       if (n > 0 && getenv("CCACHE_DEBUG_HASH")) {
-               FILE *f = fopen("ccache-debug-hash.bin", "a");
-               fwrite(in, 1, n, f);
-               fclose(f);
-       }
-#endif
-
        m = md;
 
        if (!in) {

diff --git a/src/mdfour.h b/src/mdfour.h
index c196a09e952ef7b164caa94d272ca031d02e32f4..1970723a6a04dd7453e52f5d0163a3f9a58619d7 100644 (file)
--- a/src/mdfour.h
+++ b/src/mdfour.h
@@ -10,9 +10,11 @@ struct mdfour {
        unsigned char tail[64];
        size_t tail_len;
        int finalized;
+       int identifier;
 };
 
 void mdfour_begin(struct mdfour *md);
+void mdfour_identify(struct mdfour *md, int identifier);
 void mdfour_update(struct mdfour *md, const unsigned char *in, size_t n);
 void mdfour_result(struct mdfour *md, unsigned char *out);