From: Anders Björklund Date: Fri, 13 Nov 2015 20:08:02 +0000 (+0100) Subject: Add feature to ignore headers in manifest X-Git-Tag: v3.3~135 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=316cdda7572501e4fcef09155de82f6e9ddf6786;p=thirdparty%2Fccache.git Add feature to ignore headers in manifest Certain headers are never changed, or perhaps not so important (timestamps). Avoid repeatedly hashing these in the manifest, to make direct mode faster. One can either use the full path to the include file, or perhaps a directory of headers to ignore all the headers under that directory (e.g. compiler) --- diff --git a/MANUAL.txt b/MANUAL.txt index 31750954f..3af3383f6 100644 --- a/MANUAL.txt +++ b/MANUAL.txt @@ -374,6 +374,13 @@ WRAPPERS>>. If you strike problems with GDB not using the correct directory then enable this option. +*ignore_headers_in_manifest* (*CCACHE_IGNOREHEADERS*):: + + This setting is a list of paths to files (or directories with headers) that + ccache will *not* include in the manifest list that makes up the direct mode. + Note that this can cause stale cache hits, if those headers do indeed change. + The list separator is semicolon on Windows systems and colon on other systems. + *log_file* (*CCACHE_LOGFILE*):: If set to a file path, ccache will write information on what it is doing to diff --git a/ccache.c b/ccache.c index 810b2940a..0fa17c1c2 100644 --- a/ccache.c +++ b/ccache.c @@ -177,6 +177,12 @@ time_t time_of_compilation; */ static struct hashtable *included_files; +/* List of headers to ignore */ +static char **ignore_headers; + +/* Size of headers to ignore list */ +static size_t ignore_headers_len; + /* is gcc being asked to output dependencies? */ static bool generating_dependencies; @@ -524,6 +530,9 @@ remember_include_file(char *path, struct mdfour *cpp_hash, bool system) size_t size; bool is_pch; size_t path_len = strlen(path); + char *ignore; + size_t ignore_len; + size_t i; if (path_len >= 2 && (path[0] == '<' && path[path_len - 1] == '>')) { /* Typically or . */ @@ -566,6 +575,17 @@ remember_include_file(char *path, struct mdfour *cpp_hash, bool system) goto failure; } + for (i = 0; i < ignore_headers_len; i++) { + ignore = ignore_headers[i]; + ignore_len = strlen(ignore); + if (ignore_len > path_len) { + continue; + } + if (strncmp(path, ignore, ignore_len) == 0) { + goto ignore; + } + } + /* Let's hash the include file. */ if (!(conf->sloppiness & SLOPPY_INCLUDE_FILE_MTIME) && st.st_mtime >= time_of_compilation) { @@ -709,6 +729,21 @@ process_preprocessed_file(struct mdfour *hash, const char *path) return false; } + ignore_headers = NULL; + ignore_headers_len = 0; + if (!str_eq(conf->ignore_headers_in_manifest, "")) { + char *header, *p, *q, *saveptr = NULL; + p = x_strdup(conf->ignore_headers_in_manifest); + q = p; + while ((header = strtok_r(q, PATH_DELIM, &saveptr))) { + ignore_headers = x_realloc(ignore_headers, + (ignore_headers_len+1) * sizeof(char*)); + ignore_headers[ignore_headers_len++] = x_strdup(header); + q = NULL; + } + free(p); + } + included_files = create_hashtable(1000, hash_from_string, strings_equal); /* Bytes between p and q are pending to be hashed. */ @@ -2917,6 +2952,8 @@ initialize(void) void cc_reset(void) { + size_t i; + conf_free(conf); conf = NULL; free(primary_config_path); primary_config_path = NULL; free(secondary_config_path); secondary_config_path = NULL; @@ -2939,6 +2976,12 @@ cc_reset(void) free(cached_dia); cached_dia = NULL; free(manifest_path); manifest_path = NULL; time_of_compilation = 0; + for (i = 0; i < ignore_headers_len; i++) { + free(ignore_headers[i]); + ignore_headers[i] = NULL; + } + free(ignore_headers); ignore_headers = NULL; + ignore_headers_len = 0; if (included_files) { hashtable_destroy(included_files, 1); included_files = NULL; } diff --git a/conf.c b/conf.c index 1461ed586..6ff9dddae 100644 --- a/conf.c +++ b/conf.c @@ -315,6 +315,7 @@ conf_create(void) conf->extra_files_to_hash = x_strdup(""); conf->hard_link = false; conf->hash_dir = false; + conf->ignore_headers_in_manifest = x_strdup(""); conf->log_file = x_strdup(""); conf->max_files = 0; conf->max_size = (uint64_t)5 * 1000 * 1000 * 1000; @@ -348,6 +349,7 @@ conf_free(struct conf *conf) free(conf->compiler_check); free(conf->cpp_extension); free(conf->extra_files_to_hash); + free(conf->ignore_headers_in_manifest); free(conf->log_file); free(conf->path); free(conf->prefix_command); @@ -566,6 +568,10 @@ conf_print_items(struct conf *conf, reformat(&s, "hash_dir = %s", bool_to_string(conf->hash_dir)); printer(s, conf->item_origins[find_conf("hash_dir")->number], context); + reformat(&s, "ignore_headers_in_manifest = %s", conf->ignore_headers_in_manifest); + printer(s, conf->item_origins[find_conf("ignore_headers_in_manifest")->number], + context); + reformat(&s, "log_file = %s", conf->log_file); printer(s, conf->item_origins[find_conf("log_file")->number], context); diff --git a/conf.h b/conf.h index 0e3002854..df0ac98e9 100644 --- a/conf.h +++ b/conf.h @@ -17,6 +17,7 @@ struct conf { char *extra_files_to_hash; bool hard_link; bool hash_dir; + char *ignore_headers_in_manifest; char *log_file; unsigned max_files; uint64_t max_size; diff --git a/confitems.gperf b/confitems.gperf index 6fb21ef5c..3f0920696 100644 --- a/confitems.gperf +++ b/confitems.gperf @@ -20,17 +20,18 @@ disable, 9, ITEM(disable, bool) extra_files_to_hash, 10, ITEM(extra_files_to_hash, env_string) hard_link, 11, ITEM(hard_link, bool) hash_dir, 12, ITEM(hash_dir, bool) -log_file, 13, ITEM(log_file, env_string) -max_files, 14, ITEM(max_files, unsigned) -max_size, 15, ITEM(max_size, size) -path, 16, ITEM(path, env_string) -prefix_command, 17, ITEM(prefix_command, env_string) -read_only, 18, ITEM(read_only, bool) -read_only_direct, 19, ITEM(read_only_direct, bool) -recache, 20, ITEM(recache, bool) -run_second_cpp, 21, ITEM(run_second_cpp, bool) -sloppiness, 22, ITEM(sloppiness, sloppiness) -stats, 23, ITEM(stats, bool) -temporary_dir, 24, ITEM(temporary_dir, env_string) -umask, 25, ITEM(umask, umask) -unify, 26, ITEM(unify, bool) +ignore_headers_in_manifest, 13, ITEM(ignore_headers_in_manifest, env_string) +log_file, 14, ITEM(log_file, env_string) +max_files, 15, ITEM(max_files, unsigned) +max_size, 16, ITEM(max_size, size) +path, 17, ITEM(path, env_string) +prefix_command, 18, ITEM(prefix_command, env_string) +read_only, 19, ITEM(read_only, bool) +read_only_direct, 20, ITEM(read_only_direct, bool) +recache, 21, ITEM(recache, bool) +run_second_cpp, 22, ITEM(run_second_cpp, bool) +sloppiness, 23, ITEM(sloppiness, sloppiness) +stats, 24, ITEM(stats, bool) +temporary_dir, 25, ITEM(temporary_dir, env_string) +umask, 26, ITEM(umask, umask) +unify, 27, ITEM(unify, bool) diff --git a/confitems_lookup.c b/confitems_lookup.c index 39e95afaf..539f8601a 100644 --- a/confitems_lookup.c +++ b/confitems_lookup.c @@ -55,7 +55,7 @@ confitems_hash (register const char *str, register unsigned int len) 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 20, 5, 0, - 10, 0, 50, 50, 15, 5, 50, 50, 20, 10, + 10, 0, 50, 0, 15, 5, 50, 50, 20, 10, 0, 0, 10, 50, 0, 0, 0, 5, 50, 50, 30, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, @@ -87,9 +87,9 @@ confitems_get (register const char *str, register unsigned int len) { enum { - TOTAL_KEYWORDS = 27, + TOTAL_KEYWORDS = 28, MIN_WORD_LENGTH = 4, - MAX_WORD_LENGTH = 19, + MAX_WORD_LENGTH = 26, MIN_HASH_VALUE = 5, MAX_HASH_VALUE = 49 }; @@ -99,64 +99,66 @@ confitems_get (register const char *str, register unsigned int len) {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, -#line 33 "confitems.gperf" - {"stats", 23, ITEM(stats, bool)}, +#line 34 "confitems.gperf" + {"stats", 24, ITEM(stats, bool)}, {"",0,NULL,0,NULL}, -#line 30 "confitems.gperf" - {"recache", 20, ITEM(recache, bool)}, +#line 31 "confitems.gperf" + {"recache", 21, ITEM(recache, bool)}, #line 13 "confitems.gperf" {"compiler", 3, ITEM(compiler, string)}, -#line 28 "confitems.gperf" - {"read_only", 18, ITEM(read_only, bool)}, -#line 36 "confitems.gperf" - {"unify", 26, ITEM(unify, bool)}, +#line 29 "confitems.gperf" + {"read_only", 19, ITEM(read_only, bool)}, +#line 37 "confitems.gperf" + {"unify", 27, ITEM(unify, bool)}, #line 15 "confitems.gperf" {"compression", 5, ITEM(compression, bool)}, {"",0,NULL,0,NULL}, -#line 34 "confitems.gperf" - {"temporary_dir", 24, ITEM(temporary_dir, env_string)}, +#line 35 "confitems.gperf" + {"temporary_dir", 25, ITEM(temporary_dir, env_string)}, #line 14 "confitems.gperf" {"compiler_check", 4, ITEM(compiler_check, string)}, {"",0,NULL,0,NULL}, -#line 29 "confitems.gperf" - {"read_only_direct", 19, ITEM(read_only_direct, bool)}, +#line 30 "confitems.gperf" + {"read_only_direct", 20, ITEM(read_only_direct, bool)}, #line 16 "confitems.gperf" {"compression_level", 6, ITEM(compression_level, unsigned)}, {"",0,NULL,0,NULL}, -#line 31 "confitems.gperf" - {"run_second_cpp", 21, ITEM(run_second_cpp, bool)}, -#line 35 "confitems.gperf" - {"umask", 25, ITEM(umask, umask)}, +#line 32 "confitems.gperf" + {"run_second_cpp", 22, ITEM(run_second_cpp, bool)}, +#line 36 "confitems.gperf" + {"umask", 26, ITEM(umask, umask)}, {"",0,NULL,0,NULL}, #line 19 "confitems.gperf" {"disable", 9, ITEM(disable, bool)}, #line 17 "confitems.gperf" {"cpp_extension", 7, ITEM(cpp_extension, string)}, -#line 27 "confitems.gperf" - {"prefix_command", 17, ITEM(prefix_command, env_string)}, +#line 28 "confitems.gperf" + {"prefix_command", 18, ITEM(prefix_command, env_string)}, {"",0,NULL,0,NULL}, #line 18 "confitems.gperf" {"direct_mode", 8, ITEM(direct_mode, bool)}, {"",0,NULL,0,NULL}, -#line 23 "confitems.gperf" - {"log_file", 13, ITEM(log_file, env_string)}, +#line 24 "confitems.gperf" + {"log_file", 14, ITEM(log_file, env_string)}, #line 11 "confitems.gperf" {"cache_dir", 1, ITEM(cache_dir, env_string)}, -#line 32 "confitems.gperf" - {"sloppiness", 22, ITEM(sloppiness, sloppiness)}, - {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, +#line 33 "confitems.gperf" + {"sloppiness", 23, ITEM(sloppiness, sloppiness)}, +#line 23 "confitems.gperf" + {"ignore_headers_in_manifest", 13, ITEM(ignore_headers_in_manifest, env_string)}, + {"",0,NULL,0,NULL}, #line 10 "confitems.gperf" {"base_dir", 0, ITEM_V(base_dir, env_string, absolute_path)}, -#line 26 "confitems.gperf" - {"path", 16, ITEM(path, env_string)}, +#line 27 "confitems.gperf" + {"path", 17, ITEM(path, env_string)}, {"",0,NULL,0,NULL}, #line 12 "confitems.gperf" {"cache_dir_levels", 2, ITEM_V(cache_dir_levels, unsigned, dir_levels)}, {"",0,NULL,0,NULL}, +#line 26 "confitems.gperf" + {"max_size", 16, ITEM(max_size, size)}, #line 25 "confitems.gperf" - {"max_size", 15, ITEM(max_size, size)}, -#line 24 "confitems.gperf" - {"max_files", 14, ITEM(max_files, unsigned)}, + {"max_files", 15, ITEM(max_files, unsigned)}, {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, {"",0,NULL,0,NULL}, #line 22 "confitems.gperf" @@ -183,4 +185,4 @@ confitems_get (register const char *str, register unsigned int len) } return 0; } -static const size_t CONFITEMS_TOTAL_KEYWORDS = 27; +static const size_t CONFITEMS_TOTAL_KEYWORDS = 28; diff --git a/envtoconfitems.gperf b/envtoconfitems.gperf index f623c4512..340e71555 100644 --- a/envtoconfitems.gperf +++ b/envtoconfitems.gperf @@ -21,6 +21,7 @@ EXTENSION, "cpp_extension" EXTRAFILES, "extra_files_to_hash" HARDLINK, "hard_link" HASHDIR, "hash_dir" +IGNOREHEADERS, "ignore_headers_in_manifest" LOGFILE, "log_file" MAXFILES, "max_files" MAXSIZE, "max_size" diff --git a/envtoconfitems_lookup.c b/envtoconfitems_lookup.c index cb8b53e52..2d9a33e08 100644 --- a/envtoconfitems_lookup.c +++ b/envtoconfitems_lookup.c @@ -52,7 +52,7 @@ envtoconfitems_hash (register const char *str, register unsigned int len) 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 20, 0, 0, 10, - 20, 43, 15, 43, 10, 43, 5, 10, 15, 0, + 20, 43, 15, 25, 10, 43, 5, 10, 15, 0, 5, 10, 5, 0, 0, 0, 43, 43, 43, 43, 10, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, 43, @@ -101,7 +101,7 @@ envtoconfitems_get (register const char *str, register unsigned int len) { enum { - TOTAL_KEYWORDS = 27, + TOTAL_KEYWORDS = 28, MIN_WORD_LENGTH = 2, MAX_WORD_LENGTH = 15, MIN_HASH_VALUE = 2, @@ -117,7 +117,7 @@ envtoconfitems_get (register const char *str, register unsigned int len) {"DIR", "cache_dir"}, #line 16 "envtoconfitems.gperf" {"CPP2", "run_second_cpp"}, -#line 34 "envtoconfitems.gperf" +#line 35 "envtoconfitems.gperf" {"STATS", "stats"}, #line 18 "envtoconfitems.gperf" {"DIRECT", "direct_mode"}, @@ -125,40 +125,40 @@ envtoconfitems_get (register const char *str, register unsigned int len) {"DISABLE", "disable"}, #line 14 "envtoconfitems.gperf" {"COMPRESS", "compression"}, -#line 28 "envtoconfitems.gperf" +#line 29 "envtoconfitems.gperf" {"PATH", "path"}, -#line 36 "envtoconfitems.gperf" +#line 37 "envtoconfitems.gperf" {"UMASK", "umask"}, {"",""}, -#line 32 "envtoconfitems.gperf" +#line 33 "envtoconfitems.gperf" {"RECACHE", "recache"}, #line 15 "envtoconfitems.gperf" {"COMPRESSLEVEL", "compression_level"}, {"",""}, -#line 37 "envtoconfitems.gperf" +#line 38 "envtoconfitems.gperf" {"UNIFY", "unify"}, {"",""}, -#line 35 "envtoconfitems.gperf" +#line 36 "envtoconfitems.gperf" {"TEMPDIR", "temporary_dir"}, -#line 30 "envtoconfitems.gperf" +#line 31 "envtoconfitems.gperf" {"READONLY", "read_only"}, #line 20 "envtoconfitems.gperf" {"EXTENSION", "cpp_extension"}, -#line 33 "envtoconfitems.gperf" +#line 34 "envtoconfitems.gperf" {"SLOPPINESS", "sloppiness"}, -#line 29 "envtoconfitems.gperf" +#line 30 "envtoconfitems.gperf" {"PREFIX", "prefix_command"}, -#line 24 "envtoconfitems.gperf" +#line 25 "envtoconfitems.gperf" {"LOGFILE", "log_file"}, #line 13 "envtoconfitems.gperf" {"COMPILERCHECK", "compiler_check"}, {"",""}, -#line 31 "envtoconfitems.gperf" +#line 32 "envtoconfitems.gperf" {"READONLY_DIRECT", "read_only_direct"}, {"",""}, -#line 26 "envtoconfitems.gperf" +#line 27 "envtoconfitems.gperf" {"MAXSIZE", "max_size"}, -#line 25 "envtoconfitems.gperf" +#line 26 "envtoconfitems.gperf" {"MAXFILES", "max_files"}, {"",""}, {"",""}, {"",""}, #line 23 "envtoconfitems.gperf" @@ -168,11 +168,13 @@ envtoconfitems_get (register const char *str, register unsigned int len) {"",""}, {"",""}, {"",""}, #line 11 "envtoconfitems.gperf" {"BASEDIR", "base_dir"}, - {"",""}, {"",""}, +#line 24 "envtoconfitems.gperf" + {"IGNOREHEADERS", "ignore_headers_in_manifest"}, + {"",""}, #line 21 "envtoconfitems.gperf" {"EXTRAFILES", "extra_files_to_hash"}, {"",""}, -#line 27 "envtoconfitems.gperf" +#line 28 "envtoconfitems.gperf" {"NLEVELS", "cache_dir_levels"} }; @@ -190,4 +192,4 @@ envtoconfitems_get (register const char *str, register unsigned int len) } return 0; } -static const size_t ENVTOCONFITEMS_TOTAL_KEYWORDS = 27; +static const size_t ENVTOCONFITEMS_TOTAL_KEYWORDS = 28; diff --git a/test.sh b/test.sh index 1090649b2..7038e4fa1 100755 --- a/test.sh +++ b/test.sh @@ -30,6 +30,7 @@ unset CCACHE_EXTENSION unset CCACHE_EXTRAFILES unset CCACHE_HARDLINK unset CCACHE_HASHDIR +unset CCACHE_IGNOREHEADERS unset CCACHE_LOGFILE unset CCACHE_NLEVELS unset CCACHE_NODIRECT @@ -1828,6 +1829,29 @@ EOF checkstat 'error hashing extra file' 1 } +ignoreheaders_suite() { + unset CCACHE_NODIRECT + + ################################################################# + # Check the ignore headers in the manifest feature + testname="ignore headers in manifest" + $CCACHE -Cz >/dev/null + cat <ignore.h +/* We don't want this header in the manifest */ +EOF + backdate ignore.h + cat <ignore.c +#include "ignore.h" +int foo; +EOF + CCACHE_IGNOREHEADERS="ignore.h" $CCACHE $COMPILER -c ignore.c + manifest=`find $CCACHE_DIR -name '*.manifest'` + data="`$CCACHE --dump-manifest $manifest | grep 'ignore.h'`" + if [ -n "$data" ]; then + test_failed "$manifest contained ignored header(s): $data" + fi +} + prepare_cleanup_test() { dir=$1 rm -rf $dir @@ -2519,6 +2543,7 @@ compression readonly readonly_direct extrafiles +ignoreheaders cleanup pch upgrade diff --git a/test/test_conf.c b/test/test_conf.c index 4c6f25211..01cdfd842 100644 --- a/test/test_conf.c +++ b/test/test_conf.c @@ -20,7 +20,7 @@ #include "test/framework.h" #include "test/util.h" -#define N_CONFIG_ITEMS 27 +#define N_CONFIG_ITEMS 28 static struct { char *descr; const char *origin; @@ -64,6 +64,7 @@ TEST(conf_create) CHECK_STR_EQ("", conf->extra_files_to_hash); CHECK(!conf->hard_link); CHECK(!conf->hash_dir); + CHECK_STR_EQ("", conf->ignore_headers_in_manifest); CHECK_STR_EQ("", conf->log_file); CHECK_INT_EQ(0, conf->max_files); CHECK_INT_EQ((uint64_t)5 * 1000 * 1000 * 1000, conf->max_size); @@ -107,6 +108,7 @@ TEST(conf_read_valid_config) "extra_files_to_hash = a:b c:$USER\n" "hard_link = true\n" "hash_dir = true\n" + "ignore_headers_in_manifest = a:b/c\n" "log_file = $USER${USER} \n" "max_files = 17\n" "max_size = 123M\n" @@ -137,6 +139,7 @@ TEST(conf_read_valid_config) CHECK_STR_EQ_FREE1(format("a:b c:%s", user), conf->extra_files_to_hash); CHECK(conf->hard_link); CHECK(conf->hash_dir); + CHECK_STR_EQ("a:b/c", conf->ignore_headers_in_manifest); CHECK_STR_EQ_FREE1(format("%s%s", user, user), conf->log_file); CHECK_INT_EQ(17, conf->max_files); CHECK_INT_EQ(123 * 1000 * 1000, conf->max_size); @@ -359,6 +362,7 @@ TEST(conf_print_items) "efth", true, true, + "ihim", "lf", 4711, 98.7 * 1000 * 1000, @@ -399,6 +403,7 @@ TEST(conf_print_items) CHECK_STR_EQ("extra_files_to_hash = efth", received_conf_items[n++].descr); CHECK_STR_EQ("hard_link = true", received_conf_items[n++].descr); CHECK_STR_EQ("hash_dir = true", received_conf_items[n++].descr); + CHECK_STR_EQ("ignore_headers_in_manifest = ihim", received_conf_items[n++].descr); CHECK_STR_EQ("log_file = lf", received_conf_items[n++].descr); CHECK_STR_EQ("max_files = 4711", received_conf_items[n++].descr); CHECK_STR_EQ("max_size = 98.7M", received_conf_items[n++].descr);