AC_CHECK_HEADERS(ctype.h pwd.h stdlib.h string.h strings.h sys/time.h sys/mman.h)
AC_CHECK_HEADERS(syslog.h)
AC_CHECK_HEADERS(termios.h)
+AC_CHECK_HEADERS(sys/ioctl.h)
+AC_CHECK_HEADERS(linux/fs.h)
AC_CHECK_FUNCS(gethostname)
AC_CHECK_FUNCS(getopt_long)
* Low overhead.
* Compresses data in the cache to save disk space.
* Checksums data in the cache to detect corruption.
+* Optionally uses file cloning (AKA “copy on write” or reflinks) to avoid
+ copies (not supported by all file systems).
* Optionally uses hard links avoid copies (there are caveats, though).
One exception is if the cache is located on a compressed file system, in which
case the compression performed by ccache of course is redundant.
+
-Compression will be disabled if hard linking (the *hard_link* setting) is
-enabled.
+Compression will be disabled if file cloning (the *file_clone* setting) or hard
+linking (the *hard_link* setting) is enabled.
*compression_level* (*CCACHE_COMPRESSLEVEL*)::
the hash sum that identifies the build. The list separator is semicolon on
Windows systems and colon on other systems.
+*file_clone* (*CCACHE_FILECLONE* or *CCACHE_NOFILECLONE*, see <<_boolean_values,Boolean values>> above)::
+
+ If true, ccache will attempt to use file cloning (also known as “copy on
+ write”, “CoW” or “reflinks”) to store and fetch cached compiler results.
+ *file_clone* has priority over *hard_link*. The default is false.
++
+Files stored by cloning cannot be compressed, so the cache size will likely be
+significantly larger if this option is enabled. However, performance may be
+improved depending on the use case.
++
+Unlike the *hard_link* setting, *file_clone* is completely safe to use, but not
+all file systems support the feature. For such file systems, ccache will fall
+back to use plain copying (or hard links if *hard_link* is enabled).
+
*hard_link* (*CCACHE_HARDLINK* or *CCACHE_NOHARDLINK*, see <<_boolean_values,Boolean values>> above)::
- If true, ccache will attempt to use hard links to store compiler output
- files in the cache, and similarly use hard links when retrieving files from
- the cache. The default is false.
+ If true, ccache will attempt to use hard links to store and fetch cached
+ compiler results. The default is false.
+
An exception is dependency files (`.d`) which are never stored as hard links.
+
environment = {"CCACHE_DIR": ccache_dir, "PATH": environ["PATH"]}
environment["CCACHE_COMPILERCHECK"] = options.compilercheck
- if options.no_compression:
- environment["CCACHE_NOCOMPRESS"] = "1"
if options.compression_level:
environment["CCACHE_COMPRESSLEVEL"] = str(options.compression_level)
+ if options.file_clone:
+ environment["CCACHE_FILECLONE"] = "1"
if options.hardlink:
environment["CCACHE_HARDLINK"] = "1"
+ if options.no_compression:
+ environment["CCACHE_NOCOMPRESS"] = "1"
if options.nostats:
environment["CCACHE_NOSTATS"] = "1"
" files (default: %s)" % DEFAULT_DIRECTORY
),
)
+ op.add_option("--file-clone", help="use file cloning", action="store_true")
op.add_option("--hardlink", help="use hard links", action="store_true")
op.add_option(
"--hit-factor",
print("Compilercheck:", options.compilercheck)
print("Compression:", on_off(not options.no_compression))
print("Compression level:", options.compression_level or "default")
- print("Hardlink:", on_off(options.hardlink))
+ print("File cloning:", on_off(options.file_clone))
+ print("Hard linking:", on_off(options.hardlink))
print("Nostats:", on_off(options.nostats))
tmp_dir = "%s/perfdir.%d" % (abspath(options.directory), getpid())
char *get_path_in_cache(const char *name, const char *suffix);
bool copy_fd(int fd_in, int fd_out);
+bool clone_file(const char *src, const char *dest, bool via_tmp_file);
bool copy_file(const char *src, const char *dest, bool via_tmp_file);
bool move_file(const char *src, const char *dest);
int create_dir(const char *dir);
conf_create(void)
{
struct conf *conf = x_malloc(sizeof(*conf));
+
conf->base_dir = x_strdup("");
conf->cache_dir = format("%s/.ccache", get_home_directory());
conf->cache_dir_levels = 2;
conf->direct_mode = true;
conf->disable = false;
conf->extra_files_to_hash = x_strdup("");
+ conf->file_clone = false;
conf->hard_link = false;
conf->hash_dir = true;
conf->ignore_headers_in_manifest = x_strdup("");
conf->temporary_dir = x_strdup("");
conf->umask = UINT_MAX; // Default: don't set umask.
conf->unify = false;
+
conf->item_origins = x_malloc(confitems_count() * sizeof(char *));
for (size_t i = 0; i < confitems_count(); ++i) {
conf->item_origins[i] = "default";
ok &= print_item(conf, "direct_mode", printer, context);
ok &= print_item(conf, "disable", printer, context);
ok &= print_item(conf, "extra_files_to_hash", printer, context);
+ ok &= print_item(conf, "file_clone", printer, context);
ok &= print_item(conf, "hard_link", printer, context);
ok &= print_item(conf, "hash_dir", printer, context);
ok &= print_item(conf, "ignore_headers_in_manifest", printer, context);
bool direct_mode;
bool disable;
char *extra_files_to_hash;
+ bool file_clone;
bool hard_link;
bool hash_dir;
char *ignore_headers_in_manifest;
direct_mode, ITEM(direct_mode, bool)
disable, ITEM(disable, bool)
extra_files_to_hash, ITEM(extra_files_to_hash, env_string)
+file_clone, ITEM(file_clone, bool)
hard_link, ITEM(hard_link, bool)
hash_dir, ITEM(hash_dir, bool)
ignore_headers_in_manifest, ITEM(ignore_headers_in_manifest, env_string)
DISABLE, "disable"
EXTENSION, "cpp_extension"
EXTRAFILES, "extra_files_to_hash"
+FILECLONE, "file_clone"
HARDLINK, "hard_link"
HASHDIR, "hash_dir"
IGNOREHEADERS, "ignore_headers_in_manifest"
static bool
copy_raw_file(const char *source, const char *dest, bool to_cache)
{
+ if (conf->file_clone) {
+ cc_log("Cloning %s to %s", source, dest);
+ if (clone_file(source, dest, to_cache)) {
+ return true;
+ }
+ cc_log("Failed to clone: %s", strerror(errno));
+ }
if (conf->hard_link) {
x_try_unlink(dest);
cc_log("Hard linking %s to %s", source, dest);
if (ret == 0) {
return true;
}
- cc_log("Failed to hard link %s to %s: %s", source, dest, strerror(errno));
+ cc_log("Failed to hard link: %s", strerror(errno));
}
cc_log("Copying %s to %s", source, dest);
WRITE_BYTE(list->n_files);
for (uint32_t i = 0; i < list->n_files; i++) {
- write_entry_fn write_entry;
- if (conf->hard_link && should_hard_link_suffix(list->files[i].suffix)) {
- write_entry = write_raw_file_entry;
- } else {
- write_entry = write_embedded_file_entry;
- }
-
+ bool store_raw =
+ conf->file_clone
+ || (conf->hard_link && should_hard_link_suffix(list->files[i].suffix));
+ write_entry_fn write_entry =
+ store_raw ? write_raw_file_entry : write_embedded_file_entry;
if (!write_entry(
compressor, compr_state, result_path_in_cache, i, &list->files[i])) {
goto error;
#include <sys/time.h>
#endif
+#ifdef __linux__
+# ifdef HAVE_SYS_IOCTL_H
+# include <sys/ioctl.h>
+# endif
+# ifdef HAVE_LINUX_FS_H
+# include <linux/fs.h>
+# ifndef FICLONE
+# define FICLONE _IOW(0x94, 9, int)
+# endif
+# define FILE_CLONING_SUPPORTED 1
+# endif
+#endif
+
#ifdef _WIN32
#include <windows.h>
#include <sys/locking.h>
}
#endif
+// Clone a file from src to dest. If via_tmp_file is true, the file is cloned
+// to a temporary file and then renamed to dest.
+bool
+clone_file(const char *src, const char *dest, bool via_tmp_file)
+{
+ bool result;
+
+#ifdef FILE_CLONING_SUPPORTED
+
+#if defined(__linux__)
+ int src_fd = open(src, O_RDONLY);
+ if (src_fd == -1) {
+ return false;
+ }
+
+ int dest_fd;
+ char *tmp_file = NULL;
+ if (via_tmp_file) {
+ tmp_file = x_strdup(dest);
+ dest_fd = create_tmp_fd(&tmp_file);
+ } else {
+ dest_fd = open(dest, O_WRONLY | O_CREAT | O_BINARY, 0666);
+ if (dest_fd == -1) {
+ close(dest_fd);
+ close(src_fd);
+ return false;
+ }
+ }
+
+ int saved_errno = 0;
+ if (ioctl(dest_fd, FICLONE, src_fd) == 0) {
+ result = true;
+ } else {
+ result = false;
+ saved_errno = errno;
+ }
+
+ close(dest_fd);
+ close(src_fd);
+
+ if (via_tmp_file) {
+ x_try_unlink(dest);
+ if (x_rename(tmp_file, dest) != 0) {
+ result = false;
+ }
+ free(tmp_file);
+ }
+
+ errno = saved_errno;
+#endif
+
+#else // FILE_CLONING_SUPPORTED
+
+ (void)src;
+ (void)dest;
+ (void)via_tmp_file;
+ errno = EOPNOTSUPP;
+ result = false;
+
+#endif // FILE_CLONING_SUPPORTED
+
+ return result;
+}
+
// Copy a file from src to dest. If via_tmp_file is true, the file is copied to
// a temporary file and then renamed to dest.
bool
split_dwarf
masquerading
hardlink
+fileclone
direct
direct_gcc
depend
--- /dev/null
+SUITE_fileclone_PROBE() {
+ touch file1
+ if ! cp --reflink=always file1 file2 >/dev/null 2>&1; then
+ echo "file system doesn't support file cloning"
+ fi
+}
+
+SUITE_fileclone() {
+ # -------------------------------------------------------------------------
+ TEST "Base case"
+
+ generate_code 1 test.c
+
+ $REAL_COMPILER -c -o reference_test.o test.c
+
+ CCACHE_FILECLONE=1 CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (preprocessed)' 0
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 2
+ expect_equal_object_files reference_test.o test.o
+
+ CCACHE_FILECLONE=1 CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (preprocessed)' 1
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 2
+ expect_equal_object_files reference_test.o test.o
+ if ! grep -q 'Cloning.*to test.o' test.o.ccache-log; then
+ test_failed "Did not try to clone file"
+ fi
+ if grep -q 'Failed to clone' test.o.ccache-log; then
+ test_failed "Failed to clone"
+ fi
+
+ # -------------------------------------------------------------------------
+ TEST "Cloning not used for stored non-raw result"
+
+ generate_code 1 test.c
+
+ $REAL_COMPILER -c -o reference_test.o test.c
+
+ $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (preprocessed)' 0
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 1
+ expect_equal_object_files reference_test.o test.o
+
+ CCACHE_FILECLONE=1 CCACHE_DEBUG=1 $CCACHE_COMPILE -c test.c
+ expect_stat 'cache hit (preprocessed)' 1
+ expect_stat 'cache miss' 1
+ expect_stat 'files in cache' 1
+ expect_equal_object_files reference_test.o test.o
+ if grep -q 'Cloning' test.o.ccache-log; then
+ test_failed "Tried to clone"
+ fi
+}
#include "framework.h"
#include "util.h"
-#define N_CONFIG_ITEMS 34
+#define N_CONFIG_ITEMS 35
static struct {
char *descr;
char *origin;
CHECK(conf->direct_mode);
CHECK(!conf->disable);
CHECK_STR_EQ("", conf->extra_files_to_hash);
+ CHECK(!conf->file_clone);
CHECK(!conf->hard_link);
CHECK(conf->hash_dir);
CHECK_STR_EQ("", conf->ignore_headers_in_manifest);
"direct_mode = false\n"
"disable = true\n"
"extra_files_to_hash = a:b c:$USER\n"
+ "file_clone = true\n"
"hard_link = true\n"
"hash_dir = false\n"
"ignore_headers_in_manifest = a:b/c\n"
CHECK(!conf->direct_mode);
CHECK(conf->disable);
CHECK_STR_EQ_FREE1(format("a:b c:%s", user), conf->extra_files_to_hash);
+ CHECK(conf->file_clone);
CHECK(conf->hard_link);
CHECK(!conf->hash_dir);
CHECK_STR_EQ("a:b/c", conf->ignore_headers_in_manifest);
false,
true,
"efth",
+ .file_clone = true,
true,
.hash_dir = false,
"ihim",
CHECK_STR_EQ("direct_mode = false", received_conf_items[n++].descr);
CHECK_STR_EQ("disable = true", received_conf_items[n++].descr);
CHECK_STR_EQ("extra_files_to_hash = efth", received_conf_items[n++].descr);
+ CHECK_STR_EQ("file_clone = true", received_conf_items[n++].descr);
CHECK_STR_EQ("hard_link = true", received_conf_items[n++].descr);
CHECK_STR_EQ("hash_dir = false", received_conf_items[n++].descr);
CHECK_STR_EQ("ignore_headers_in_manifest = ihim",
projects / thirdparty / ccache.git / commitdiff
