src/Compressor.cpp \
src/Config.cpp \
src/Context.cpp \
+ src/Counters.cpp \
src/Decompressor.cpp \
src/NullCompressor.cpp \
src/NullDecompressor.cpp \
src/cleanup.cpp \
src/compopt.cpp \
src/compress.cpp \
- src/counters.cpp \
src/execute.cpp \
src/exitfn.cpp \
src/hash.cpp \
test_suites += unittest/test_args.cpp
test_suites += unittest/test_argument_processing.cpp
test_suites += unittest/test_compopt.cpp
-test_suites += unittest/test_counters.cpp
test_suites += unittest/test_hash.cpp
test_suites += unittest/test_hashutil.cpp
test_suites += unittest/test_legacy_util.cpp
test_suites += unittest/test_lockfile.cpp
-test_suites += unittest/test_stats.cpp
test_sources += unittest/catch2_tests.cpp
test_sources += unittest/framework.cpp
#include "Context.hpp"
+#include "Counters.hpp"
#include "Util.hpp"
#include "args.hpp"
-#include "counters.hpp"
Context::Context()
: actual_cwd(Util::get_actual_cwd()),
free(ignore_headers[i]);
}
free(ignore_headers);
-
- counters_free(counter_updates);
}
// Current working directory according to $PWD (falling back to getcwd(3)).
std::string apparent_cwd;
- // Full path to the statistics file in the subdirectory where the cached
- // result belongs (<cache_dir>/<x>/stats).
- std::string stats_file;
-
// The original argument list.
struct args* orig_args = nullptr;
char** ignore_headers = nullptr;
size_t ignore_headers_len = 0;
- struct counters* counter_updates = nullptr;
+ // Full path to the statistics file in the subdirectory where the cached
+ // result belongs (<cache_dir>/<x>/stats).
+ std::string stats_file;
+
+ // Statistics which get written into the `stats_file` upon exit.
+ Counters counter_updates;
};
// this program; if not, write to the Free Software Foundation, Inc., 51
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-// This file contains tests for statistics handling.
+#include "Counters.hpp"
-#include "../src/counters.hpp"
-#include "../src/stats.hpp"
-#include "framework.hpp"
-#include "util.hpp"
+#include "stats.hpp"
-TEST_SUITE(stats)
+#include <algorithm>
-TEST(forward_compatibility)
+Counters::Counters() : m_counters(STATS_END)
{
- unsigned i;
- FILE* f;
- struct counters* counters = counters_init(0);
+}
- f = fopen("stats", "w");
- for (i = 0; i < 100; i++) {
- fprintf(f, "%u\n", i);
+unsigned& Counters::operator[](size_t index)
+{
+ if (index >= m_counters.size()) {
+ m_counters.resize(index + 1);
}
- fclose(f);
-
- stats_read("stats", counters);
- CHECK_INT_EQ(100, counters->size);
- CHECK_INT_EQ(73, counters->data[73]);
+ return m_counters.at(index);
+}
- stats_write("stats", counters);
- CHECK_INT_EQ(100, counters->size);
- CHECK_INT_EQ(99, counters->data[99]);
+unsigned Counters::operator[](size_t index) const
+{
+ return index < m_counters.size() ? m_counters.at(index) : 0;
+}
- counters_free(counters);
+size_t
+Counters::size() const
+{
+ return m_counters.size();
}
-TEST_SUITE_END
+bool
+Counters::all_zero() const
+{
+ return !std::any_of(
+ m_counters.begin(), m_counters.end(), [](unsigned v) { return v != 0; });
+}
#include "system.hpp"
-#include <cstddef>
+#include <vector>
-struct counters
+// A simple wrapper around a vector of integers
+// used for the statistics counters.
+class Counters
{
- unsigned* data; // counter value
- size_t size; // logical array size
- size_t allocated; // allocated size
-};
+public:
+ Counters();
+
+ unsigned& operator[](size_t index);
+ unsigned operator[](size_t index) const;
+
+ size_t size() const;
-struct counters* counters_init(size_t initial_size);
-void counters_resize(struct counters* c, size_t new_size);
-void counters_free(struct counters* c);
+ // Return true if all counters are zero, false otherwise.
+ bool all_zero() const;
+
+private:
+ std::vector<unsigned> m_counters;
+};
cc_log("Failed to add result name to %s", ctx.manifest_path.c_str());
} else {
auto st = Stat::stat(ctx.manifest_path, Stat::OnError::log);
- stats_update_size(ctx,
- ctx.manifest_stats_file,
- st.size_on_disk() - old_st.size_on_disk(),
- !old_st && st ? 1 : 0);
+
+ int64_t size_delta = st.size_on_disk() - old_st.size_on_disk();
+ int nof_files_delta = !old_st && st ? 1 : 0;
+
+ if (ctx.stats_file == ctx.manifest_stats_file) {
+ stats_update_size(ctx.counter_updates, size_delta, nof_files_delta);
+ } else {
+ Counters counters;
+ stats_update_size(counters, size_delta, nof_files_delta);
+ stats_flush_to_file(ctx.config, ctx.manifest_stats_file, counters);
+ }
}
MTR_END("manifest", "manifest_put");
}
if (!new_dest_stat) {
failed(STATS_ERROR);
}
- stats_update_size(ctx,
- ctx.stats_file,
+ stats_update_size(ctx.counter_updates,
new_dest_stat.size_on_disk()
- orig_dest_stat.size_on_disk(),
orig_dest_stat ? 0 : 1);
#include "system.hpp"
-#include "counters.hpp"
+#include "Counters.hpp"
#include "stats.hpp"
#include "third_party/nonstd/optional.hpp"
uint64_t new_size =
Stat::stat(cache_file.path(), Stat::OnError::log).size_on_disk();
- stats_update_size(ctx, stats_file, new_size - old_size, 0);
+ size_t size_delta = new_size - old_size;
+ if (ctx.stats_file == stats_file) {
+ stats_update_size(ctx.counter_updates, size_delta, 0);
+ } else {
+ Counters counters;
+ stats_update_size(counters, size_delta, 0);
+ stats_flush_to_file(ctx.config, stats_file, counters);
+ }
+
cc_log("Recompression of %s done", cache_file.path().c_str());
}
+++ /dev/null
-// Copyright (C) 2010-2020 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-// A simple array of unsigned integers used for the statistics counters.
-
-#include "counters.hpp"
-
-#include "legacy_util.hpp"
-
-// Allocate and initialize a struct counters. Data entries up to the size are
-// set to 0.
-struct counters*
-counters_init(size_t initial_size)
-{
- auto c = static_cast<counters*>(x_malloc(sizeof(counters)));
- c->data = nullptr;
- c->size = 0;
- c->allocated = 0;
- counters_resize(c, initial_size);
- return c;
-}
-
-// Free a counters struct.
-void
-counters_free(struct counters* c)
-{
- if (c) {
- free(c->data);
- free(c);
- }
-}
-
-// Set a new size. New data entries are set to 0.
-void
-counters_resize(struct counters* c, size_t new_size)
-{
- if (new_size > c->size) {
- bool realloc = false;
- while (c->allocated < new_size) {
- c->allocated += 32 + c->allocated;
- realloc = true;
- }
- if (realloc) {
- c->data = static_cast<unsigned*>(
- x_realloc(c->data, c->allocated * sizeof(c->data[0])));
- }
- for (size_t i = c->size; i < new_size; i++) {
- c->data[i] = 0;
- }
- }
-
- c->size = new_size;
-}
}
auto new_stat = Stat::stat(raw_file);
- stats_update_size(ctx,
- ctx.stats_file,
+ stats_update_size(ctx.counter_updates,
new_stat.size_on_disk() - old_stat.size_on_disk(),
(new_stat ? 1 : 0) - (old_stat ? 1 : 0));
}
#include "AtomicFile.hpp"
#include "Context.hpp"
+#include "Counters.hpp"
#include "cleanup.hpp"
-#include "counters.hpp"
#include "hashutil.hpp"
#include "lockfile.hpp"
#include "logging.hpp"
static char* format_size_times_1024(uint64_t size);
static char* format_timestamp(uint64_t timestamp);
-static void stats_flush_to_file(const Config& config,
- std::string sfile,
- struct counters* updates);
// Statistics fields in display order.
static struct
// Parse a stats file from a buffer, adding to the counters.
static void
-parse_stats(struct counters* counters, const char* buf)
+parse_stats(Counters& counters, const char* buf)
{
size_t i = 0;
const char* p = buf;
if (p2 == p) {
break;
}
- if (counters->size < i + 1) {
- counters_resize(counters, i + 1);
- }
- counters->data[i] += val;
+ counters[i] += val;
i++;
p = p2;
}
// Write out a stats file.
void
-stats_write(const std::string& path, struct counters* counters)
+stats_write(const std::string& path, const Counters& counters)
{
AtomicFile file(path, AtomicFile::Mode::text);
- for (size_t i = 0; i < counters->size; ++i) {
- file.write(fmt::format("{}\n", counters->data[i]));
+ for (size_t i = 0; i < counters.size(); ++i) {
+ file.write(fmt::format("{}\n", counters[i]));
}
try {
file.commit();
}
}
-static void
-init_counter_updates(Context& ctx)
-{
- if (!ctx.counter_updates) {
- ctx.counter_updates = counters_init(STATS_END);
- }
-}
-
static double
-stats_hit_rate(struct counters* counters)
+stats_hit_rate(const Counters& counters)
{
- unsigned direct = counters->data[STATS_CACHEHIT_DIR];
- unsigned preprocessed = counters->data[STATS_CACHEHIT_CPP];
+ unsigned direct = counters[STATS_CACHEHIT_DIR];
+ unsigned preprocessed = counters[STATS_CACHEHIT_CPP];
unsigned hit = direct + preprocessed;
- unsigned miss = counters->data[STATS_CACHEMISS];
+ unsigned miss = counters[STATS_CACHEMISS];
unsigned total = hit + miss;
return total > 0 ? (100.0 * hit) / total : 0.0;
}
static void
-stats_collect(const Config& config,
- struct counters* counters,
- time_t* last_updated)
+stats_collect(const Config& config, Counters& counters, time_t* last_updated)
{
unsigned zero_timestamp = 0;
fname = format("%s/%1x/stats", config.cache_dir().c_str(), dir);
}
- counters->data[STATS_ZEROTIMESTAMP] = 0; // Don't add
+ counters[STATS_ZEROTIMESTAMP] = 0; // Don't add
stats_read(fname, counters);
- zero_timestamp =
- std::max(counters->data[STATS_ZEROTIMESTAMP], zero_timestamp);
+ zero_timestamp = std::max(counters[STATS_ZEROTIMESTAMP], zero_timestamp);
auto st = Stat::stat(fname);
if (st && st.mtime() > *last_updated) {
*last_updated = st.mtime();
free(fname);
}
- counters->data[STATS_ZEROTIMESTAMP] = zero_timestamp;
+ counters[STATS_ZEROTIMESTAMP] = zero_timestamp;
}
// Record that a number of bytes and files have been added to the cache. Size
// is in bytes.
void
-stats_update_size(Context& ctx,
- const std::string& sfile,
- int64_t size,
- int files)
+stats_update_size(Counters& counters, int64_t size, int files)
{
if (size == 0 && files == 0) {
return;
}
- struct counters* updates;
- if (sfile == ctx.stats_file) {
- init_counter_updates(ctx);
- updates = ctx.counter_updates;
- } else {
- updates = counters_init(STATS_END);
- }
- updates->data[STATS_NUMFILES] += files;
- updates->data[STATS_TOTALSIZE] += size / 1024;
- if (sfile != ctx.stats_file) {
- stats_flush_to_file(ctx.config, sfile, updates);
- counters_free(updates);
- }
+ counters[STATS_TOTALSIZE] += size / 1024;
+ counters[STATS_NUMFILES] += files;
}
// Read in the stats from one directory and add to the counters.
void
-stats_read(const std::string& sfile, struct counters* counters)
+stats_read(const std::string& sfile, Counters& counters)
{
char* data = read_text_file(sfile.c_str(), 1024);
if (data) {
}
// Write counter updates in updates to sfile.
-static void
+void
stats_flush_to_file(const Config& config,
std::string sfile,
- struct counters* updates)
+ const Counters& updates)
{
- if (!updates) {
+ if (updates.all_zero()) {
return;
}
if (!config.log_file().empty() || config.debug()) {
for (auto& info : stats_info) {
- if (updates->data[info.stat] != 0 && !(info.flags & FLAG_NOZERO)) {
+ if (updates[info.stat] != 0 && !(info.flags & FLAG_NOZERO)) {
cc_log("Result: %s", info.message);
}
}
return;
}
- bool should_flush = false;
- for (int i = 0; i < STATS_END; ++i) {
- if (updates->data[i] > 0) {
- should_flush = true;
- break;
- }
- }
- if (!should_flush) {
- return;
- }
-
if (sfile.empty()) {
// An empty sfile means that we didn't get past calculate_object_hash(), so
// we just choose one of stats files in the 16 subdirectories.
return;
}
- struct counters* counters = counters_init(STATS_END);
+ Counters counters;
stats_read(sfile, counters);
for (int i = 0; i < STATS_END; ++i) {
- counters->data[i] += updates->data[i];
+ counters[i] += updates[i];
}
stats_write(sfile, counters);
lockfile_release(sfile.c_str());
bool need_cleanup = false;
if (config.max_files() != 0
- && counters->data[STATS_NUMFILES] > config.max_files() / 16) {
+ && counters[STATS_NUMFILES] > config.max_files() / 16) {
cc_log("Need to clean up %s since it holds %u files (limit: %u files)",
subdir.c_str(),
- counters->data[STATS_NUMFILES],
+ counters[STATS_NUMFILES],
config.max_files() / 16);
need_cleanup = true;
}
if (config.max_size() != 0
- && counters->data[STATS_TOTALSIZE] > config.max_size() / 1024 / 16) {
+ && counters[STATS_TOTALSIZE] > config.max_size() / 1024 / 16) {
cc_log("Need to clean up %s since it holds %u KiB (limit: %lu KiB)",
subdir.c_str(),
- counters->data[STATS_TOTALSIZE],
+ counters[STATS_TOTALSIZE],
(unsigned long)config.max_size() / 1024 / 16);
need_cleanup = true;
}
uint32_t max_files = round(config.max_files() * factor);
clean_up_dir(subdir, max_size, max_files, [](double) {});
}
-
- counters_free(counters);
}
// Write counter updates in counter_updates to disk.
stats_update(Context& ctx, enum stats stat)
{
assert(stat > STATS_NONE && stat < STATS_END);
- init_counter_updates(ctx);
- ctx.counter_updates->data[stat]++;
+ ctx.counter_updates[stat] += 1;
}
// Sum and display the total stats for all cache dirs.
void
stats_summary(const Config& config)
{
- struct counters* counters = counters_init(STATS_END);
+ Counters counters;
time_t last_updated;
stats_collect(config, counters, &last_updated);
if (stats_info[i].flags & FLAG_NEVER) {
continue;
}
- if (counters->data[stat] == 0 && !(stats_info[i].flags & FLAG_ALWAYS)) {
+ if (counters[stat] == 0 && !(stats_info[i].flags & FLAG_ALWAYS)) {
continue;
}
char* value;
if (stats_info[i].format) {
- value = stats_info[i].format(counters->data[stat]);
+ value = stats_info[i].format(counters[stat]);
} else {
- value = format("%8u", counters->data[stat]);
+ value = format("%8u", counters[stat]);
}
if (value) {
printf("%-31s %s\n", stats_info[i].message, value);
printf("max cache size %s\n", value);
free(value);
}
-
- counters_free(counters);
}
// Print machine-parsable (tab-separated) statistics counters.
void
stats_print(const Config& config)
{
- struct counters* counters = counters_init(STATS_END);
+ Counters counters;
time_t last_updated;
stats_collect(config, counters, &last_updated);
for (int i = 0; stats_info[i].message; i++) {
if (!(stats_info[i].flags & FLAG_NEVER)) {
- printf("%s\t%u\n", stats_info[i].id, counters->data[stats_info[i].stat]);
+ printf("%s\t%u\n", stats_info[i].id, counters[stats_info[i].stat]);
}
}
-
- counters_free(counters);
}
// Zero all the stats structures.
time_t timestamp = time(nullptr);
for (int dir = 0; dir <= 0xF; dir++) {
- struct counters* counters = counters_init(STATS_END);
+ Counters counters;
fname = format("%s/%1x/stats", config.cache_dir().c_str(), dir);
if (!Stat::stat(fname)) {
// No point in trying to reset the stats file if it doesn't exist.
stats_read(fname, counters);
for (unsigned i = 0; stats_info[i].message; i++) {
if (!(stats_info[i].flags & FLAG_NOZERO)) {
- counters->data[stats_info[i].stat] = 0;
+ counters[stats_info[i].stat] = 0;
}
}
- counters->data[STATS_ZEROTIMESTAMP] = timestamp;
+ counters[STATS_ZEROTIMESTAMP] = timestamp;
stats_write(fname, counters);
lockfile_release(fname);
}
- counters_free(counters);
free(fname);
}
}
unsigned* maxfiles,
uint64_t* maxsize)
{
- struct counters* counters = counters_init(STATS_END);
+ Counters counters;
char* sname = format("%s/stats", dir);
stats_read(sname, counters);
- *maxfiles = counters->data[STATS_OBSOLETE_MAXFILES];
- *maxsize = (uint64_t)counters->data[STATS_OBSOLETE_MAXSIZE] * 1024;
+ *maxfiles = counters[STATS_OBSOLETE_MAXFILES];
+ *maxsize = (uint64_t)counters[STATS_OBSOLETE_MAXSIZE] * 1024;
free(sname);
- counters_free(counters);
}
// Set the per-directory sizes.
void
stats_set_sizes(const char* dir, unsigned num_files, uint64_t total_size)
{
- struct counters* counters = counters_init(STATS_END);
+ Counters counters;
char* statsfile = format("%s/stats", dir);
if (lockfile_acquire(statsfile, k_lock_staleness_limit)) {
stats_read(statsfile, counters);
- counters->data[STATS_NUMFILES] = num_files;
- counters->data[STATS_TOTALSIZE] = total_size / 1024;
+ counters[STATS_NUMFILES] = num_files;
+ counters[STATS_TOTALSIZE] = total_size / 1024;
stats_write(statsfile, counters);
lockfile_release(statsfile);
}
free(statsfile);
- counters_free(counters);
}
// Count directory cleanup run.
void
stats_add_cleanup(const char* dir, unsigned count)
{
- struct counters* counters = counters_init(STATS_END);
+ Counters counters;
char* statsfile = format("%s/stats", dir);
if (lockfile_acquire(statsfile, k_lock_staleness_limit)) {
stats_read(statsfile, counters);
- counters->data[STATS_NUMCLEANUPS] += count;
+ counters[STATS_NUMCLEANUPS] += count;
stats_write(statsfile, counters);
lockfile_release(statsfile);
}
free(statsfile);
- counters_free(counters);
}
#include "system.hpp"
+#include "Counters.hpp"
+
#include <string>
class Config;
void stats_update(Context& ctx, enum stats stat);
void stats_flush(void* context);
+void stats_flush_to_file(const Config& config,
+ std::string sfile,
+ const Counters& updates);
void stats_zero(const Config& config);
void stats_summary(const Config& config);
void stats_print(const Config& config);
-void stats_update_size(Context& ctx,
- const std::string& sfile,
- int64_t size,
- int files);
+
+void stats_update_size(Counters& counters, int64_t size, int files);
void stats_get_obsolete_limits(const char* dir,
unsigned* maxfiles,
uint64_t* maxsize);
void stats_set_sizes(const char* dir, unsigned num_files, uint64_t total_size);
void stats_add_cleanup(const char* dir, unsigned count);
-void stats_timestamp(time_t time, struct counters* counters);
-void stats_read(const std::string& path, struct counters* counters);
-void stats_write(const std::string& path, struct counters* counters);
+void stats_read(const std::string& path, Counters& counters);
+void stats_write(const std::string& path, const Counters& counters);
expect_stat 'cache hit (preprocessed)' 0
expect_stat 'cache miss' 0
+ # -------------------------------------------------------------------------
+ TEST "stats file forward compatibility"
+
+ mkdir -p "$CCACHE_DIR/4/"
+ stats_file="$CCACHE_DIR/4/stats"
+ touch "$CCACHE_DIR/timestamp_reference"
+
+ for i in `seq 101`; do
+ echo $i
+ done > "$stats_file"
+
+ expect_stat 'cache miss' 5
+ $CCACHE_COMPILE -c test1.c
+ expect_stat 'cache miss' 6
+ expect_file_contains "$stats_file" 101
+ expect_file_newer_than "$stats_file" "$CCACHE_DIR/timestamp_reference"
+
# -------------------------------------------------------------------------
TEST "CCACHE_RECACHE"
unsigned suite_argument_processing(unsigned);
unsigned suite_compopt(unsigned);
unsigned suite_conf(unsigned);
-unsigned suite_counters(unsigned);
unsigned suite_hash(unsigned);
unsigned suite_hashutil(unsigned);
unsigned suite_legacy_util(unsigned);
unsigned suite_lockfile(unsigned);
-unsigned suite_stats(unsigned);
const suite_fn k_legacy_suites[] = {
&suite_args,
&suite_argument_processing,
&suite_compopt,
- &suite_counters,
&suite_hash,
&suite_hashutil,
&suite_legacy_util,
&suite_lockfile,
- &suite_stats,
nullptr,
};
+++ /dev/null
-// Copyright (C) 2010-2020 Joel Rosdahl and other contributors
-//
-// See doc/AUTHORS.adoc for a complete list of contributors.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation; either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along with
-// this program; if not, write to the Free Software Foundation, Inc., 51
-// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-#include "../src/counters.hpp"
-#include "framework.hpp"
-#include "util.hpp"
-
-TEST_SUITE(counters)
-
-TEST(counters_init_0_should_allocate_0)
-{
- struct counters* counters = counters_init(0);
-
- CHECK_INT_EQ(0, counters->allocated);
- CHECK_INT_EQ(0, counters->size);
-
- counters_free(counters);
-}
-
-TEST(counters_init_7_should_allocate_32)
-{
- int i;
- struct counters* counters = counters_init(7);
-
- CHECK_INT_EQ(32, counters->allocated);
- CHECK_INT_EQ(7, counters->size);
- for (i = 0; i < 7; i++) {
- CHECK_INT_EQ(0, counters->data[i]);
- }
-
- counters_free(counters);
-}
-
-TEST(counters_resize_50_should_allocate_96)
-{
- struct counters* counters = counters_init(0);
-
- CHECK_INT_EQ(0, counters->allocated);
- counters_resize(counters, 50);
- CHECK_INT_EQ(50, counters->size);
- CHECK_INT_EQ(96, counters->allocated);
-
- counters_free(counters);
-}
-
-TEST_SUITE_END
projects / thirdparty / ccache.git / commitdiff
