From: Joel Rosdahl Date: Wed, 4 Aug 2010 09:03:40 +0000 (+0200) Subject: Make reading and writing of statistics counters forward-compatible X-Git-Tag: v3.1~98 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=16b25cac09f8e6e4e59a2e90834dfde5b8587bbb;p=thirdparty%2Fccache.git Make reading and writing of statistics counters forward-compatible Previously, ccache threw away counters values it didn't know about, i.e., new counters coming from later ccache versions. --- diff --git a/Makefile.in b/Makefile.in index e8ae7beca..ef2584800 100644 --- a/Makefile.in +++ b/Makefile.in @@ -21,7 +21,8 @@ libs = @LIBS@ -lm -lz base_sources = \ ccache.c mdfour.c hash.c execute.c util.c args.c stats.c version.c \ cleanup.c snprintf.c unify.c manifest.c hashtable.c hashtable_itr.c \ - murmurhashneutral2.c hashutil.c getopt_long.c exitfn.c lockfile.c + murmurhashneutral2.c hashutil.c getopt_long.c exitfn.c lockfile.c \ + counters.c base_objs = $(base_sources:.c=.o) ccache_sources = main.c $(base_sources) diff --git a/ccache.h b/ccache.h index 4bf1f680f..9f3cc7c43 100644 --- a/ccache.h +++ b/ccache.h @@ -3,6 +3,7 @@ #include "config.h" #include "mdfour.h" +#include "counters.h" #include #include @@ -138,6 +139,8 @@ int stats_set_limits(long maxfiles, long maxsize); size_t value_units(const char *s); char *format_size(size_t v); void stats_set_sizes(const char *dir, size_t num_files, size_t total_size); +void stats_read(const char *path, struct counters *counters); +void stats_write(const char *path, struct counters *counters); /* ------------------------------------------------------------------------- */ /* unify.c */ diff --git a/counters.c b/counters.c new file mode 100644 index 000000000..d237522fb --- /dev/null +++ b/counters.c @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2010 Joel Rosdahl + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* A simple array of unsigned integers used for the statistics counters. */ + +#include "ccache.h" +#include + +/* + * Allocate and initialize a struct counters. Data entries up to the size are + * set to 0. + */ +struct counters * +counters_init(size_t initial_size) +{ + struct counters *c = x_malloc(sizeof(*c)); + c->data = NULL; + c->size = 0; + c->allocated = 0; + counters_resize(c, initial_size); + return c; +} + +/* + * Free a struct counters. + */ +void +counters_free(struct counters *c) +{ + free(c->data); + free(c); +} + +/* + * Set a new size. New data entries are set to 0. + */ +void +counters_resize(struct counters *c, size_t new_size) +{ + if (new_size > c->size) { + size_t i; + int realloc = 0; + + while (c->allocated < new_size) { + c->allocated += 32 + c->allocated; + realloc = 1; + } + if (realloc) { + c->data = x_realloc(c->data, c->allocated * sizeof(c->data[0])); + } + for (i = c->size; i < new_size; i++) { + c->data[i] = 0; + } + } + + c->size = new_size; +} diff --git a/counters.h b/counters.h new file mode 100644 index 000000000..455237940 --- /dev/null +++ b/counters.h @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2010 Joel Rosdahl + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef COUNTERS_H +#define COUNTERS_H + +struct counters { + unsigned *data; /* counter value */ + size_t size; /* logical array size */ + size_t allocated; /* allocated size */ +}; + +struct counters *counters_init(size_t initial_size); +void counters_resize(struct counters *c, size_t new_size); +void counters_free(struct counters *c); + +#endif diff --git a/dev.mk.in b/dev.mk.in index cd3a1c248..9aa5c2f83 100644 --- a/dev.mk.in +++ b/dev.mk.in @@ -22,7 +22,7 @@ built_dist_files = $(generated_docs) headers = \ ccache.h hashtable.h hashtable_itr.h hashtable_private.h hashutil.h \ - manifest.h mdfour.h murmurhashneutral2.h getopt_long.h \ + manifest.h mdfour.h counters.h murmurhashneutral2.h getopt_long.h \ test/framework.h test/suites.h test/util.h files_to_clean += *.tar.bz2 *.tar.gz *.xml .deps/* diff --git a/stats.c b/stats.c index 3994ebbe5..745ee568f 100644 --- a/stats.c +++ b/stats.c @@ -37,7 +37,7 @@ extern char *stats_file; extern char *cache_dir; extern unsigned lock_staleness_limit; -static unsigned counter_updates[STATS_END]; +static struct counters *counter_updates; /* default maximum cache size */ #ifndef DEFAULT_MAXSIZE @@ -94,24 +94,33 @@ display_size(size_t v) /* parse a stats file from a buffer - adding to the counters */ static void -parse_stats(unsigned counters[STATS_END], char *buf) +parse_stats(struct counters *counters, const char *buf) { - int i; - char *p, *p2; + size_t i = 0; + const char *p; + char *p2; + long val; p = buf; - for (i = 0; i < STATS_END; i++) { - counters[i] += strtol(p, &p2, 10); - if (!p2 || p2 == p) break; + while (1) { + val = strtol(p, &p2, 10); + if (p2 == p) { + break; + } + if (counters->size < i + 1) { + counters_resize(counters, i + 1); + } + counters->data[i] += val; + i++; p = p2; } } /* write out a stats file */ -static void -write_stats(const char *path, unsigned counters[STATS_END]) +void +stats_write(const char *path, struct counters *counters) { - int i; + size_t i; char *tmp_file; FILE *f; @@ -119,22 +128,33 @@ write_stats(const char *path, unsigned counters[STATS_END]) f = fopen(tmp_file, "wb"); if (!f) { cc_log("Failed to open %s", tmp_file); - return; + goto end; } - for (i = 0; i < STATS_END; i++) { - if (fprintf(f, "%u\n", counters[i]) < 0) { + for (i = 0; i < counters->size; i++) { + if (fprintf(f, "%u\n", counters->data[i]) < 0) { fatal("Failed to write to %s", tmp_file); } } fclose(f); x_rename(tmp_file, path); + +end: + free(tmp_file); } /* fill in some default stats values */ static void -stats_default(unsigned counters[STATS_END]) +stats_default(struct counters *counters) { - counters[STATS_MAXSIZE] += DEFAULT_MAXSIZE / 16; + counters->data[STATS_MAXSIZE] += DEFAULT_MAXSIZE / 16; +} + +static void +init_counter_updates(void) +{ + if (!counter_updates) { + counter_updates = counters_init(STATS_END); + } } /* @@ -144,33 +164,25 @@ stats_default(unsigned counters[STATS_END]) void stats_update_size(enum stats stat, size_t size, unsigned files) { + init_counter_updates(); if (stat != STATS_NONE) { - counter_updates[stat]++; + counter_updates->data[stat]++; } - counter_updates[STATS_NUMFILES] += files; - counter_updates[STATS_TOTALSIZE] += size; + counter_updates->data[STATS_NUMFILES] += files; + counter_updates->data[STATS_TOTALSIZE] += size; } /* Read in the stats from one directory and add to the counters. */ -static void -stats_read(const char *path, unsigned counters[STATS_END]) +void +stats_read(const char *sfile, struct counters *counters) { - int fd; - - fd = open(path, O_RDONLY|O_BINARY); - if (fd == -1) { - stats_default(counters); + char *data = read_file(sfile); + if (data) { + parse_stats(counters, data); } else { - char buf[1024]; - int len = read(fd, buf, sizeof(buf)-1); - if (len <= 0) { - stats_default(counters); - return; - } - buf[len] = 0; - parse_stats(counters, buf); - close(fd); + stats_default(counters); } + free(data); } /* @@ -179,15 +191,17 @@ stats_read(const char *path, unsigned counters[STATS_END]) void stats_flush(void) { - unsigned counters[STATS_END]; + struct counters *counters; int need_cleanup = 0; int should_flush = 0; int i; if (getenv("CCACHE_NOSTATS")) return; + init_counter_updates(); + for (i = 0; i < STATS_END; ++i) { - if (counter_updates[i] > 0) { + if (counter_updates->data[i] > 0) { should_flush = 1; break; } @@ -208,30 +222,31 @@ stats_flush(void) free(stats_dir); } - memset(counters, 0, sizeof(counters)); - if (!lockfile_acquire(stats_file, lock_staleness_limit)) { return; } + counters = counters_init(STATS_END); stats_read(stats_file, counters); for (i = 0; i < STATS_END; ++i) { - counters[i] += counter_updates[i]; + counters->data[i] += counter_updates->data[i]; } - write_stats(stats_file, counters); + stats_write(stats_file, counters); lockfile_release(stats_file); - if (counters[STATS_MAXFILES] != 0 && - counters[STATS_NUMFILES] > counters[STATS_MAXFILES]) { + if (counters->data[STATS_MAXFILES] != 0 && + counters->data[STATS_NUMFILES] > counters->data[STATS_MAXFILES]) { need_cleanup = 1; } - if (counters[STATS_MAXSIZE] != 0 && - counters[STATS_TOTALSIZE] > counters[STATS_MAXSIZE]) { + if (counters->data[STATS_MAXSIZE] != 0 && + counters->data[STATS_TOTALSIZE] > counters->data[STATS_MAXSIZE]) { need_cleanup = 1; } if (need_cleanup) { char *p = dirname(stats_file); - cleanup_dir(p, counters[STATS_MAXFILES], counters[STATS_MAXSIZE]); + cleanup_dir(p, + counters->data[STATS_MAXFILES], + counters->data[STATS_MAXSIZE]); free(p); } } @@ -243,11 +258,12 @@ stats_update(enum stats stat) stats_update_size(stat, 0, 0); } -/* Get the pending update a counter value. */ +/* Get the pending update of a counter value. */ unsigned stats_get_pending(enum stats stat) { - return counter_updates[stat]; + init_counter_updates(); + return counter_updates->data[stat]; } /* sum and display the total stats for all cache dirs */ @@ -255,9 +271,7 @@ void stats_summary(void) { int dir, i; - unsigned counters[STATS_END]; - - memset(counters, 0, sizeof(counters)); + struct counters *counters = counters_init(STATS_END); /* add up the stats in each directory */ for (dir = -1; dir <= 0xF; dir++) { @@ -274,9 +288,8 @@ stats_summary(void) /* oh what a nasty hack ... */ if (dir == -1) { - counters[STATS_MAXSIZE] = 0; + counters->data[STATS_MAXSIZE] = 0; } - } printf("cache directory %s\n", cache_dir); @@ -285,18 +298,20 @@ stats_summary(void) for (i = 0; stats_info[i].message; i++) { enum stats stat = stats_info[i].stat; - if (counters[stat] == 0 && !(stats_info[i].flags & FLAG_ALWAYS)) { + if (counters->data[stat] == 0 && !(stats_info[i].flags & FLAG_ALWAYS)) { continue; } printf("%s ", stats_info[i].message); if (stats_info[i].fn) { - stats_info[i].fn(counters[stat]); + stats_info[i].fn(counters->data[stat]); printf("\n"); } else { - printf("%8u\n", counters[stat]); + printf("%8u\n", counters->data[stat]); } } + + counters_free(counters); } /* zero all the stats structures */ @@ -306,27 +321,25 @@ stats_zero(void) int dir; unsigned i; char *fname; - unsigned counters[STATS_END]; fname = format("%s/stats", cache_dir); unlink(fname); free(fname); for (dir = 0; dir <= 0xF; dir++) { + struct counters *counters = counters_init(STATS_END); fname = format("%s/%1x/stats", cache_dir, dir); - memset(counters, 0, sizeof(counters)); - if (!lockfile_acquire(fname, lock_staleness_limit)) { - free(fname); - continue; - } - stats_read(fname, counters); - for (i = 0; stats_info[i].message; i++) { - if (!(stats_info[i].flags & FLAG_NOZERO)) { - counters[stats_info[i].stat] = 0; + if (lockfile_acquire(fname, lock_staleness_limit)) { + stats_read(fname, counters); + for (i = 0; stats_info[i].message; i++) { + if (!(stats_info[i].flags & FLAG_NOZERO)) { + counters->data[stats_info[i].stat] = 0; + } } + stats_write(fname, counters); + lockfile_release(fname); } - write_stats(fname, counters); - lockfile_release(fname); + counters_free(counters); free(fname); } } @@ -335,13 +348,13 @@ stats_zero(void) void stats_get_limits(const char *dir, unsigned *maxfiles, unsigned *maxsize) { - unsigned counters[STATS_END]; + struct counters *counters = counters_init(STATS_END); char *sname = format("%s/stats", dir); - memset(counters, 0, sizeof(counters)); stats_read(sname, counters); + *maxfiles = counters->data[STATS_MAXFILES]; + *maxsize = counters->data[STATS_MAXSIZE]; free(sname); - *maxfiles = counters[STATS_MAXFILES]; - *maxsize = counters[STATS_MAXSIZE]; + counters_free(counters); } /* set the per directory limits */ @@ -349,7 +362,6 @@ int stats_set_limits(long maxfiles, long maxsize) { int dir; - unsigned counters[STATS_END]; if (maxfiles != -1) { maxfiles /= 16; @@ -368,25 +380,25 @@ stats_set_limits(long maxfiles, long maxsize) cdir = format("%s/%1x", cache_dir, dir); if (create_dir(cdir) != 0) { + free(cdir); return 1; } fname = format("%s/stats", cdir); free(cdir); - if (!lockfile_acquire(fname, lock_staleness_limit)) { - free(fname); - continue; - } - memset(counters, 0, sizeof(counters)); - stats_read(fname, counters); - if (maxfiles != -1) { - counters[STATS_MAXFILES] = maxfiles; - } - if (maxsize != -1) { - counters[STATS_MAXSIZE] = maxsize; + if (lockfile_acquire(fname, lock_staleness_limit)) { + struct counters *counters = counters_init(STATS_END); + stats_read(fname, counters); + if (maxfiles != -1) { + counters->data[STATS_MAXFILES] = maxfiles; + } + if (maxsize != -1) { + counters->data[STATS_MAXSIZE] = maxsize; + } + stats_write(fname, counters); + lockfile_release(fname); + counters_free(counters); } - write_stats(fname, counters); - lockfile_release(fname); free(fname); } @@ -397,22 +409,19 @@ stats_set_limits(long maxfiles, long maxsize) void stats_set_sizes(const char *dir, size_t num_files, size_t total_size) { - unsigned counters[STATS_END]; + struct counters *counters = counters_init(STATS_END); char *statsfile; create_dir(dir); statsfile = format("%s/stats", dir); - memset(counters, 0, sizeof(counters)); - - if (!lockfile_acquire(statsfile, lock_staleness_limit)) { - free(statsfile); - return; + if (lockfile_acquire(statsfile, lock_staleness_limit)) { + stats_read(statsfile, counters); + counters->data[STATS_NUMFILES] = num_files; + counters->data[STATS_TOTALSIZE] = total_size; + stats_write(statsfile, counters); + lockfile_release(statsfile); } - stats_read(statsfile, counters); - counters[STATS_NUMFILES] = num_files; - counters[STATS_TOTALSIZE] = total_size; - write_stats(statsfile, counters); - lockfile_release(statsfile); free(statsfile); + counters_free(counters); } diff --git a/test/test_counters.c b/test/test_counters.c new file mode 100644 index 000000000..a00d1ee8f --- /dev/null +++ b/test/test_counters.c @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2010 Joel Rosdahl + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "ccache.h" +#include "counters.h" +#include "test/framework.h" +#include "test/util.h" + +TEST_SUITE(counters) + +TEST(counters_init_0_should_allocate_0) +{ + struct counters *counters = counters_init(0); + + CHECK_UNS_EQ(0, counters->allocated); + CHECK_UNS_EQ(0, counters->size); + + counters_free(counters); +} + +TEST(counters_init_7_should_allocate_32) +{ + int i; + struct counters *counters = counters_init(7); + + CHECK_UNS_EQ(32, counters->allocated); + CHECK_UNS_EQ(7, counters->size); + for (i = 0; i < 7; i++) { + CHECK_UNS_EQ(0, counters->data[i]); + } + + counters_free(counters); +} + +TEST(counters_resize_50_should_allocate_96) +{ + struct counters *counters = counters_init(0); + + CHECK_UNS_EQ(0, counters->allocated); + counters_resize(counters, 50); + CHECK_UNS_EQ(50, counters->size); + CHECK_UNS_EQ(96, counters->allocated); + + counters_free(counters); +} + +TEST_SUITE_END diff --git a/test/test_stats.c b/test/test_stats.c new file mode 100644 index 000000000..acf6a3423 --- /dev/null +++ b/test/test_stats.c @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2010 Joel Rosdahl + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* + * This file contains tests for statistics handling. + */ + +#include "ccache.h" +#include "counters.h" +#include "test/framework.h" +#include "test/util.h" + +TEST_SUITE(stats) + +TEST(forward_compatibility) +{ + unsigned i; + FILE *f; + struct counters *counters = counters_init(0); + + f = fopen("stats", "w"); + for (i = 0; i < 100; i++) { + fprintf(f, "%u\n", i); + } + fclose(f); + + stats_read("stats", counters); + CHECK_UNS_EQ(100, counters->size); + CHECK_UNS_EQ(73, counters->data[73]); + + stats_write("stats", counters); + CHECK_UNS_EQ(100, counters->size); + CHECK_UNS_EQ(99, counters->data[99]); + + counters_free(counters); +} + +TEST_SUITE_END