Make reading and writing of statistics counters forward-compatible

Previously, ccache threw away counters values it didn't know about, i.e., new
counters coming from later ccache versions.

diff --git a/Makefile.in b/Makefile.in
index e8ae7beca8ab92014b666b1f666893ab2ccdbbdd..ef258480068493c544caf463378d35ece7cd1d4d 100644 (file)
--- a/Makefile.in
+++ b/Makefile.in
@@ -21,7 +21,8 @@ libs = @LIBS@ -lm -lz
 base_sources = \
     ccache.c mdfour.c hash.c execute.c util.c args.c stats.c version.c \
     cleanup.c snprintf.c unify.c manifest.c hashtable.c hashtable_itr.c \
-    murmurhashneutral2.c hashutil.c getopt_long.c exitfn.c lockfile.c
+    murmurhashneutral2.c hashutil.c getopt_long.c exitfn.c lockfile.c \
+    counters.c
 base_objs = $(base_sources:.c=.o)
 
 ccache_sources = main.c $(base_sources)

diff --git a/ccache.h b/ccache.h
index 4bf1f680fd7f5d7a2335cb5acc1bc69e2ae048c4..9f3cc7c43df1b1e8bcd600cf4472a7b44c484910 100644 (file)
--- a/ccache.h
+++ b/ccache.h
@@ -3,6 +3,7 @@
 
 #include "config.h"
 #include "mdfour.h"
+#include "counters.h"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -138,6 +139,8 @@ int stats_set_limits(long maxfiles, long maxsize);
 size_t value_units(const char *s);
 char *format_size(size_t v);
 void stats_set_sizes(const char *dir, size_t num_files, size_t total_size);
+void stats_read(const char *path, struct counters *counters);
+void stats_write(const char *path, struct counters *counters);
 
 /* ------------------------------------------------------------------------- */
 /* unify.c */

diff --git a/counters.c b/counters.c
new file mode 100644 (file)
index 0000000..d237522
--- /dev/null
+++ b/counters.c
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2010 Joel Rosdahl
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* A simple array of unsigned integers used for the statistics counters. */
+
+#include "ccache.h"
+#include <stdlib.h>
+
+/*
+ * Allocate and initialize a struct counters. Data entries up to the size are
+ * set to 0.
+ */
+struct counters *
+counters_init(size_t initial_size)
+{
+       struct counters *c = x_malloc(sizeof(*c));
+       c->data = NULL;
+       c->size = 0;
+       c->allocated = 0;
+       counters_resize(c, initial_size);
+       return c;
+}
+
+/*
+ * Free a struct counters.
+ */
+void
+counters_free(struct counters *c)
+{
+       free(c->data);
+       free(c);
+}
+
+/*
+ * Set a new size. New data entries are set to 0.
+ */
+void
+counters_resize(struct counters *c, size_t new_size)
+{
+       if (new_size > c->size) {
+               size_t i;
+               int realloc = 0;
+
+               while (c->allocated < new_size) {
+                       c->allocated += 32 + c->allocated;
+                       realloc = 1;
+               }
+               if (realloc) {
+                       c->data = x_realloc(c->data, c->allocated * sizeof(c->data[0]));
+               }
+               for (i = c->size; i < new_size; i++) {
+                       c->data[i] = 0;
+               }
+       }
+
+       c->size = new_size;
+}

diff --git a/counters.h b/counters.h
new file mode 100644 (file)
index 0000000..4552379
--- /dev/null
+++ b/counters.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2010 Joel Rosdahl
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef COUNTERS_H
+#define COUNTERS_H
+
+struct counters {
+       unsigned *data;   /* counter value */
+       size_t size;      /* logical array size */
+       size_t allocated; /* allocated size */
+};
+
+struct counters *counters_init(size_t initial_size);
+void counters_resize(struct counters *c, size_t new_size);
+void counters_free(struct counters *c);
+
+#endif

diff --git a/dev.mk.in b/dev.mk.in
index cd3a1c248daec49f1fc2e67be402ccac46a37ff6..9aa5c2f830d4dc04213414167ed58d0788523091 100644 (file)
--- a/dev.mk.in
+++ b/dev.mk.in
@@ -22,7 +22,7 @@ built_dist_files = $(generated_docs)
 
 headers = \
     ccache.h hashtable.h hashtable_itr.h hashtable_private.h hashutil.h \
-    manifest.h mdfour.h murmurhashneutral2.h getopt_long.h \
+    manifest.h mdfour.h counters.h murmurhashneutral2.h getopt_long.h \
     test/framework.h test/suites.h test/util.h
 
 files_to_clean += *.tar.bz2 *.tar.gz *.xml .deps/*

diff --git a/stats.c b/stats.c
index 3994ebbe5cebf25a0c382d18218ee01a8c5eef3f..745ee568f01fc2e345595d69cec26b85c108cad8 100644 (file)
--- a/stats.c
+++ b/stats.c
@@ -37,7 +37,7 @@ extern char *stats_file;
 extern char *cache_dir;
 extern unsigned lock_staleness_limit;
 
-static unsigned counter_updates[STATS_END];
+static struct counters *counter_updates;
 
 /* default maximum cache size */
 #ifndef DEFAULT_MAXSIZE
@@ -94,24 +94,33 @@ display_size(size_t v)
 
 /* parse a stats file from a buffer - adding to the counters */
 static void
-parse_stats(unsigned counters[STATS_END], char *buf)
+parse_stats(struct counters *counters, const char *buf)
 {
-       int i;
-       char *p, *p2;
+       size_t i = 0;
+       const char *p;
+       char *p2;
+       long val;
 
        p = buf;
-       for (i = 0; i < STATS_END; i++) {
-               counters[i] += strtol(p, &p2, 10);
-               if (!p2 || p2 == p) break;
+       while (1) {
+               val = strtol(p, &p2, 10);
+               if (p2 == p) {
+                       break;
+               }
+               if (counters->size < i + 1) {
+                       counters_resize(counters, i + 1);
+               }
+               counters->data[i] += val;
+               i++;
                p = p2;
        }
 }
 
 /* write out a stats file */
-static void
-write_stats(const char *path, unsigned counters[STATS_END])
+void
+stats_write(const char *path, struct counters *counters)
 {
-       int i;
+       size_t i;
        char *tmp_file;
        FILE *f;
 
@@ -119,22 +128,33 @@ write_stats(const char *path, unsigned counters[STATS_END])
        f = fopen(tmp_file, "wb");
        if (!f) {
                cc_log("Failed to open %s", tmp_file);
-               return;
+               goto end;
        }
-       for (i = 0; i < STATS_END; i++) {
-               if (fprintf(f, "%u\n", counters[i]) < 0) {
+       for (i = 0; i < counters->size; i++) {
+               if (fprintf(f, "%u\n", counters->data[i]) < 0) {
                        fatal("Failed to write to %s", tmp_file);
                }
        }
        fclose(f);
        x_rename(tmp_file, path);
+
+end:
+       free(tmp_file);
 }
 
 /* fill in some default stats values */
 static void
-stats_default(unsigned counters[STATS_END])
+stats_default(struct counters *counters)
 {
-       counters[STATS_MAXSIZE] += DEFAULT_MAXSIZE / 16;
+       counters->data[STATS_MAXSIZE] += DEFAULT_MAXSIZE / 16;
+}
+
+static void
+init_counter_updates(void)
+{
+       if (!counter_updates) {
+               counter_updates = counters_init(STATS_END);
+       }
 }
 
 /*
@@ -144,33 +164,25 @@ stats_default(unsigned counters[STATS_END])
 void
 stats_update_size(enum stats stat, size_t size, unsigned files)
 {
+       init_counter_updates();
        if (stat != STATS_NONE) {
-               counter_updates[stat]++;
+               counter_updates->data[stat]++;
        }
-       counter_updates[STATS_NUMFILES] += files;
-       counter_updates[STATS_TOTALSIZE] += size;
+       counter_updates->data[STATS_NUMFILES] += files;
+       counter_updates->data[STATS_TOTALSIZE] += size;
 }
 
 /* Read in the stats from one directory and add to the counters. */
-static void
-stats_read(const char *path, unsigned counters[STATS_END])
+void
+stats_read(const char *sfile, struct counters *counters)
 {
-       int fd;
-
-       fd = open(path, O_RDONLY|O_BINARY);
-       if (fd == -1) {
-               stats_default(counters);
+       char *data = read_file(sfile);
+       if (data) {
+               parse_stats(counters, data);
        } else {
-               char buf[1024];
-               int len = read(fd, buf, sizeof(buf)-1);
-               if (len <= 0) {
-                       stats_default(counters);
-                       return;
-               }
-               buf[len] = 0;
-               parse_stats(counters, buf);
-               close(fd);
+               stats_default(counters);
        }
+       free(data);
 }
 
 /*
@@ -179,15 +191,17 @@ stats_read(const char *path, unsigned counters[STATS_END])
 void
 stats_flush(void)
 {
-       unsigned counters[STATS_END];
+       struct counters *counters;
        int need_cleanup = 0;
        int should_flush = 0;
        int i;
 
        if (getenv("CCACHE_NOSTATS")) return;
 
+       init_counter_updates();
+
        for (i = 0; i < STATS_END; ++i) {
-               if (counter_updates[i] > 0) {
+               if (counter_updates->data[i] > 0) {
                        should_flush = 1;
                        break;
                }
@@ -208,30 +222,31 @@ stats_flush(void)
                free(stats_dir);
        }
 
-       memset(counters, 0, sizeof(counters));
-
        if (!lockfile_acquire(stats_file, lock_staleness_limit)) {
                return;
        }
+       counters = counters_init(STATS_END);
        stats_read(stats_file, counters);
        for (i = 0; i < STATS_END; ++i) {
-               counters[i] += counter_updates[i];
+               counters->data[i] += counter_updates->data[i];
        }
-       write_stats(stats_file, counters);
+       stats_write(stats_file, counters);
        lockfile_release(stats_file);
 
-       if (counters[STATS_MAXFILES] != 0 &&
-           counters[STATS_NUMFILES] > counters[STATS_MAXFILES]) {
+       if (counters->data[STATS_MAXFILES] != 0 &&
+           counters->data[STATS_NUMFILES] > counters->data[STATS_MAXFILES]) {
                need_cleanup = 1;
        }
-       if (counters[STATS_MAXSIZE] != 0 &&
-           counters[STATS_TOTALSIZE] > counters[STATS_MAXSIZE]) {
+       if (counters->data[STATS_MAXSIZE] != 0 &&
+           counters->data[STATS_TOTALSIZE] > counters->data[STATS_MAXSIZE]) {
                need_cleanup = 1;
        }
 
        if (need_cleanup) {
                char *p = dirname(stats_file);
-               cleanup_dir(p, counters[STATS_MAXFILES], counters[STATS_MAXSIZE]);
+               cleanup_dir(p,
+                           counters->data[STATS_MAXFILES],
+                           counters->data[STATS_MAXSIZE]);
                free(p);
        }
 }
@@ -243,11 +258,12 @@ stats_update(enum stats stat)
        stats_update_size(stat, 0, 0);
 }
 
-/* Get the pending update a counter value. */
+/* Get the pending update of a counter value. */
 unsigned
 stats_get_pending(enum stats stat)
 {
-       return counter_updates[stat];
+       init_counter_updates();
+       return counter_updates->data[stat];
 }
 
 /* sum and display the total stats for all cache dirs */
@@ -255,9 +271,7 @@ void
 stats_summary(void)
 {
        int dir, i;
-       unsigned counters[STATS_END];
-
-       memset(counters, 0, sizeof(counters));
+       struct counters *counters = counters_init(STATS_END);
 
        /* add up the stats in each directory */
        for (dir = -1; dir <= 0xF; dir++) {
@@ -274,9 +288,8 @@ stats_summary(void)
 
                /* oh what a nasty hack ... */
                if (dir == -1) {
-                       counters[STATS_MAXSIZE] = 0;
+                       counters->data[STATS_MAXSIZE] = 0;
                }
-
        }
 
        printf("cache directory                     %s\n", cache_dir);
@@ -285,18 +298,20 @@ stats_summary(void)
        for (i = 0; stats_info[i].message; i++) {
                enum stats stat = stats_info[i].stat;
 
-               if (counters[stat] == 0 && !(stats_info[i].flags & FLAG_ALWAYS)) {
+               if (counters->data[stat] == 0 && !(stats_info[i].flags & FLAG_ALWAYS)) {
                        continue;
                }
 
                printf("%s ", stats_info[i].message);
                if (stats_info[i].fn) {
-                       stats_info[i].fn(counters[stat]);
+                       stats_info[i].fn(counters->data[stat]);
                        printf("\n");
                } else {
-                       printf("%8u\n", counters[stat]);
+                       printf("%8u\n", counters->data[stat]);
                }
        }
+
+       counters_free(counters);
 }
 
 /* zero all the stats structures */
@@ -306,27 +321,25 @@ stats_zero(void)
        int dir;
        unsigned i;
        char *fname;
-       unsigned counters[STATS_END];
 
        fname = format("%s/stats", cache_dir);
        unlink(fname);
        free(fname);
 
        for (dir = 0; dir <= 0xF; dir++) {
+               struct counters *counters = counters_init(STATS_END);
                fname = format("%s/%1x/stats", cache_dir, dir);
-               memset(counters, 0, sizeof(counters));
-               if (!lockfile_acquire(fname, lock_staleness_limit)) {
-                       free(fname);
-                       continue;
-               }
-               stats_read(fname, counters);
-               for (i = 0; stats_info[i].message; i++) {
-                       if (!(stats_info[i].flags & FLAG_NOZERO)) {
-                               counters[stats_info[i].stat] = 0;
+               if (lockfile_acquire(fname, lock_staleness_limit)) {
+                       stats_read(fname, counters);
+                       for (i = 0; stats_info[i].message; i++) {
+                               if (!(stats_info[i].flags & FLAG_NOZERO)) {
+                                       counters->data[stats_info[i].stat] = 0;
+                               }
                        }
+                       stats_write(fname, counters);
+                       lockfile_release(fname);
                }
-               write_stats(fname, counters);
-               lockfile_release(fname);
+               counters_free(counters);
                free(fname);
        }
 }
@@ -335,13 +348,13 @@ stats_zero(void)
 void
 stats_get_limits(const char *dir, unsigned *maxfiles, unsigned *maxsize)
 {
-       unsigned counters[STATS_END];
+       struct counters *counters = counters_init(STATS_END);
        char *sname = format("%s/stats", dir);
-       memset(counters, 0, sizeof(counters));
        stats_read(sname, counters);
+       *maxfiles = counters->data[STATS_MAXFILES];
+       *maxsize = counters->data[STATS_MAXSIZE];
        free(sname);
-       *maxfiles = counters[STATS_MAXFILES];
-       *maxsize = counters[STATS_MAXSIZE];
+       counters_free(counters);
 }
 
 /* set the per directory limits */
@@ -349,7 +362,6 @@ int
 stats_set_limits(long maxfiles, long maxsize)
 {
        int dir;
-       unsigned counters[STATS_END];
 
        if (maxfiles != -1) {
                maxfiles /= 16;
@@ -368,25 +380,25 @@ stats_set_limits(long maxfiles, long maxsize)
 
                cdir = format("%s/%1x", cache_dir, dir);
                if (create_dir(cdir) != 0) {
+                       free(cdir);
                        return 1;
                }
                fname = format("%s/stats", cdir);
                free(cdir);
 
-               if (!lockfile_acquire(fname, lock_staleness_limit)) {
-                       free(fname);
-                       continue;
-               }
-               memset(counters, 0, sizeof(counters));
-               stats_read(fname, counters);
-               if (maxfiles != -1) {
-                       counters[STATS_MAXFILES] = maxfiles;
-               }
-               if (maxsize != -1) {
-                       counters[STATS_MAXSIZE] = maxsize;
+               if (lockfile_acquire(fname, lock_staleness_limit)) {
+                       struct counters *counters = counters_init(STATS_END);
+                       stats_read(fname, counters);
+                       if (maxfiles != -1) {
+                               counters->data[STATS_MAXFILES] = maxfiles;
+                       }
+                       if (maxsize != -1) {
+                               counters->data[STATS_MAXSIZE] = maxsize;
+                       }
+                       stats_write(fname, counters);
+                       lockfile_release(fname);
+                       counters_free(counters);
                }
-               write_stats(fname, counters);
-               lockfile_release(fname);
                free(fname);
        }
 
@@ -397,22 +409,19 @@ stats_set_limits(long maxfiles, long maxsize)
 void
 stats_set_sizes(const char *dir, size_t num_files, size_t total_size)
 {
-       unsigned counters[STATS_END];
+       struct counters *counters = counters_init(STATS_END);
        char *statsfile;
 
        create_dir(dir);
        statsfile = format("%s/stats", dir);
 
-       memset(counters, 0, sizeof(counters));
-
-       if (!lockfile_acquire(statsfile, lock_staleness_limit)) {
-               free(statsfile);
-               return;
+       if (lockfile_acquire(statsfile, lock_staleness_limit)) {
+               stats_read(statsfile, counters);
+               counters->data[STATS_NUMFILES] = num_files;
+               counters->data[STATS_TOTALSIZE] = total_size;
+               stats_write(statsfile, counters);
+               lockfile_release(statsfile);
        }
-       stats_read(statsfile, counters);
-       counters[STATS_NUMFILES] = num_files;
-       counters[STATS_TOTALSIZE] = total_size;
-       write_stats(statsfile, counters);
-       lockfile_release(statsfile);
        free(statsfile);
+       counters_free(counters);
 }

diff --git a/test/test_counters.c b/test/test_counters.c
new file mode 100644 (file)
index 0000000..a00d1ee
--- /dev/null
+++ b/test/test_counters.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2010 Joel Rosdahl
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "ccache.h"
+#include "counters.h"
+#include "test/framework.h"
+#include "test/util.h"
+
+TEST_SUITE(counters)
+
+TEST(counters_init_0_should_allocate_0)
+{
+       struct counters *counters = counters_init(0);
+
+       CHECK_UNS_EQ(0, counters->allocated);
+       CHECK_UNS_EQ(0, counters->size);
+
+       counters_free(counters);
+}
+
+TEST(counters_init_7_should_allocate_32)
+{
+       int i;
+       struct counters *counters = counters_init(7);
+
+       CHECK_UNS_EQ(32, counters->allocated);
+       CHECK_UNS_EQ(7, counters->size);
+       for (i = 0; i < 7; i++) {
+               CHECK_UNS_EQ(0, counters->data[i]);
+       }
+
+       counters_free(counters);
+}
+
+TEST(counters_resize_50_should_allocate_96)
+{
+       struct counters *counters = counters_init(0);
+
+       CHECK_UNS_EQ(0, counters->allocated);
+       counters_resize(counters, 50);
+       CHECK_UNS_EQ(50, counters->size);
+       CHECK_UNS_EQ(96, counters->allocated);
+
+       counters_free(counters);
+}
+
+TEST_SUITE_END

diff --git a/test/test_stats.c b/test/test_stats.c
new file mode 100644 (file)
index 0000000..acf6a34
--- /dev/null
+++ b/test/test_stats.c
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2010 Joel Rosdahl
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/*
+ * This file contains tests for statistics handling.
+ */
+
+#include "ccache.h"
+#include "counters.h"
+#include "test/framework.h"
+#include "test/util.h"
+
+TEST_SUITE(stats)
+
+TEST(forward_compatibility)
+{
+       unsigned i;
+       FILE *f;
+       struct counters *counters = counters_init(0);
+
+       f = fopen("stats", "w");
+       for (i = 0; i < 100; i++) {
+               fprintf(f, "%u\n", i);
+       }
+       fclose(f);
+
+       stats_read("stats", counters);
+       CHECK_UNS_EQ(100, counters->size);
+       CHECK_UNS_EQ(73, counters->data[73]);
+
+       stats_write("stats", counters);
+       CHECK_UNS_EQ(100, counters->size);
+       CHECK_UNS_EQ(99, counters->data[99]);
+
+       counters_free(counters);
+}
+
+TEST_SUITE_END