stats: Implement group_by

group_by allows dynamic creation of sub-metrics for metric.

Example:

metric imap_command {
   event_name = imap_command_finished
   group_by = cmd_name tagged_reply_state
}

will generate following metrics

imap_command_select_ok
imap_command_select_bad
imap_command_select_no

etc.

based on what sort of events come in. new metrics are generated
when event comes in and then they are tracked.

only strings can be used for group_by for now.

diff --git a/src/stats/stats-metrics.c b/src/stats/stats-metrics.c
index 23a2072240df8c562015333538952cc05b390d9e..d1f0a4a92aa3d384700819f2484041e26be24e86 100644 (file)
--- a/src/stats/stats-metrics.c
+++ b/src/stats/stats-metrics.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "array.h"
+#include "str.h"
 #include "stats-dist.h"
 #include "time-util.h"
 #include "event-filter.h"
@@ -9,6 +10,8 @@
 #include "stats-settings.h"
 #include "stats-metrics.h"
 
+#include <ctype.h>
+
 struct stats_metrics {
        pool_t pool;
        struct event_filter *stats_filter; /* stats-only */
@@ -18,6 +21,28 @@ struct stats_metrics {
        ARRAY(struct metric *) metrics;
 };
 
+static void
+stats_metric_event(struct metric *metric, struct event *event, pool_t pool);
+static struct metric *
+stats_metric_sub_metric_alloc(struct metric *metric, const char *name, pool_t pool);
+
+/* This does not need to be unique as it's a display name */
+static const char *sub_metric_name_create(pool_t pool, const char *name)
+{
+       string_t *sub_name = str_new(pool, 32);
+       /* use up to 32 bytes */
+       for (const char *p = name; *p != '\0' && sub_name->used < 32;
+           p++) {
+               char c = *p;
+               if (!i_isalnum(c))
+                       c = '_';
+               else
+                       c = i_tolower(c);
+               str_append_c(sub_name, c);
+       }
+       return str_c(sub_name);
+}
+
 static void
 stats_metric_settings_to_query(const struct stats_metric_settings *set,
                               struct event_filter_query *query_r)
@@ -127,6 +152,10 @@ static void stats_metrics_add_set(struct stats_metrics *metrics,
 
        fields = t_strsplit_spaces(set->fields, " ");
        metric = stats_metric_alloc(metrics->pool, set->name, fields);
+
+       if (*set->group_by != '\0')
+               metric->group_by = (const char *const *)p_strsplit_spaces(metrics->pool, set->group_by, " ");
+
        array_push_back(&metrics->metrics, &metric);
 
        stats_metric_settings_to_query(set, &query);
@@ -281,8 +310,97 @@ stats_metrics_get_event_filter(struct stats_metrics *metrics)
        return metrics->combined_filter;
 }
 
+static struct metric *
+stats_metric_get_sub_metric(struct metric *metric,
+                           const struct metric_value *value)
+{
+       struct metric *const *sub_metrics;
+
+       /* lookup sub-metric */
+       array_foreach (&metric->sub_metrics, sub_metrics) {
+               if ((*sub_metrics)->group_value.type == METRIC_VALUE_TYPE_STR &&
+                   memcmp((*sub_metrics)->group_value.hash, value->hash,
+                          SHA1_RESULTLEN) == 0)
+                       return *sub_metrics;
+               else if ((*sub_metrics)->group_value.type == METRIC_VALUE_TYPE_INT &&
+                   (*sub_metrics)->group_value.intmax == value->intmax)
+                       return *sub_metrics;
+       }
+       return NULL;
+}
+
+static struct metric *
+stats_metric_sub_metric_alloc(struct metric *metric, const char *name, pool_t pool)
+{
+       struct metric *sub_metric;
+       ARRAY_TYPE(const_string) fields;
+       t_array_init(&fields, metric->fields_count);
+       for (unsigned int i = 0; i < metric->fields_count; i++)
+               array_append(&fields, &metric->fields[i].field_key, 1);
+       array_append_zero(&fields);
+       sub_metric = stats_metric_alloc(pool, metric->name,
+                                       array_idx(&fields, 0));
+       sub_metric->sub_name = sub_metric_name_create(pool, name);
+       array_append(&metric->sub_metrics, &sub_metric, 1);
+       return sub_metric;
+}
+
+static void
+stats_metric_group_by(struct metric *metric, struct event *event, pool_t pool)
+{
+       struct metric *sub_metric;
+       const char *const *group = metric->group_by;
+       const struct event_field *field =
+               event_find_field(event, *group);
+       struct metric_value value;
+
+       /* ignore missing field */
+       if (field == NULL)
+               return;
+       switch (field->value_type) {
+       case EVENT_FIELD_VALUE_TYPE_STR:
+               value.type = METRIC_VALUE_TYPE_STR;
+               /* use sha1 of value to avoid excessive memory usage in case the
+                  actual value is quite long */
+               sha1_get_digest(field->value.str, strlen(field->value.str),
+                               value.hash);
+               break;
+       case EVENT_FIELD_VALUE_TYPE_INTMAX:
+               value.type = METRIC_VALUE_TYPE_INT;
+               value.intmax = field->value.intmax;
+               break;
+       case EVENT_FIELD_VALUE_TYPE_TIMEVAL:
+               return;
+       }
+
+       if (!array_is_created(&metric->sub_metrics))
+               p_array_init(&metric->sub_metrics, pool, 8);
+
+       sub_metric = stats_metric_get_sub_metric(metric, &value);
+
+       if (sub_metric == NULL) T_BEGIN {
+               const char *value_label;
+               if (value.type == METRIC_VALUE_TYPE_STR)
+                       value_label = field->value.str;
+               else if (value.type == METRIC_VALUE_TYPE_INT)
+                       value_label = dec2str(field->value.intmax);
+               else
+                       i_unreached();
+               sub_metric = stats_metric_sub_metric_alloc(metric, value_label,
+                                                          pool);
+               if (group[1] != NULL)
+                       sub_metric->group_by = group+1;
+               sub_metric->group_value.intmax = value.intmax;
+               memcpy(sub_metric->group_value.hash, value.hash, SHA1_RESULTLEN);
+       } T_END;
+
+       /* sub-metrics are recursive, so each sub-metric can have additional
+          sub-metrics. */
+       stats_metric_event(sub_metric, event, pool);
+}
+
 static void
-stats_metric_event(struct metric *metric, struct event *event)
+stats_metric_event(struct metric *metric, struct event *event, pool_t pool)
 {
        intmax_t duration;
 
@@ -309,6 +427,9 @@ stats_metric_event(struct metric *metric, struct event *event)
                }
                stats_dist_add(metric->fields[i].stats, num);
        }
+
+       if (metric->group_by != NULL)
+               stats_metric_group_by(metric, event, pool);
 }
 
 static void
@@ -343,7 +464,7 @@ void stats_metrics_event(struct stats_metrics *metrics, struct event *event,
        /* process stats */
        iter = event_filter_match_iter_init(metrics->stats_filter, event, ctx);
        while ((metric = event_filter_match_iter_next(iter)) != NULL) T_BEGIN {
-               stats_metric_event(metric, event);
+               stats_metric_event(metric, event, metrics->pool);
        } T_END;
        event_filter_match_iter_deinit(&iter);
 

diff --git a/src/stats/stats-metrics.h b/src/stats/stats-metrics.h
index 685aba6fe9dfea258b227f2108ef8a4ffe231353..6ec100d1744590d4c1556c35d20ec6633d6d9e90 100644 (file)
--- a/src/stats/stats-metrics.h
+++ b/src/stats/stats-metrics.h
@@ -2,6 +2,7 @@
 #define STATS_METRICS_H
 
 #include "stats-settings.h"
+#include "sha1.h"
 
 struct metric;
 
@@ -51,6 +52,17 @@ struct metric_field {
        struct stats_dist *stats;
 };
 
+enum metric_value_type {
+       METRIC_VALUE_TYPE_STR,
+       METRIC_VALUE_TYPE_INT,
+};
+
+struct metric_value {
+       enum metric_value_type type;
+       unsigned char hash[SHA1_RESULTLEN];
+       intmax_t intmax;
+};
+
 struct metric {
        const char *name;
        /* When this metric is a sub-metric, then this is the
@@ -75,6 +87,9 @@ struct metric {
 
        unsigned int fields_count;
        struct metric_field *fields;
+
+       const char *const *group_by;
+       struct metric_value group_value;
        ARRAY(struct metric *) sub_metrics;
 
        struct metric_export_info export_info;

diff --git a/src/stats/stats-settings.c b/src/stats/stats-settings.c
index 61a77a2d89af72fd675afb74da3ba2eed569268a..306ce2ee693e0ddc5015007d765707b5f801752e 100644 (file)
--- a/src/stats/stats-settings.c
+++ b/src/stats/stats-settings.c
@@ -102,6 +102,7 @@ static const struct setting_define stats_metric_setting_defines[] = {
        DEF(SET_STR, source_location),
        DEF(SET_STR, categories),
        DEF(SET_STR, fields),
+       DEF(SET_STR, group_by),
        { SET_STRLIST, "filter", offsetof(struct stats_metric_settings, filter), NULL },
        DEF(SET_STR, exporter),
        DEF(SET_STR, exporter_include),
@@ -115,6 +116,7 @@ static const struct stats_metric_settings stats_metric_default_settings = {
        .categories = "",
        .fields = "",
        .exporter = "",
+       .group_by = "",
        .exporter_include = "name hostname timestamps categories fields",
 };
 

diff --git a/src/stats/stats-settings.h b/src/stats/stats-settings.h
index 64ff7b83cde10883cbec5fd1c6d3b33a917cf065..00086a5863f132beafe136db871a8d1fe4ed7193 100644 (file)
--- a/src/stats/stats-settings.h
+++ b/src/stats/stats-settings.h
@@ -75,6 +75,7 @@ struct stats_metric_settings {
        const char *source_location;
        const char *categories;
        const char *fields;
+       const char *group_by;
        ARRAY(const char *) filter;
 
        unsigned int parsed_source_linenum;