From: Aki Tuomi Date: Sun, 1 Dec 2019 14:06:25 +0000 (+0200) Subject: stats: Implement group_by X-Git-Tag: 2.3.10~176 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ec64f5bf4b17dc6a608ad69845cdc7132450aea3;p=thirdparty%2Fdovecot%2Fcore.git stats: Implement group_by group_by allows dynamic creation of sub-metrics for metric. Example: metric imap_command { event_name = imap_command_finished group_by = cmd_name tagged_reply_state } will generate following metrics imap_command_select_ok imap_command_select_bad imap_command_select_no etc. based on what sort of events come in. new metrics are generated when event comes in and then they are tracked. only strings can be used for group_by for now. --- diff --git a/src/stats/stats-metrics.c b/src/stats/stats-metrics.c index 23a2072240..d1f0a4a92a 100644 --- a/src/stats/stats-metrics.c +++ b/src/stats/stats-metrics.c @@ -2,6 +2,7 @@ #include "lib.h" #include "array.h" +#include "str.h" #include "stats-dist.h" #include "time-util.h" #include "event-filter.h" @@ -9,6 +10,8 @@ #include "stats-settings.h" #include "stats-metrics.h" +#include + struct stats_metrics { pool_t pool; struct event_filter *stats_filter; /* stats-only */ @@ -18,6 +21,28 @@ struct stats_metrics { ARRAY(struct metric *) metrics; }; +static void +stats_metric_event(struct metric *metric, struct event *event, pool_t pool); +static struct metric * +stats_metric_sub_metric_alloc(struct metric *metric, const char *name, pool_t pool); + +/* This does not need to be unique as it's a display name */ +static const char *sub_metric_name_create(pool_t pool, const char *name) +{ + string_t *sub_name = str_new(pool, 32); + /* use up to 32 bytes */ + for (const char *p = name; *p != '\0' && sub_name->used < 32; + p++) { + char c = *p; + if (!i_isalnum(c)) + c = '_'; + else + c = i_tolower(c); + str_append_c(sub_name, c); + } + return str_c(sub_name); +} + static void stats_metric_settings_to_query(const struct stats_metric_settings *set, struct event_filter_query *query_r) @@ -127,6 +152,10 @@ static void stats_metrics_add_set(struct stats_metrics *metrics, fields = t_strsplit_spaces(set->fields, " "); metric = stats_metric_alloc(metrics->pool, set->name, fields); + + if (*set->group_by != '\0') + metric->group_by = (const char *const *)p_strsplit_spaces(metrics->pool, set->group_by, " "); + array_push_back(&metrics->metrics, &metric); stats_metric_settings_to_query(set, &query); @@ -281,8 +310,97 @@ stats_metrics_get_event_filter(struct stats_metrics *metrics) return metrics->combined_filter; } +static struct metric * +stats_metric_get_sub_metric(struct metric *metric, + const struct metric_value *value) +{ + struct metric *const *sub_metrics; + + /* lookup sub-metric */ + array_foreach (&metric->sub_metrics, sub_metrics) { + if ((*sub_metrics)->group_value.type == METRIC_VALUE_TYPE_STR && + memcmp((*sub_metrics)->group_value.hash, value->hash, + SHA1_RESULTLEN) == 0) + return *sub_metrics; + else if ((*sub_metrics)->group_value.type == METRIC_VALUE_TYPE_INT && + (*sub_metrics)->group_value.intmax == value->intmax) + return *sub_metrics; + } + return NULL; +} + +static struct metric * +stats_metric_sub_metric_alloc(struct metric *metric, const char *name, pool_t pool) +{ + struct metric *sub_metric; + ARRAY_TYPE(const_string) fields; + t_array_init(&fields, metric->fields_count); + for (unsigned int i = 0; i < metric->fields_count; i++) + array_append(&fields, &metric->fields[i].field_key, 1); + array_append_zero(&fields); + sub_metric = stats_metric_alloc(pool, metric->name, + array_idx(&fields, 0)); + sub_metric->sub_name = sub_metric_name_create(pool, name); + array_append(&metric->sub_metrics, &sub_metric, 1); + return sub_metric; +} + +static void +stats_metric_group_by(struct metric *metric, struct event *event, pool_t pool) +{ + struct metric *sub_metric; + const char *const *group = metric->group_by; + const struct event_field *field = + event_find_field(event, *group); + struct metric_value value; + + /* ignore missing field */ + if (field == NULL) + return; + switch (field->value_type) { + case EVENT_FIELD_VALUE_TYPE_STR: + value.type = METRIC_VALUE_TYPE_STR; + /* use sha1 of value to avoid excessive memory usage in case the + actual value is quite long */ + sha1_get_digest(field->value.str, strlen(field->value.str), + value.hash); + break; + case EVENT_FIELD_VALUE_TYPE_INTMAX: + value.type = METRIC_VALUE_TYPE_INT; + value.intmax = field->value.intmax; + break; + case EVENT_FIELD_VALUE_TYPE_TIMEVAL: + return; + } + + if (!array_is_created(&metric->sub_metrics)) + p_array_init(&metric->sub_metrics, pool, 8); + + sub_metric = stats_metric_get_sub_metric(metric, &value); + + if (sub_metric == NULL) T_BEGIN { + const char *value_label; + if (value.type == METRIC_VALUE_TYPE_STR) + value_label = field->value.str; + else if (value.type == METRIC_VALUE_TYPE_INT) + value_label = dec2str(field->value.intmax); + else + i_unreached(); + sub_metric = stats_metric_sub_metric_alloc(metric, value_label, + pool); + if (group[1] != NULL) + sub_metric->group_by = group+1; + sub_metric->group_value.intmax = value.intmax; + memcpy(sub_metric->group_value.hash, value.hash, SHA1_RESULTLEN); + } T_END; + + /* sub-metrics are recursive, so each sub-metric can have additional + sub-metrics. */ + stats_metric_event(sub_metric, event, pool); +} + static void -stats_metric_event(struct metric *metric, struct event *event) +stats_metric_event(struct metric *metric, struct event *event, pool_t pool) { intmax_t duration; @@ -309,6 +427,9 @@ stats_metric_event(struct metric *metric, struct event *event) } stats_dist_add(metric->fields[i].stats, num); } + + if (metric->group_by != NULL) + stats_metric_group_by(metric, event, pool); } static void @@ -343,7 +464,7 @@ void stats_metrics_event(struct stats_metrics *metrics, struct event *event, /* process stats */ iter = event_filter_match_iter_init(metrics->stats_filter, event, ctx); while ((metric = event_filter_match_iter_next(iter)) != NULL) T_BEGIN { - stats_metric_event(metric, event); + stats_metric_event(metric, event, metrics->pool); } T_END; event_filter_match_iter_deinit(&iter); diff --git a/src/stats/stats-metrics.h b/src/stats/stats-metrics.h index 685aba6fe9..6ec100d174 100644 --- a/src/stats/stats-metrics.h +++ b/src/stats/stats-metrics.h @@ -2,6 +2,7 @@ #define STATS_METRICS_H #include "stats-settings.h" +#include "sha1.h" struct metric; @@ -51,6 +52,17 @@ struct metric_field { struct stats_dist *stats; }; +enum metric_value_type { + METRIC_VALUE_TYPE_STR, + METRIC_VALUE_TYPE_INT, +}; + +struct metric_value { + enum metric_value_type type; + unsigned char hash[SHA1_RESULTLEN]; + intmax_t intmax; +}; + struct metric { const char *name; /* When this metric is a sub-metric, then this is the @@ -75,6 +87,9 @@ struct metric { unsigned int fields_count; struct metric_field *fields; + + const char *const *group_by; + struct metric_value group_value; ARRAY(struct metric *) sub_metrics; struct metric_export_info export_info; diff --git a/src/stats/stats-settings.c b/src/stats/stats-settings.c index 61a77a2d89..306ce2ee69 100644 --- a/src/stats/stats-settings.c +++ b/src/stats/stats-settings.c @@ -102,6 +102,7 @@ static const struct setting_define stats_metric_setting_defines[] = { DEF(SET_STR, source_location), DEF(SET_STR, categories), DEF(SET_STR, fields), + DEF(SET_STR, group_by), { SET_STRLIST, "filter", offsetof(struct stats_metric_settings, filter), NULL }, DEF(SET_STR, exporter), DEF(SET_STR, exporter_include), @@ -115,6 +116,7 @@ static const struct stats_metric_settings stats_metric_default_settings = { .categories = "", .fields = "", .exporter = "", + .group_by = "", .exporter_include = "name hostname timestamps categories fields", }; diff --git a/src/stats/stats-settings.h b/src/stats/stats-settings.h index 64ff7b83cd..00086a5863 100644 --- a/src/stats/stats-settings.h +++ b/src/stats/stats-settings.h @@ -75,6 +75,7 @@ struct stats_metric_settings { const char *source_location; const char *categories; const char *fields; + const char *group_by; ARRAY(const char *) filter; unsigned int parsed_source_linenum;