stats_SOURCES = \
client-reader.c \
client-writer.c \
+ event-exporter-fmt.c \
+ event-exporter-fmt-none.c \
+ event-exporter-transport-drop.c \
main.c \
stats-event-category.c \
stats-metrics.c \
noinst_HEADERS = \
client-reader.h \
client-writer.h \
+ event-exporter.h \
stats-event-category.h \
stats-metrics.h \
stats-settings.h
--- /dev/null
+/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "event-exporter.h"
+
+void event_export_fmt_none(const struct metric *metric ATTR_UNUSED,
+ struct event *event ATTR_UNUSED,
+ buffer_t *dest ATTR_UNUSED)
+{
+ /* nothing to do */
+}
--- /dev/null
+/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "ioloop.h"
+#include "event-exporter.h"
+
+void event_export_helper_fmt_unix_time(string_t *dest,
+ const struct timeval *time)
+{
+ str_printfa(dest, "%"PRIdTIME_T".%06u", time->tv_sec,
+ (unsigned int) time->tv_usec);
+}
+
+void event_export_helper_fmt_rfc3339_time(string_t *dest,
+ const struct timeval *time)
+{
+ const struct tm *tm;
+
+ tm = gmtime(&time->tv_sec);
+
+ str_printfa(dest, "%04d-%02d-%02dT%02d:%02d:%02d.%06luZ",
+ tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday,
+ tm->tm_hour, tm->tm_min, tm->tm_sec,
+ time->tv_usec);
+}
--- /dev/null
+/* Copyright (c) 2019 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "event-exporter.h"
+
+void event_export_transport_drop(const struct exporter *exporter ATTR_UNUSED,
+ const buffer_t *buf ATTR_UNUSED)
+{
+}
--- /dev/null
+#ifndef EVENT_EXPORTER_H
+#define EVENT_EXPORTER_H
+
+#include "stats-metrics.h"
+
+/* fmt functions */
+void event_export_fmt_none(const struct metric *metric, struct event *event, buffer_t *dest);
+
+/* transport functions */
+void event_export_transport_drop(const struct exporter *exporter, const buffer_t *buf);
+
+/* append a microsecond resolution RFC3339 UTC timestamp */
+void event_export_helper_fmt_rfc3339_time(string_t *dest, const struct timeval *time);
+/* append a microsecond resolution unix timestamp in seconds (i.e., %u.%06u) */
+void event_export_helper_fmt_unix_time(string_t *dest, const struct timeval *time);
+
+#endif
#include "stats-dist.h"
#include "time-util.h"
#include "event-filter.h"
+#include "event-exporter.h"
#include "stats-settings.h"
#include "stats-metrics.h"
struct stats_metrics {
pool_t pool;
- struct event_filter *stats_filter;
+ struct event_filter *stats_filter; /* stats-only */
+ struct event_filter *export_filter; /* export-only */
+ struct event_filter *combined_filter; /* stats & export */
+ ARRAY(struct exporter *) exporters;
ARRAY(struct metric *) metrics;
};
query_r->source_linenum = set->parsed_source_linenum;
}
+static void stats_exporters_add_set(struct stats_metrics *metrics,
+ const struct stats_exporter_settings *set)
+{
+ struct exporter *exporter;
+
+ exporter = p_new(metrics->pool, struct exporter, 1);
+ exporter->name = p_strdup(metrics->pool, set->name);
+ exporter->transport_args = p_strdup(metrics->pool, set->transport_args);
+ exporter->time_format = set->parsed_time_format;
+
+ /* TODO: The following should be plugable.
+ *
+ * Note: Make sure to mirror any changes to the below code in
+ * stats_exporter_settings_check().
+ */
+ if (strcmp(set->format, "none") == 0) {
+ exporter->format = event_export_fmt_none;
+ exporter->format_mime_type = "application/octet-stream";
+ } else {
+ i_unreached();
+ }
+
+ /* TODO: The following should be plugable.
+ *
+ * Note: Make sure to mirror any changes to the below code in
+ * stats_exporter_settings_check().
+ */
+ if (strcmp(set->transport, "drop") == 0) {
+ exporter->transport = event_export_transport_drop;
+ } else {
+ i_unreached();
+ }
+
+ exporter->transport_args = set->transport_args;
+
+ array_push_back(&metrics->exporters, &exporter);
+}
+
static void stats_metrics_add_set(struct stats_metrics *metrics,
const struct stats_metric_settings *set)
{
struct event_filter_query query;
+ struct exporter *const *exporter;
struct metric *metric;
const char *const *fields;
+ const char *const *tmp;
metric = p_new(metrics->pool, struct metric, 1);
metric->name = p_strdup(metrics->pool, set->name);
stats_metric_settings_to_query(set, &query);
query.context = metric;
event_filter_add(metrics->stats_filter, &query);
+ event_filter_add(metrics->combined_filter, &query);
+
+ /*
+ * Done with statistics setup, now onto exporter setup
+ */
+
+ if (set->exporter[0] == '\0')
+ return; /* not exported */
+
+ array_foreach(&metrics->exporters, exporter) {
+ if (strcmp(set->exporter, (*exporter)->name) == 0) {
+ metric->export_info.exporter = *exporter;
+ break;
+ }
+ }
+
+ if (metric->export_info.exporter == NULL)
+ i_panic("Could not find exporter (%s) for metric (%s)",
+ set->exporter, set->name);
+
+ /* Defaults */
+ metric->export_info.include = EVENT_EXPORTER_INCL_NONE;
+
+ tmp = t_strsplit_spaces(set->exporter_include, " ");
+ for (; *tmp != NULL; tmp++) {
+ if (strcmp(*tmp, "name") == 0)
+ metric->export_info.include |= EVENT_EXPORTER_INCL_NAME;
+ else if (strcmp(*tmp, "hostname") == 0)
+ metric->export_info.include |= EVENT_EXPORTER_INCL_HOSTNAME;
+ else if (strcmp(*tmp, "timestamps") == 0)
+ metric->export_info.include |= EVENT_EXPORTER_INCL_TIMESTAMPS;
+ else if (strcmp(*tmp, "categories") == 0)
+ metric->export_info.include |= EVENT_EXPORTER_INCL_CATEGORIES;
+ else if (strcmp(*tmp, "fields") == 0)
+ metric->export_info.include |= EVENT_EXPORTER_INCL_FIELDS;
+ else
+ i_warning("Ignoring unknown exporter include '%s'", *tmp);
+ }
+
+ /* query already constructed */
+ event_filter_add(metrics->export_filter, &query);
}
static void
stats_metrics_add_from_settings(struct stats_metrics *metrics,
const struct stats_settings *set)
{
- struct stats_metric_settings *const *metric_setp;
+ /* add all the exporters first */
+ if (!array_is_created(&set->exporters)) {
+ p_array_init(&metrics->exporters, metrics->pool, 0);
+ } else {
+ struct stats_exporter_settings *const *exporter_setp;
+
+ p_array_init(&metrics->exporters, metrics->pool,
+ array_count(&set->exporters));
+ array_foreach(&set->exporters, exporter_setp)
+ stats_exporters_add_set(metrics, *exporter_setp);
+ }
+ /* then add all the metrics */
if (!array_is_created(&set->metrics)) {
p_array_init(&metrics->metrics, metrics->pool, 0);
- return;
+ } else {
+ struct stats_metric_settings *const *metric_setp;
+
+ p_array_init(&metrics->metrics, metrics->pool,
+ array_count(&set->metrics));
+ array_foreach(&set->metrics, metric_setp) T_BEGIN {
+ stats_metrics_add_set(metrics, *metric_setp);
+ } T_END;
}
-
- p_array_init(&metrics->metrics, metrics->pool,
- array_count(&set->metrics));
- array_foreach(&set->metrics, metric_setp) T_BEGIN {
- stats_metrics_add_set(metrics, *metric_setp);
- } T_END;
}
struct stats_metrics *stats_metrics_init(const struct stats_settings *set)
metrics = p_new(pool, struct stats_metrics, 1);
metrics->pool = pool;
metrics->stats_filter = event_filter_create();
+ metrics->export_filter = event_filter_create();
+ metrics->combined_filter = event_filter_create();
stats_metrics_add_from_settings(metrics, set);
return metrics;
}
stats_dist_deinit(&metric->fields[i].stats);
}
+static void stats_export_deinit(void)
+{
+ /* no need for event_export_transport_drop_deinit() - no-op */
+}
+
void stats_metrics_deinit(struct stats_metrics **_metrics)
{
struct stats_metrics *metrics = *_metrics;
*_metrics = NULL;
+ stats_export_deinit();
+
array_foreach(&metrics->metrics, metricp)
stats_metric_free(*metricp);
event_filter_unref(&metrics->stats_filter);
+ event_filter_unref(&metrics->export_filter);
+ event_filter_unref(&metrics->combined_filter);
pool_unref(&metrics->pool);
}
struct event_filter *
stats_metrics_get_event_filter(struct stats_metrics *metrics)
{
- return metrics->stats_filter;
+ return metrics->combined_filter;
}
static void
}
}
+static void
+stats_export_event(struct metric *metric, struct event *oldevent)
+{
+ const struct metric_export_info *info = &metric->export_info;
+ const struct exporter *exporter = info->exporter;
+ struct event *event;
+
+ i_assert(exporter != NULL);
+
+ event = event_flatten(oldevent);
+
+ T_BEGIN {
+ buffer_t *buf;
+
+ buf = t_buffer_create(128);
+
+ exporter->format(metric, event, buf);
+ exporter->transport(exporter, buf);
+ } T_END;
+
+ event_unref(&event);
+}
+
void stats_metrics_event(struct stats_metrics *metrics, struct event *event,
const struct failure_context *ctx)
{
struct event_filter_match_iter *iter;
struct metric *metric;
+ /* process stats */
iter = event_filter_match_iter_init(metrics->stats_filter, event, ctx);
while ((metric = event_filter_match_iter_next(iter)) != NULL)
stats_metric_event(metric, event);
event_filter_match_iter_deinit(&iter);
+
+ /* process exports */
+ iter = event_filter_match_iter_init(metrics->export_filter, event, ctx);
+ while ((metric = event_filter_match_iter_next(iter)) != NULL)
+ stats_export_event(metric, event);
+ event_filter_match_iter_deinit(&iter);
}
struct stats_metrics_iter {
#ifndef STATS_METRICS_H
#define STATS_METRICS_H
-struct stats_settings;
+#include "stats-settings.h"
+
+struct metric;
+
+struct exporter {
+ const char *name;
+
+ /*
+ * serialization format options
+ *
+ * the "how do we encode the event before sending it" knobs
+ */
+ enum event_exporter_time_fmt time_format;
+
+ /* function to serialize the event */
+ void (*format)(const struct metric *, struct event *, buffer_t *);
+
+ /* mime type for the format */
+ const char *format_mime_type;
+
+ /*
+ * transport options
+ *
+ * the "how do we get the event to the external location" knobs
+ */
+ const char *transport_args;
+
+ /* function to send the event */
+ void (*transport)(const struct exporter *, const buffer_t *);
+};
+
+struct metric_export_info {
+ const struct exporter *exporter;
+
+ enum event_exporter_includes {
+ EVENT_EXPORTER_INCL_NONE = 0,
+ EVENT_EXPORTER_INCL_NAME = 0x01,
+ EVENT_EXPORTER_INCL_HOSTNAME = 0x02,
+ EVENT_EXPORTER_INCL_TIMESTAMPS = 0x04,
+ EVENT_EXPORTER_INCL_CATEGORIES = 0x08,
+ EVENT_EXPORTER_INCL_FIELDS = 0x10,
+ } include;
+};
struct metric_field {
const char *field_key;
unsigned int fields_count;
struct metric_field *fields;
+
+ struct metric_export_info export_info;
};
struct stats_metrics *stats_metrics_init(const struct stats_settings *set);
#include "settings-parser.h"
#include "service-settings.h"
#include "stats-settings.h"
+#include "array.h"
static bool stats_metric_settings_check(void *_set, pool_t pool, const char **error_r);
+static bool stats_exporter_settings_check(void *_set, pool_t pool, const char **error_r);
+static bool stats_settings_check(void *_set, pool_t pool, const char **error_r);
/* <settings checks> */
static struct file_listener_settings stats_unix_listeners_array[] = {
.inet_listeners = ARRAY_INIT,
};
+/*
+ * event_exporter { } block settings
+ */
+
+#undef DEF
+#define DEF(type, name) \
+ { type, #name, offsetof(struct stats_exporter_settings, name), NULL }
+
+static const struct setting_define stats_exporter_setting_defines[] = {
+ DEF(SET_STR, name),
+ DEF(SET_STR, transport),
+ DEF(SET_STR, transport_args),
+ DEF(SET_STR, format),
+ DEF(SET_STR, format_args),
+ SETTING_DEFINE_LIST_END
+};
+
+static const struct stats_exporter_settings stats_exporter_default_settings = {
+ .name = "",
+ .transport = "",
+ .transport_args = "",
+ .format = "",
+ .format_args = "",
+};
+
+const struct setting_parser_info stats_exporter_setting_parser_info = {
+ .defines = stats_exporter_setting_defines,
+ .defaults = &stats_exporter_default_settings,
+
+ .type_offset = offsetof(struct stats_exporter_settings, name),
+ .struct_size = sizeof(struct stats_exporter_settings),
+
+ .parent_offset = (size_t)-1,
+ .check_func = stats_exporter_settings_check,
+};
+
+/*
+ * metric { } block settings
+ */
+
#undef DEF
#define DEF(type, name) \
{ type, #name, offsetof(struct stats_metric_settings, name), NULL }
DEF(SET_STR, categories),
DEF(SET_STR, fields),
{ SET_STRLIST, "filter", offsetof(struct stats_metric_settings, filter), NULL },
+ DEF(SET_STR, exporter),
+ DEF(SET_STR, exporter_include),
SETTING_DEFINE_LIST_END
};
.source_location = "",
.categories = "",
.fields = "",
+ .exporter = "",
+ .exporter_include = "name hostname timestamps categories fields",
};
const struct setting_parser_info stats_metric_setting_parser_info = {
.check_func = stats_metric_settings_check,
};
+/*
+ * top-level settings
+ */
+
#undef DEFLIST_UNIQUE
#define DEFLIST_UNIQUE(field, name, defines) \
{ SET_DEFLIST_UNIQUE, name, \
static const struct setting_define stats_setting_defines[] = {
DEFLIST_UNIQUE(metrics, "metric", &stats_metric_setting_parser_info),
+ DEFLIST_UNIQUE(exporters, "event_exporter", &stats_exporter_setting_parser_info),
SETTING_DEFINE_LIST_END
};
const struct stats_settings stats_default_settings = {
- .metrics = ARRAY_INIT
+ .metrics = ARRAY_INIT,
+ .exporters = ARRAY_INIT,
};
const struct setting_parser_info stats_setting_parser_info = {
.type_offset = (size_t)-1,
.struct_size = sizeof(struct stats_settings),
- .parent_offset = (size_t)-1
+ .parent_offset = (size_t)-1,
+ .check_func = stats_settings_check,
};
/* <settings checks> */
+static bool parse_format_args_set_time(struct stats_exporter_settings *set,
+ enum event_exporter_time_fmt fmt,
+ const char **error_r)
+{
+ if ((set->parsed_time_format != EVENT_EXPORTER_TIME_FMT_NATIVE) &&
+ (set->parsed_time_format != fmt)) {
+ *error_r = t_strdup_printf("Exporter '%s' specifies multiple "
+ "time format args", set->name);
+ return FALSE;
+ }
+
+ set->parsed_time_format = fmt;
+
+ return TRUE;
+}
+
+static bool parse_format_args(struct stats_exporter_settings *set,
+ const char **error_r)
+{
+ const char *const *tmp;
+
+ /* Defaults */
+ set->parsed_time_format = EVENT_EXPORTER_TIME_FMT_NATIVE;
+
+ tmp = t_strsplit_spaces(set->format_args, " ");
+
+ /*
+ * If the config contains multiple types of the same type (e.g.,
+ * both time-rfc3339 and time-unix) we fail the config check.
+ *
+ * Note: At the moment, we have only time-* tokens. In the future
+ * when we have other tokens, they should be parsed here.
+ */
+ for (; *tmp != NULL; tmp++) {
+ enum event_exporter_time_fmt fmt;
+
+ if (strcmp(*tmp, "time-rfc3339") == 0) {
+ fmt = EVENT_EXPORTER_TIME_FMT_RFC3339;
+ } else if (strcmp(*tmp, "time-unix") == 0) {
+ fmt = EVENT_EXPORTER_TIME_FMT_UNIX;
+ } else {
+ *error_r = t_strdup_printf("Unknown exporter format "
+ "arg: %s", *tmp);
+ return FALSE;
+ }
+
+ if (!parse_format_args_set_time(set, fmt, error_r))
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static bool stats_exporter_settings_check(void *_set, pool_t pool ATTR_UNUSED,
+ const char **error_r)
+{
+ struct stats_exporter_settings *set = _set;
+ bool time_fmt_required;
+
+ if (set->name[0] == '\0') {
+ *error_r = "Exporter name can't be empty";
+ return FALSE;
+ }
+
+ /* TODO: The following should be plugable.
+ *
+ * Note: Make sure to mirror any changes to the below code in
+ * stats_exporters_add_set().
+ */
+ if (set->format[0] == '\0') {
+ *error_r = "Exporter format name can't be empty";
+ return FALSE;
+ } else if (strcmp(set->format, "none") == 0) {
+ time_fmt_required = FALSE;
+ } else {
+ *error_r = t_strdup_printf("Unknown exporter format '%s'",
+ set->format);
+ return FALSE;
+ }
+
+ /* TODO: The following should be plugable.
+ *
+ * Note: Make sure to mirror any changes to the below code in
+ * stats_exporters_add_set().
+ */
+ if (set->transport[0] == '\0') {
+ *error_r = "Exporter transport name can't be empty";
+ return FALSE;
+ } else if (strcmp(set->transport, "drop") == 0) {
+ /* no-op */
+ } else {
+ *error_r = t_strdup_printf("Unknown transport type '%s'",
+ set->transport);
+ return FALSE;
+ }
+
+ if (!parse_format_args(set, error_r))
+ return FALSE;
+
+ /* Some formats don't have a native way of serializing time stamps */
+ if (time_fmt_required &&
+ set->parsed_time_format == EVENT_EXPORTER_TIME_FMT_NATIVE) {
+ *error_r = t_strdup_printf("%s exporter format requires a "
+ "time-* argument", set->format);
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
static bool stats_metric_settings_check(void *_set, pool_t pool ATTR_UNUSED,
const char **error_r)
{
return FALSE;
}
}
+
+ return TRUE;
+}
+
+static bool stats_settings_check(void *_set, pool_t pool ATTR_UNUSED,
+ const char **error_r)
+{
+ struct stats_settings *set = _set;
+ struct stats_exporter_settings *const *exporter;
+ struct stats_metric_settings *const *metric;
+
+ if (!array_is_created(&set->metrics) || !array_is_created(&set->exporters))
+ return TRUE;
+
+ /* check that all metrics refer to exporters that exist */
+ array_foreach(&set->metrics, metric) {
+ bool found = FALSE;
+
+ if ((*metric)->exporter[0] == '\0')
+ continue; /* metric not exported */
+
+ array_foreach(&set->exporters, exporter) {
+ if (strcmp((*metric)->exporter, (*exporter)->name) == 0) {
+ found = TRUE;
+ break;
+ }
+ }
+
+ if (!found) {
+ *error_r = t_strdup_printf("metric %s refers to "
+ "non-existent exporter '%s'",
+ (*metric)->name,
+ (*metric)->exporter);
+ return FALSE;
+ }
+ }
+
return TRUE;
}
/* </settings checks> */
#ifndef STATS_SETTINGS_H
#define STATS_SETTINGS_H
+/* <settings checks> */
+/*
+ * We allow a selection of a timestamp format.
+ *
+ * The 'time-unix' format generates a number with the number of seconds
+ * since 1970-01-01 00:00 UTC.
+ *
+ * The 'time-rfc3339' format uses the YYYY-MM-DDTHH:MM:SS.uuuuuuZ format as
+ * defined by RFC 3339.
+ *
+ * The special native format (not explicitly selectable in the config, but
+ * default if no time-* token is used) uses the format's native timestamp
+ * format. Note that not all formats have a timestamp data format.
+ *
+ * The native format and the rules below try to address the question: can a
+ * parser that doesn't have any knowledge of fields' values' types losslessly
+ * reconstruct the fields?
+ *
+ * For example, JSON only has strings and numbers, so it cannot represent a
+ * timestamp in a "context-free lossless" way. Therefore, when making a
+ * JSON blob, we need to decide which way to serialize timestamps. No
+ * matter how we do it, we incur some loss. If a decoder sees 1557232304 in
+ * a field, it cannot be certain if the field is an integer that just
+ * happens to be a reasonable timestamp, or if it actually is a timestamp.
+ * Same goes with RFC3339 - it could just be that the user supplied a string
+ * that looks like a timestamp, and that string made it into an event field.
+ *
+ * Other common serialization formats, such as CBOR, have a lossless way of
+ * encoding timestamps.
+ *
+ * Note that there are two concepts at play: native and default.
+ *
+ * The rules for how the format's timestamp formats are used:
+ *
+ * 1. The default time format is the native format.
+ * 2. The native time format may or may not exist for a given format (e.g.,
+ * in JSON)
+ * 3. If the native format doesn't exist and no time format was specified in
+ * the config, it is a config error.
+ *
+ * We went with these rules because:
+ *
+ * 1. It prevents type information loss by default.
+ * 2. It completely isolates the policy from the algorithm.
+ * 3. It defers the decision whether each format without a native timestamp
+ * type should have a default acting as native until after we've had some
+ * operational experience.
+ * 4. A future decision to add a default (via 3. point) will be 100% compatible.
+ */
+enum event_exporter_time_fmt {
+ EVENT_EXPORTER_TIME_FMT_NATIVE = 0,
+ EVENT_EXPORTER_TIME_FMT_UNIX,
+ EVENT_EXPORTER_TIME_FMT_RFC3339,
+};
+/* </settings checks> */
+
+struct stats_exporter_settings {
+ const char *name;
+ const char *transport;
+ const char *transport_args;
+ const char *format;
+ const char *format_args;
+
+ /* parsed values */
+ enum event_exporter_time_fmt parsed_time_format;
+};
+
struct stats_metric_settings {
const char *name;
const char *event_name;
ARRAY(const char *) filter;
unsigned int parsed_source_linenum;
+
+ /* exporter related fields */
+ const char *exporter;
+ const char *exporter_include;
};
struct stats_settings {
+ ARRAY(struct stats_exporter_settings *) exporters;
ARRAY(struct stats_metric_settings *) metrics;
};
projects / thirdparty / dovecot / core.git / commitdiff
