From: Florian Forster Date: Wed, 24 Jun 2020 09:08:01 +0000 (+0200) Subject: format_json: Reimplement format_json_metric(). X-Git-Tag: 6.0.0-rc0~144^2~74 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=eeb6ffa1ffa9ed007dac1869f277ba3c2a134815;p=thirdparty%2Fcollectd.git format_json: Reimplement format_json_metric(). This new implementation uses the new format of the metric_t type. It produces output that is compatible with the prometheus/prom2json project. --- diff --git a/Makefile.am b/Makefile.am index 0a3346696..559fa712c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -495,7 +495,7 @@ libformat_json_la_SOURCES = \ src/utils/format_json/format_json.h libformat_json_la_CPPFLAGS = $(AM_CPPFLAGS) libformat_json_la_LDFLAGS = $(AM_LDFLAGS) -libformat_json_la_LIBADD = +libformat_json_la_LIBADD = libmetric.la if BUILD_WITH_LIBYAJL libformat_json_la_CPPFLAGS += $(BUILD_WITH_LIBYAJL_CPPFLAGS) libformat_json_la_LDFLAGS += $(BUILD_WITH_LIBYAJL_LDFLAGS) diff --git a/src/daemon/plugin_mock.c b/src/daemon/plugin_mock.c index 311ccdfd9..788503fc6 100644 --- a/src/daemon/plugin_mock.c +++ b/src/daemon/plugin_mock.c @@ -244,3 +244,21 @@ int plugin_thread_create(__attribute__((unused)) pthread_t *thread, * would be to hard-code the top-level config keys in daemon/collectd.c to avoid * having these references in daemon/configfile.c. */ int fc_configure(const oconfig_item_t *ci) { return ENOTSUP; } + +int plugin_convert_values_to_metrics(__attribute__((unused)) + value_list_t const *vl, + __attribute__((unused)) + metrics_list_t **ml) { + return ENOTSUP; +} + +void destroy_metrics_list(metrics_list_t *ml) { + while (ml != NULL) { + identity_destroy(ml->metric.identity); + meta_data_destroy(ml->metric.meta); + + metrics_list_t *next = ml->next_p; + free(ml); + ml = next; + } +} diff --git a/src/utils/format_json/format_json.c b/src/utils/format_json/format_json.c index d3f97f190..e9028af86 100644 --- a/src/utils/format_json/format_json.c +++ b/src/utils/format_json/format_json.c @@ -376,131 +376,6 @@ static int value_list_to_json(char *buffer, size_t buffer_size, /* {{{ */ return 0; } /* }}} int value_list_to_json */ - -int format_json_metric(char *buffer_p, size_t *ret_buffer_fill, /* {{{ */ - size_t *ret_buffer_free, const metric_t *metric_p, - int store_rates) { - char temp[1024]; - gauge_t rate = -1; - int status = 0; - - if ((buffer_p == NULL) || (ret_buffer_fill == NULL) || - (ret_buffer_free == NULL) || (metric_p == NULL) || - (metric_p->identity == NULL) || (metric_p->ds == NULL)) - return -EINVAL; - - if (*ret_buffer_free < 3) - return -ENOMEM; - - /* Respect high water marks and fere size */ - char *buffer = buffer_p + (*ret_buffer_fill); - size_t buffer_size = *ret_buffer_free; - size_t offset = 0; - -#define BUFFER_ADD(...) \ - do { \ - status = snprintf(buffer + offset, buffer_size - offset, __VA_ARGS__); \ - if (status < 1) \ - return -1; \ - else if (((size_t)status) >= (buffer_size - offset)) \ - return -ENOMEM; \ - else \ - offset += ((size_t)status); \ - } while (0) - -#define BUFFER_ADD_KEYVAL(key, value) \ - do { \ - status = json_escape_string(temp, sizeof(temp), (value)); \ - if (status != 0) \ - return status; \ - BUFFER_ADD(",\"%s\":%s", (key), temp); \ - } while (0) - - /* Designed to be called multiple times, as when adding metrics from a - metrics_lisat_t object. When finalizing, the initial leading comma - will be replaced by a [ */ - BUFFER_ADD(",{"); - BUFFER_ADD_KEYVAL("name", metric_p->identity->name); - if (metric_p->value_type == DS_TYPE_GAUGE) { - if (isfinite(metric_p->value.gauge)) - BUFFER_ADD(JSON_GAUGE_FORMAT, metric_p->value.gauge); - else - BUFFER_ADD("null"); - } else if (store_rates) { - if (rate == -1) - status = uc_get_rate(metric_p, &rate); - if (status != 0) { - WARNING("utils_format_json: uc_get_rate failed."); - buffer_p[*ret_buffer_fill] = '0'; - return -1; - } - - if (isfinite(rate)) - BUFFER_ADD(JSON_GAUGE_FORMAT, rate); - else - BUFFER_ADD("null"); - } else if (metric_p->value_type == DS_TYPE_COUNTER) - BUFFER_ADD("%" PRIu64, (uint64_t)metric_p->value.counter); - else if (metric_p->value_type == DS_TYPE_DERIVE) - BUFFER_ADD("%" PRIi64, metric_p->value.derive); - else if (metric_p->value_type == DS_TYPE_ABSOLUTE) - BUFFER_ADD("%" PRIu64, metric_p->value.absolute); - else { - ERROR("format_json: Unknown data source type: %i", metric_p->value_type); - buffer_p[*ret_buffer_fill] = '0'; - return -1; - } - - BUFFER_ADD(",\"time\":%.3f", CDTIME_T_TO_DOUBLE(metric_p->time)); - BUFFER_ADD(",\"interval\":%.3f", CDTIME_T_TO_DOUBLE(metric_p->interval)); - BUFFER_ADD_KEYVAL("plugin", metric_p->plugin); - BUFFER_ADD_KEYVAL("type", metric_p->type); - BUFFER_ADD_KEYVAL("dsname", metric_p->ds->name); - BUFFER_ADD_KEYVAL("dstype", DS_TYPE_TO_STRING(metric_p->value_type)); - - if (metric_p->identity->root_p != NULL) { - c_avl_iterator_t *iter_p = c_avl_get_iterator(metric_p->identity->root_p); - if (iter_p != NULL) { - char *key_p = NULL; - char *value_p = NULL; - BUFFER_ADD("\"labels\":{"); - while ((c_avl_iterator_next(iter_p, (void **)&key_p, - (void **)&value_p)) == 0) { - if ((key_p != NULL) && (value_p != NULL)) { - BUFFER_ADD_KEYVAL(key_p, value_p); - } - } - BUFFER_ADD("},"); - c_avl_iterator_destroy(iter_p); - } - } - - if (metric_p->meta != NULL) { - char meta_buffer[buffer_size]; - memset(meta_buffer, 0, sizeof(meta_buffer)); - status = meta_data_to_json(meta_buffer, sizeof(meta_buffer), - metric_p->meta->meta); - if (status != 0) { - buffer_p[*ret_buffer_fill] = '0'; - return status; - } - - BUFFER_ADD(",\"meta\":%s", meta_buffer); - } /* if (vl->meta != NULL) */ - - BUFFER_ADD("}"); - -#undef BUFFER_ADD_KEYVAL -#undef BUFFER_ADD - - /* Update hihg water mark and free size */ - (*ret_buffer_fill) += offset; - (*ret_buffer_free) -= offset; - - return 0; -} /* }}} */ - - static int format_json_value_list_nocheck(char *buffer, /* {{{ */ size_t *ret_buffer_fill, size_t *ret_buffer_free, @@ -668,6 +543,142 @@ static int format_time(yajl_gen g, cdtime_t t) /* {{{ */ return 0; } /* }}} int format_time */ +/* TODO(octo): format_metric should export the interval, too. */ +/* TODO(octo): Decide whether format_metric should export meta data. */ +static int format_metric(yajl_gen g, metric_t const *m) { + CHECK_SUCCESS(yajl_gen_map_open(g)); /* BEGIN metric */ + + if (c_avl_size(m->identity->labels) != 0) { + JSON_ADD(g, "labels"); + CHECK_SUCCESS(yajl_gen_map_open(g)); /* BEGIN labels */ + + c_avl_iterator_t *iter = c_avl_get_iterator(m->identity->labels); + char *k = NULL, *v = NULL; + while (c_avl_iterator_next(iter, (void *)&k, (void *)&v) == 0) { + JSON_ADD(g, k); + JSON_ADD(g, v); + } + c_avl_iterator_destroy(iter); + + CHECK_SUCCESS(yajl_gen_map_close(g)); /* END labels */ + } + + if (m->time != 0) { + JSON_ADD(g, "timestamp_ms"); + JSON_ADDF(g, "%" PRIu64, CDTIME_T_TO_MS(m->time)); + } + + strbuf_t buf = STRBUF_CREATE; + int status = value_marshal_text(&buf, m->value, m->value_type); + if (status != 0) { + STRBUF_DESTROY(buf); + return status; + } + JSON_ADD(g, "value"); + JSON_ADD(g, buf.ptr); + STRBUF_DESTROY(buf); + + CHECK_SUCCESS(yajl_gen_map_close(g)); /* END metric */ + + return 0; +} + +/* format_metrics_list that all metrics in ml have the same name and value_type. + */ +static int format_metrics_list(yajl_gen g, metrics_list_t const *ml) { + CHECK_SUCCESS(yajl_gen_map_open(g)); /* BEGIN metric family */ + + metric_t const *m = &ml->metric; + + JSON_ADD(g, "name"); + JSON_ADD(g, m->identity->name); + + char const *type = NULL; + switch (m->value_type) { + /* TODO(octo): handle store_rates. */ + case VALUE_TYPE_GAUGE: + type = "GAUGE"; + break; + case VALUE_TYPE_DERIVE: + case DS_TYPE_COUNTER: + type = "COUNTER"; + break; + default: + ERROR("format_json_metric: Unknown value type: %d", m->value_type); + return EINVAL; + } + JSON_ADD(g, "type"); + JSON_ADD(g, type); + + JSON_ADD(g, "metrics"); + CHECK_SUCCESS(yajl_gen_array_open(g)); + for (metrics_list_t const *ptr = ml; ptr != NULL; ptr = ptr->next_p) { + int status = format_metric(g, &ptr->metric); + if (status != 0) { + return status; + } + } + CHECK_SUCCESS(yajl_gen_array_close(g)); + + CHECK_SUCCESS(yajl_gen_map_close(g)); /* END metric family */ + + return 0; +} + +int format_json_metric(strbuf_t *buf, metric_t const *m, bool store_rates) { + if ((buf == NULL) || (m == NULL)) + return EINVAL; + +#if HAVE_YAJL_V2 + yajl_gen g = yajl_gen_alloc(NULL); + if (g == NULL) + return -1; +#if COLLECT_DEBUG + yajl_gen_config(g, yajl_gen_beautify, 1); + yajl_gen_config(g, yajl_gen_validate_utf8, 1); +#endif + +#else /* !HAVE_YAJL_V2 */ + yajl_gen_config conf = {0}; +#if COLLECT_DEBUG + conf.beautify = 1; + conf.indentString = " "; +#endif + yajl_gen g = yajl_gen_alloc(&conf, NULL); + if (g == NULL) + return -1; +#endif + + yajl_gen_array_open(g); + + int status = format_metrics_list(g, &(metrics_list_t){.metric = *m}); + if (status != 0) { + yajl_gen_clear(g); + yajl_gen_free(g); + return status; + } + + yajl_gen_array_close(g); + + /* copy to output buffer */ + unsigned char const *out = NULL; +#if HAVE_YAJL_V2 + size_t unused_out_len = 0; +#else + unsigned int unused_out_len = 0; +#endif + if (yajl_gen_get_buf(g, &out, &unused_out_len) != yajl_gen_status_ok) { + yajl_gen_clear(g); + yajl_gen_free(g); + return -1; + } + status = strbuf_print(buf, (void *)out); + + yajl_gen_clear(g); + yajl_gen_free(g); + return status; +} /* }}} format_json_metric */ + static int format_alert(yajl_gen g, notification_t const *n) /* {{{ */ { CHECK_SUCCESS(yajl_gen_array_open(g)); /* BEGIN array */ diff --git a/src/utils/format_json/format_json.h b/src/utils/format_json/format_json.h index a59ed6237..02214609a 100644 --- a/src/utils/format_json/format_json.h +++ b/src/utils/format_json/format_json.h @@ -30,6 +30,7 @@ #include "collectd.h" #include "plugin.h" +#include "utils/strbuf/strbuf.h" #ifndef JSON_GAUGE_FORMAT #define JSON_GAUGE_FORMAT GAUGE_FORMAT @@ -40,11 +41,13 @@ int format_json_initialize(char *buffer, size_t *ret_buffer_fill, int format_json_value_list(char *buffer, size_t *ret_buffer_fill, size_t *ret_buffer_free, const data_set_t *ds, const value_list_t *vl, int store_rates); -int format_json_metric(char *buffer_p, size_t *ret_buffer_fill, - size_t *ret_buffer_free, const metric_t *metric_p, - int store_rates); int format_json_finalize(char *buffer, size_t *ret_buffer_fill, size_t *ret_buffer_free); + +/* format_json_metric writes m to buf in JSON format. The format produces is + * compatible to the "prometheus/prom2json" project. */ +int format_json_metric(strbuf_t *buf, metric_t const *m, bool store_rates); + int format_json_notification(char *buffer, size_t buffer_size, notification_t const *n); diff --git a/src/utils/format_json/format_json_test.c b/src/utils/format_json/format_json_test.c index ad735e449..c36963451 100644 --- a/src/utils/format_json/format_json_test.c +++ b/src/utils/format_json/format_json_test.c @@ -165,8 +165,98 @@ DEF_TEST(notification) { return expect_json_labels(got, labels, STATIC_ARRAY_SIZE(labels)); } +DEF_TEST(metric) { + struct { + char const *identity; + value_t value; + int value_type; + cdtime_t time; + cdtime_t interval; + meta_data_t *meta; + char const *want; + } cases[] = { + { + .identity = "metric_name", + .value = (value_t){.gauge = 42}, + .value_type = VALUE_TYPE_GAUGE, + .want = "[{\"name\":\"metric_name\",\"type\":\"GAUGE\",\"metrics\":[" + "{\"value\":\"42\"}" + "]}]", + }, + { + .identity = + "metric_with_labels{sorted=\"true\",alphabetically=\"yes\"}", + .value = (value_t){.gauge = 42}, + .value_type = VALUE_TYPE_GAUGE, + .want = "[{\"name\":\"metric_with_labels\",\"type\":\"GAUGE\"," + "\"metrics\":[" + "{\"labels\":{\"alphabetically\":\"yes\",\"sorted\":\"true\"}" + ",\"value\":\"42\"}" + "]}]", + }, + { + .identity = "metric_with_time", + .value = (value_t){.gauge = 42}, + .value_type = VALUE_TYPE_GAUGE, + .time = MS_TO_CDTIME_T(1592987324125), + .want = + "[{\"name\":\"metric_with_time\",\"type\":\"GAUGE\",\"metrics\":[" + "{\"timestamp_ms\":\"1592987324125\",\"value\":\"42\"}" + "]}]", + }, + { + .identity = "derive_max", + .value = (value_t){.derive = INT64_MAX}, + .value_type = VALUE_TYPE_DERIVE, + .want = "[{\"name\":\"derive_max\",\"type\":\"COUNTER\",\"metrics\":[" + "{\"value\":\"9223372036854775807\"}" + "]}]", + }, + { + .identity = "derive_min", + .value = (value_t){.derive = INT64_MIN}, + .value_type = VALUE_TYPE_DERIVE, + .want = "[{\"name\":\"derive_min\",\"type\":\"COUNTER\",\"metrics\":[" + "{\"value\":\"-9223372036854775808\"}" + "]}]", + }, + { + .identity = "counter_max", + .value = (value_t){.counter = UINT64_MAX}, + .value_type = DS_TYPE_COUNTER, + .want = + "[{\"name\":\"counter_max\",\"type\":\"COUNTER\",\"metrics\":[" + "{\"value\":\"18446744073709551615\"}" + "]}]", + }, + }; + + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + identity_t *id; + CHECK_NOT_NULL(id = identity_unmarshal_text(cases[i].identity)); + + metric_t m = { + .identity = id, + .value = cases[i].value, + .value_type = cases[i].value_type, + .time = cases[i].time, + .interval = cases[i].interval, + .meta = cases[i].meta, + }; + + strbuf_t buf = STRBUF_CREATE; + CHECK_ZERO(format_json_metric(&buf, &m, false)); + + EXPECT_EQ_STR(cases[i].want, buf.ptr); + STRBUF_DESTROY(buf); + } + + return 0; +} + int main(void) { RUN_TEST(notification); + RUN_TEST(metric); END_TEST; }