]> git.ipfire.org Git - thirdparty/dovecot/core.git/commitdiff
stats: stats-service-openmetrics - Restructure metrics export for incremental output.
authorStephan Bosch <stephan.bosch@open-xchange.com>
Fri, 22 Nov 2019 18:55:48 +0000 (19:55 +0100)
committermartti.rannanjarvi <martti.rannanjarvi@open-xchange.com>
Sat, 18 Apr 2020 14:55:11 +0000 (14:55 +0000)
This way, the string buffer can be written to an output stream after every
exported (sub-) metric.

src/stats/stats-service-openmetrics.c

index 88264267592f5028158149e8b8a1c1c02b67c54f..e5714e065109196cc99f6eacd21b14015816992d 100644 (file)
@@ -5,6 +5,7 @@
 #include "str.h"
 #include "array.h"
 #include "ioloop.h"
+#include "ostream.h"
 #include "stats-dist.h"
 #include "http-server.h"
 #include "client-http.h"
@@ -28,10 +29,29 @@ enum openmetrics_metric_type {
        OPENMETRICS_METRIC_TYPE_DURATION,
 };
 
+enum openmetrics_request_state {
+       OPENMETRICS_REQUEST_STATE_INIT = 0,
+       OPENMETRICS_REQUEST_STATE_METRIC,
+       OPENMETRICS_REQUEST_STATE_METRIC_HEADER,
+       OPENMETRICS_REQUEST_STATE_SUB_METRICS,
+       OPENMETRICS_REQUEST_STATE_METRIC_BODY,
+       OPENMETRICS_REQUEST_STATE_FINISHED,
+};
+
+struct openmetrics_request_sub_metric {
+       size_t labels_pos;
+       const struct metric *metric;
+       unsigned int sub_index;
+};
+
 struct openmetrics_request {
+       enum openmetrics_request_state state;
+       struct stats_metrics_iter *stats_iter;
        const struct metric *metric;
        enum openmetrics_metric_type metric_type;
        string_t *labels;
+       size_t labels_pos;
+       ARRAY(struct openmetrics_request_sub_metric) sub_metric_stack;
 
        bool has_submetric:1;
 };
@@ -47,10 +67,6 @@ struct openmetrics_request {
    must match the regex [a-zA-Z_:][a-zA-Z0-9_:]*.
  */
 
-static void
-openmetrics_export_submetrics(struct openmetrics_request *req, string_t *out,
-                             const struct metric *metric, int64_t timestamp);
-
 static bool openmetrics_check_name(const char *name)
 {
        const unsigned char *p, *pend;
@@ -260,86 +276,273 @@ static void
 openmetrics_export_submetric(struct openmetrics_request *req, string_t *out,
                             const struct metric *metric, int64_t timestamp)
 {
-       if (!openmetrics_check_name(metric->sub_name))
-               return;
        str_append_c(req->labels, '"');
        openmetrics_escape_string(req->labels, metric->sub_name);
        str_append_c(req->labels, '"');
 
        openmetrics_export_metric_value(req, out, metric, timestamp);
 
-       size_t label_pos = str_len(req->labels);
-       openmetrics_export_submetrics(req, out, metric, timestamp);
-       str_truncate(req->labels, label_pos);
-
        req->has_submetric = TRUE;
 }
 
-static void
-openmetrics_export_submetrics(struct openmetrics_request *req, string_t *out,
-                             const struct metric *metric, int64_t timestamp)
+static const struct metric *
+openmetrics_export_sub_metric_get(struct openmetrics_request_sub_metric *reqsm)
 {
        struct metric *const *sub_metric;
-       if (!array_is_created(&metric->sub_metrics))
-               return;
+
+       /* Get the first valid sub-metric */
+
+       if (reqsm->sub_index >= array_count(&reqsm->metric->sub_metrics))
+               return NULL;
+
+       sub_metric = array_idx(&reqsm->metric->sub_metrics, reqsm->sub_index);
+       while (((*sub_metric)->group_by == NULL ||
+               !openmetrics_check_name((*sub_metric)->group_by->field)) &&
+              ++reqsm->sub_index < array_count(&reqsm->metric->sub_metrics)) {
+               sub_metric = array_idx(&reqsm->metric->sub_metrics,
+                                      reqsm->sub_index);
+       }
+       if (reqsm->sub_index == array_count(&reqsm->metric->sub_metrics))
+               return NULL;
+
+       return *sub_metric;
+}
+
+static const struct metric *
+openmetrics_export_sub_metric_get_next(
+       struct openmetrics_request_sub_metric *reqsm)
+{
+       /* Get the next valid sub-metric */
+       reqsm->sub_index++;
+       return openmetrics_export_sub_metric_get(reqsm);
+}
+
+static struct openmetrics_request_sub_metric *
+openmetrics_export_sub_metric_down(struct openmetrics_request *req)
+{
+       struct openmetrics_request_sub_metric *reqsm =
+               array_back_modifiable(&req->sub_metric_stack);
+       const struct metric *sub_metric;
+
+       /* Descend further into sub-metric tree */
+
+       if (reqsm->metric->group_by == NULL ||
+           !array_is_created(&reqsm->metric->sub_metrics) ||
+           array_count(&reqsm->metric->sub_metrics) == 0)
+               return NULL;
+
+       /* Find sub-metric to descend into */
+       sub_metric = openmetrics_export_sub_metric_get(reqsm);
+       if (sub_metric == NULL) {
+               /* None valid */
+               return NULL;
+       }
+
        if (str_len(req->labels) > 0)
                str_append_c(req->labels, ',');
-       str_append(req->labels, metric->group_by->field);
+       str_append(req->labels, reqsm->metric->group_by->field);
        str_append_c(req->labels, '=');
-       array_foreach(&metric->sub_metrics, sub_metric) {
-               size_t label_pos = str_len(req->labels);
-               openmetrics_export_submetric(req, out, *sub_metric, timestamp);
-               str_truncate(req->labels, label_pos);
+       reqsm->labels_pos = str_len(req->labels);
+
+       /* Descend */
+       reqsm = array_append_space(&req->sub_metric_stack);
+       reqsm->metric = sub_metric;
+
+       return reqsm;
+}
+
+static struct openmetrics_request_sub_metric *
+openmetrics_export_sub_metric_up_next(struct openmetrics_request *req)
+{
+       struct openmetrics_request_sub_metric *reqsm;
+       const struct metric *sub_metric = NULL;
+
+       /* Ascend to next sub-metric of an ancestor */
+
+       while (array_count(&req->sub_metric_stack) > 1) {
+               /* Ascend */
+               array_pop_back(&req->sub_metric_stack);
+               reqsm = array_back_modifiable(&req->sub_metric_stack);
+               str_truncate(req->labels, reqsm->labels_pos);
+
+               /* Find next sub-metric */
+               sub_metric = openmetrics_export_sub_metric_get_next(reqsm);
+               if (sub_metric != NULL) {
+                       /* None valid */
+                       break;
+               }
+       }
+       if (sub_metric == NULL) {
+               /* End of sub-metric tree */
+               return NULL;
        }
+
+       /* Descend */
+       reqsm = array_append_space(&req->sub_metric_stack);
+       reqsm->metric = sub_metric;
+       return reqsm;
 }
 
-static void
-openmetrics_export_metric(struct openmetrics_request *req, string_t *out,
-                         int64_t timestamp)
+static struct openmetrics_request_sub_metric *
+openmetrics_export_sub_metric_current(struct openmetrics_request *req)
 {
-       const struct metric *metric = req->metric;
+       struct openmetrics_request_sub_metric *reqsm;
 
-       if (!openmetrics_check_metric(metric))
-               return;
+       /* Get state for current sub-metric */
 
-       req->labels = t_str_new(32);
-       size_t label_pos;
-       openmetrics_export_metric_labels(req->labels, metric);
+       if (!array_is_created(&req->sub_metric_stack))
+               i_array_init(&req->sub_metric_stack, 8);
+       if (array_count(&req->sub_metric_stack) >= 2) {
+               /* Already walking the sub-metric tree */
+               return array_back_modifiable(&req->sub_metric_stack);
+       }
 
-       /* Export count output */
+       /* Start tree walking */
 
-       req->metric_type = OPENMETRICS_METRIC_TYPE_COUNT;
-       openmetrics_export_metric_header(req, out);
+       reqsm = array_append_space(&req->sub_metric_stack);
+       reqsm->metric = req->metric;
+       reqsm->labels_pos = str_len(req->labels);
 
-       /* Put all sub-metrics before the actual value */
-       req->has_submetric = FALSE;
-       label_pos = str_len(req->labels);
-       openmetrics_export_submetrics(req, out, metric, timestamp);
-       str_truncate(req->labels, label_pos);
+       return openmetrics_export_sub_metric_down(req);
+}
 
-       if (!req->has_submetric)
-               openmetrics_export_metric_value(req, out, metric, timestamp);
+static bool
+openmetrics_export_sub_metrics(struct openmetrics_request *req, string_t *out,
+                              int64_t timestamp)
+{
+       struct openmetrics_request_sub_metric *reqsm = NULL;
 
-       /* Export duration output */
+       if (!array_is_created(&req->metric->sub_metrics))
+               return TRUE;
 
-       req->metric_type = OPENMETRICS_METRIC_TYPE_DURATION;
-       openmetrics_export_metric_header(req, out);
+       reqsm = openmetrics_export_sub_metric_current(req);
+       if (reqsm == NULL) {
+               /* No valid sub-metrics to export */
+               return TRUE;
+       }
+       openmetrics_export_submetric(req, out, reqsm->metric, timestamp);
+
+       /* Try do descend into sub-metrics tree for next sub-metric to export.
+        */
+       reqsm = openmetrics_export_sub_metric_down(req);
+       if (reqsm == NULL) {
+               /* Sub-metrics of this metric exhausted; ascend to the next
+                  parent sub-metric.
+                */
+               reqsm = openmetrics_export_sub_metric_up_next(req);
+       }
 
-       /* Put all sub-metrics before the actual value */
-       req->has_submetric = FALSE;
-       openmetrics_export_submetrics(req, out, metric, timestamp);
-       str_truncate(req->labels, label_pos);
+       if (reqsm == NULL) {
+               /* Finished */
+               array_clear(&req->sub_metric_stack);
+               return TRUE;
+       }
+       return FALSE;
+}
 
-       if (!req->has_submetric)
-               openmetrics_export_metric_value(req, out, metric, timestamp);
+static void
+openmetrics_export_metric_body(struct openmetrics_request *req, string_t *out,
+                              int64_t timestamp)
+{
+       openmetrics_export_metric_value(req, out, req->metric, timestamp);
+}
+
+static void openmetrics_export_next(struct openmetrics_request *req)
+{
+       /* Determine what to export next. */
+       switch (req->metric_type) {
+       case OPENMETRICS_METRIC_TYPE_COUNT:
+               /* Continue with duration output for this metric. */
+               req->metric_type = OPENMETRICS_METRIC_TYPE_DURATION;
+               req->state = OPENMETRICS_REQUEST_STATE_METRIC_HEADER;
+               break;
+       case OPENMETRICS_METRIC_TYPE_DURATION:
+               /* Continue with next metric */
+               req->state = OPENMETRICS_REQUEST_STATE_METRIC;
+               break;
+       }
+}
+
+static void openmetrics_export_continue(struct openmetrics_request *req,
+                                       string_t *out, int64_t timestamp)
+{
+       switch (req->state) {
+       case OPENMETRICS_REQUEST_STATE_INIT:
+               /* Export the Dovecot base metrics. */
+               i_assert(req->stats_iter == NULL);
+               req->stats_iter = stats_metrics_iterate_init(stats_metrics);
+               openmetrics_export_dovecot(out, timestamp);
+               req->state = OPENMETRICS_REQUEST_STATE_METRIC;
+               break;
+       case OPENMETRICS_REQUEST_STATE_METRIC:
+               /* Export the next metric. */
+               i_assert(req->stats_iter != NULL);
+               do {
+                       req->metric = stats_metrics_iterate(req->stats_iter);
+               } while (req->metric != NULL &&
+                        !openmetrics_check_metric(req->metric));
+               if (req->metric == NULL) {
+                       /* Finished exporting metrics. */
+                       req->state = OPENMETRICS_REQUEST_STATE_FINISHED;
+                       break;
+               }
+
+               if (req->labels == NULL)
+                       req->labels = str_new(default_pool, 32);
+               else
+                       str_truncate(req->labels, 0);
+               openmetrics_export_metric_labels(req->labels, req->metric);
+               req->labels_pos = str_len(req->labels);
+
+               /* Start with count output for this metric. */
+               req->metric_type = OPENMETRICS_METRIC_TYPE_COUNT;
+               req->state = OPENMETRICS_REQUEST_STATE_METRIC_HEADER;
+               /* Fall through */
+       case OPENMETRICS_REQUEST_STATE_METRIC_HEADER:
+               /* Export the HELP/TYPE header for the current metric */
+               str_truncate(req->labels, req->labels_pos);
+               req->has_submetric = FALSE;
+               openmetrics_export_metric_header(req, out);
+               req->state = OPENMETRICS_REQUEST_STATE_SUB_METRICS;
+               break;
+       case OPENMETRICS_REQUEST_STATE_SUB_METRICS:
+               /* Export the sub-metrics for the current metric. This will
+                  return for each sub-metric, so that the out string buffer
+                  stays small. */
+               if (!openmetrics_export_sub_metrics(req, out, timestamp))
+                       break;
+               /* All sub-metrics written. */
+               if (!req->has_submetric) {
+                       /* No sub-metrics; write the top-level metric body */
+                       req->state = OPENMETRICS_REQUEST_STATE_METRIC_BODY;
+               } else {
+                       /* Sub-metrics present; skip the top-level metric body
+                        */
+                       openmetrics_export_next(req);
+               }
+               break;
+       case OPENMETRICS_REQUEST_STATE_METRIC_BODY:
+               /* Export the body of the current metric. */
+               str_truncate(req->labels, req->labels_pos);
+               openmetrics_export_metric_body(req, out, timestamp);
+               openmetrics_export_next(req);
+               break;
+       case OPENMETRICS_REQUEST_STATE_FINISHED:
+               i_unreached();
+       }
+}
+
+static void openmetrics_request_deinit(struct openmetrics_request *req)
+{
+       stats_metrics_iterate_deinit(&req->stats_iter);
+       str_free(&req->labels);
+       array_free(&req->sub_metric_stack);
 }
 
 static void
 openmetrics_export(struct openmetrics_request *req,
                   struct http_server_response *resp)
 {
-       struct stats_metrics_iter *iter;
-       const struct metric *metric;
        string_t *out = t_str_new(2048);
        int64_t timestamp;
 
@@ -347,14 +550,10 @@ openmetrics_export(struct openmetrics_request *req,
        timestamp = ((int64_t)ioloop_timeval.tv_sec * 1000 +
                     (int64_t)ioloop_timeval.tv_usec / 1000);
 
-       openmetrics_export_dovecot(out, timestamp);
-       
-       iter = stats_metrics_iterate_init(stats_metrics);
-       while ((metric = stats_metrics_iterate(iter)) != NULL) {
-               req->metric = metric;
-               openmetrics_export_metric(req, out, timestamp);
-       }
-       stats_metrics_iterate_deinit(&iter);
+       while (req->state != OPENMETRICS_REQUEST_STATE_FINISHED)
+               openmetrics_export_continue(req, out, timestamp);
+
+       openmetrics_request_deinit(req);
 
        http_server_response_set_payload_data(
                resp, str_data(out), str_len(out));