From e1ae66474664e879517cb46871786d05dffad3f7 Mon Sep 17 00:00:00 2001 From: "Mozejko, MarcinX" Date: Wed, 18 Apr 2018 09:08:17 +0100 Subject: [PATCH] logparser plugin: Plugin used to parse log files using regexes utils_tail, utils_tail_match: Change _Bool to bool. utils_message_parser: Correct struct and type names, cleanup, unit tests. Change-Id: I40f3336e274f01791e7c171db5be2f21a6c9217c Signed-off-by: Mozejko, MarcinX --- Makefile.am | 27 ++ configure.ac | 2 + src/collectd.conf.in | 60 +++ src/collectd.conf.pod | 152 +++++++ src/logparser.c | 694 ++++++++++++++++++++++++++++++++ src/utils/tail/tail.c | 8 +- src/utils/tail/tail.h | 4 +- src/utils_message_parser.c | 149 ++++--- src/utils_message_parser.h | 47 ++- src/utils_message_parser_test.c | 451 +++++++++++++++++++++ src/utils_tail_match.c | 2 +- src/utils_tail_match.h | 2 +- 12 files changed, 1503 insertions(+), 95 deletions(-) create mode 100644 src/logparser.c create mode 100644 src/utils_message_parser_test.c diff --git a/Makefile.am b/Makefile.am index 258603f91..b14d50cfa 100644 --- a/Makefile.am +++ b/Makefile.am @@ -155,6 +155,7 @@ check_PROGRAMS = \ test_utils_cmds \ test_utils_heap \ test_utils_latency \ + test_utils_message_parser \ test_utils_mount \ test_utils_subst \ test_utils_time \ @@ -360,6 +361,19 @@ test_utils_heap_SOURCES = \ src/testing.h test_utils_heap_LDADD = libheap.la $(COMMON_LIBS) +test_utils_message_parser_SOURCES = \ + src/utils_message_parser_test.c \ + src/testing.h \ + src/daemon/configfile.c \ + src/daemon/types_list.c \ + src/utils_tail_match.c src/utils_tail_match.h \ + src/utils_tail.c src/utils_tail.h \ + src/utils_match.c src/utils_match.h \ + src/utils_latency.c src/utils_latency.h \ + src/utils_latency_config.c src/utils_latency_config.h +test_utils_message_parser_CPPFLAGS = $(AM_CPPFLAGS) +test_utils_message_parser_LDADD = liboconfig.la libplugin_mock.la -lm + test_utils_time_SOURCES = \ src/daemon/utils_time_test.c \ src/testing.h @@ -1200,6 +1214,19 @@ logfile_la_LDFLAGS = $(PLUGIN_LDFLAGS) logfile_la_DEPENDENCIES = $(COMMON_DEPS) endif +if BUILD_PLUGIN_LOGPARSER +pkglib_LTLIBRARIES += logparser.la +logparser_la_SOURCES = src/logparser.c \ + src/utils_message_parser.c src/utils_message_parser.h \ + src/utils_tail_match.c src/utils_tail_match.h \ + src/utils_tail.c src/utils_tail.h \ + src/utils_match.c src/utils_match.h \ + src/utils_latency.c src/utils_latency.h \ + src/utils_latency_config.c src/utils_latency_config.h +logparser_la_CPPFLAGS = $(AM_CPPFLAGS) +logparser_la_LDFLAGS = $(PLUGIN_LDFLAGS) -lm +endif + if BUILD_PLUGIN_LOG_LOGSTASH pkglib_LTLIBRARIES += log_logstash.la log_logstash_la_SOURCES = src/log_logstash.c diff --git a/configure.ac b/configure.ac index 9fc40b17d..efa440a39 100644 --- a/configure.ac +++ b/configure.ac @@ -6788,6 +6788,7 @@ AC_PLUGIN([java], [$with_java], [Embed the Java Vi AC_PLUGIN([load], [$plugin_load], [System load]) AC_PLUGIN([log_logstash], [$plugin_log_logstash], [Logstash json_event compatible logging]) AC_PLUGIN([logfile], [yes], [File logging plugin]) +AC_PLUGIN([logparser], [yes], [Log parsing plugin]) AC_PLUGIN([lpar], [$with_perfstat], [AIX logical partitions statistics]) AC_PLUGIN([lua], [$with_liblua], [Lua plugin]) AC_PLUGIN([madwifi], [$have_linux_wireless_h], [Madwifi wireless statistics]) @@ -7218,6 +7219,7 @@ AC_MSG_RESULT([ irq . . . . . . . . . $enable_irq]) AC_MSG_RESULT([ java . . . . . . . . $enable_java]) AC_MSG_RESULT([ load . . . . . . . . $enable_load]) AC_MSG_RESULT([ logfile . . . . . . . $enable_logfile]) +AC_MSG_RESULT([ logparser . . . . . . $enable_logparser]) AC_MSG_RESULT([ log_logstash . . . . $enable_log_logstash]) AC_MSG_RESULT([ lpar . . . . . . . . $enable_lpar]) AC_MSG_RESULT([ lua . . . . . . . . . $enable_lua]) diff --git a/src/collectd.conf.in b/src/collectd.conf.in index 63db8b1a3..ab57cca72 100644 --- a/src/collectd.conf.in +++ b/src/collectd.conf.in @@ -143,6 +143,7 @@ #@BUILD_PLUGIN_JAVA_TRUE@LoadPlugin java @BUILD_PLUGIN_LOAD_TRUE@@BUILD_PLUGIN_LOAD_TRUE@LoadPlugin load #@BUILD_PLUGIN_LPAR_TRUE@LoadPlugin lpar +#@BUILD_PLUGIN_LOGPARSER_TRUE@LoadPlugin logparser #@BUILD_PLUGIN_LUA_TRUE@LoadPlugin lua #@BUILD_PLUGIN_MADWIFI_TRUE@LoadPlugin madwifi #@BUILD_PLUGIN_MBMON_TRUE@LoadPlugin mbmon @@ -793,6 +794,65 @@ # ReportBySerial false # +# +# +# FirstFullRead false +# +# DefaultType "pcie_error" +# DefaultSeverity "warning" +# +# Regex "AER:.*error received" +# SubmatchIdx -1 +# +# +# Regex "(... .. ..:..:..) .* pcieport.*AER" +# SubmatchIdx 1 +# IsMandatory false +# +# +# Regex "pcieport (.*): AER:" +# SubmatchIdx 1 +# IsMandatory true +# +# +# PluginInstance true +# Regex " ([0-9a-fA-F:\\.]*): PCIe Bus Error" +# SubmatchIdx 1 +# IsMandatory false +# +# +# Regex "severity=" +# SubMatchIdx -1 +# +# +# Regex "severity=.*\\([nN]on-[fF]atal" +# TypeInstance "non_fatal" +# IsMandatory false +# +# +# Regex "severity=.*\\([fF]atal" +# Severity "failure" +# TypeInstance "fatal" +# IsMandatory false +# +# +# Regex "severity=Corrected" +# TypeInstance "correctable" +# IsMandatory false +# +# +# Regex "type=(.*)," +# SubmatchIdx 1 +# IsMandatory false +# +# +# Regex ", id=(.*)" +# SubmatchIdx 1 +# +# +# +# + # # BasePath "@prefix@/share/@PACKAGE_NAME@/lua" # Script "script1.lua" diff --git a/src/collectd.conf.pod b/src/collectd.conf.pod index cda1002c5..25e0f8e62 100644 --- a/src/collectd.conf.pod +++ b/src/collectd.conf.pod @@ -3982,6 +3982,158 @@ B: There is no need to notify the daemon after moving or removing the log file (e.Eg. when rotating the logs). The plugin reopens the file for each line it writes. +=head2 Plugin C + +The I plugin is used to parse different kinds of logs. Setting proper +options you can choose strings to collect. Plugin searches the log file for +messages which contain several matches (two or more). When all mandatory matches +are found then it sends proper notification containing all fetched values. + +B + + + + FirstFullRead false + + DefaultType "pcie_error" + DefaultSeverity "warning" + + Regex "AER:.*error received" + SubmatchIdx -1 + + + Regex "(... .. ..:..:..) .* pcieport.*AER" + SubmatchIdx 1 + IsMandatory false + + + Regex "pcieport (.*): AER:" + SubmatchIdx 1 + IsMandatory true + + + PluginInstance true + Regex " ([0-9a-fA-F:\\.]*): PCIe Bus Error" + SubmatchIdx 1 + IsMandatory false + + + Regex "severity=" + SubMatchIdx -1 + + + Regex "severity=.*\\([nN]on-[fF]atal" + TypeInstance "non_fatal" + IsMandatory false + + + Regex "severity=.*\\([fF]atal" + Severity "failure" + TypeInstance "fatal" + IsMandatory false + + + Regex "severity=Corrected" + TypeInstance "correctable" + IsMandatory false + + + Regex "type=(.*)," + SubmatchIdx 1 + IsMandatory false + + + Regex ", id=(.*)" + SubmatchIdx 1 + + + + + +B + +=over 4 + +=item B I + +The B block defines file to search. It may contain one or more +B blocks which are defined below. + +=item B I|I + +Set to true if want to read the file from the beginning first time. + +=item B I + +B block contains matches to search the log file for. + +=item B I + +Sets the default plugin instance. + +=item B I + +Sets the default type. + +=item B I + +Sets the default type instance. + +=item B I + +Sets the default severity. Must be set to "OK", "WARNING" or "FAILURE". +Default value is "OK". + +=item B I + +Multiple I blocks define regular expression patterns for extracting or +excluding specific string patterns from parsing. First and last I items +in the same I set boundaries of multiline message and are mandatory. +If these matches are not found then the whole message is discarded. + +=item B I + +Regular expression with pattern matching string. It may contain subexpressions, +so next option B specifies which subexpression should be stored. + +=item B I + +Index of subexpression to be used for notification. Multiple subexpressions are +allowed. Index value 0 takes whole regular expression match as a result. +Index value -1 does not add result to message item. Can be omitted, default +value is 0. + +=item B I + +Regular expression for excluding lines containing specific matching strings. +This is processed before checking I pattern. It is optional and can +be omitted. + +=item B I|I + +Flag indicating if I item is mandatory for message validation. If set to +true, whole message is discarded if it's missing. For false its presence is +optional. Default value is set to true. + +=item B I|I + +If set to true, it sets plugin instance to string returned by regex. It can be +overridden by user string. + +=item B I|I + +Sets notification type using rules like B. + +=item B I|I + +Sets notification type instance using rules like above. + +=item B I + +Sets notification severity to one of the options: "OK", "WARNING", "FAILURE". + +=back + =head2 Plugin C The I behaves like the logfile plugin but formats diff --git a/src/logparser.c b/src/logparser.c new file mode 100644 index 000000000..056aae34d --- /dev/null +++ b/src/logparser.c @@ -0,0 +1,694 @@ +/** + * collectd - src/logparser.c + * + * Copyright(c) 2018 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Authors: + * Marcin Mozejko + **/ + +#include "collectd.h" + +#include "common.h" +#include "utils_llist.h" +#include "utils_message_parser.h" + +#define PLUGIN_NAME "logparser" + +#define LOGPARSER_SEV_OK_STR "OK" +#define LOGPARSER_SEV_WARN_STR "WARNING" +#define LOGPARSER_SEV_FAIL_STR "FAILURE" + +#define LOGPARSER_PLUGIN_INST_STR "PluginInstance" +#define LOGPARSER_TYPE_STR "Type" +#define LOGPARSER_TYPE_INST_STR "TypeInstance" +#define LOGPARSER_SEVERITY_STR "Severity" + +#define MAX_STR_LEN 128 +#define MAX_FIELDS 4 /* PluginInstance, Type, TypeInstance, Severity */ + +#define START_IDX 0 +#define STOP_IDX (parser->patterns_len - 1) + +typedef enum message_item_type_e { + MSG_ITEM_PLUGIN_INST = 0, + MSG_ITEM_TYPE, + MSG_ITEM_TYPE_INST, + MSG_ITEM_SEVERITY +} message_item_type_t; + +typedef struct message_item_info_s { + /* Type of message item used for special processing */ + message_item_type_t type; + union { + /* If set, will override message item string with this one */ + char *str_override; + /* Used only if type is MSG_ITEM_SEVERITY */ + int severity; + } val; +} message_item_info_t; + +typedef struct message_item_user_data_s { + /* Information telling what to do when match found */ + message_item_info_t infos[MAX_FIELDS]; + size_t infos_len; +} message_item_user_data_t; + +typedef struct log_parser_s { + char *name; + parser_job_data_t *job; + message_pattern_t *patterns; + size_t patterns_len; + bool first_read; + char *filename; + char *def_plugin_inst; + char *def_type; + char *def_type_inst; + int def_severity; +} log_parser_t; + +typedef struct logparser_ctx_s { + log_parser_t *parsers; + size_t parsers_len; +} logparser_ctx_t; + +static logparser_ctx_t logparser_ctx; + +static int logparser_shutdown(void); + +static void logparser_free_user_data(void *data) { + message_item_user_data_t *user_data = (message_item_user_data_t *)data; + + if (user_data == NULL) + return; + + for (size_t i = 0; i < user_data->infos_len; i++) { + if (user_data->infos[i].type != MSG_ITEM_SEVERITY) + sfree(user_data->infos[i].val.str_override); + } + sfree(user_data); +} + +static int logparser_config_msg_item_type(oconfig_item_t *ci, + message_item_user_data_t **user_data, + message_item_type_t type) { + bool val; + int ret; + char *str = NULL; + + if (*user_data == NULL) { + *user_data = calloc(1, sizeof(**user_data)); + if (*user_data == NULL) { + ERROR(PLUGIN_NAME ": Could not allocate memory"); + return -1; + } + (*user_data)->infos_len = 0; + } + + size_t i = (*user_data)->infos_len; + + switch (ci->values[0].type) { + case OCONFIG_TYPE_STRING: + ret = cf_util_get_string(ci, &str); + break; + case OCONFIG_TYPE_BOOLEAN: + ret = cf_util_get_boolean(ci, &val); + if (val == false || type == MSG_ITEM_SEVERITY) + goto wrong_value; + break; + default: + ERROR(PLUGIN_NAME ": Wrong type for option %s", ci->key); + goto error; + } + + if (ret != 0) { + ERROR(PLUGIN_NAME ": Error getting %s option", ci->key); + goto error; + } + + if (type == MSG_ITEM_SEVERITY) { + if (strcasecmp(LOGPARSER_SEV_OK_STR, str) == 0) + (*user_data)->infos[i].val.severity = NOTIF_OKAY; + else if (strcasecmp(LOGPARSER_SEV_WARN_STR, str) == 0) + (*user_data)->infos[i].val.severity = NOTIF_WARNING; + else if (strcasecmp(LOGPARSER_SEV_FAIL_STR, str) == 0) + (*user_data)->infos[i].val.severity = NOTIF_FAILURE; + else { + sfree(str); + goto wrong_value; + } + sfree(str); + } else + (*user_data)->infos[i].val.str_override = str; + + (*user_data)->infos[i].type = type; + (*user_data)->infos_len++; + return 0; + +wrong_value: + ERROR(PLUGIN_NAME ": Wrong value for option %s", ci->key); +error: + sfree(*user_data); + return -1; +} + +static int logparser_config_match(oconfig_item_t *ci, log_parser_t *parser) { + int ret; + message_item_user_data_t *user_data = NULL; + message_pattern_t *ptr = NULL; + + ptr = realloc(parser->patterns, sizeof(*ptr) * (parser->patterns_len + 1)); + if (ptr == NULL) { + ERROR(PLUGIN_NAME ": Error reallocating memory for message patterns."); + return -1; + } + + message_pattern_t *pattern = ptr + parser->patterns_len; + + memset(pattern, 0, sizeof(*pattern)); + pattern->is_mandatory = 1; + ret = cf_util_get_string(ci, &pattern->name); + + if (ret != 0) { + ERROR(PLUGIN_NAME ": Error getting match name"); + goto free_ptr; + } + + for (int i = 0; i < ci->children_num; i++) { + oconfig_item_t *child = ci->children + i; + + if (strcasecmp("Regex", child->key) == 0) + ret = cf_util_get_string(child, &pattern->regex); + else if (strcasecmp("SubmatchIdx", child->key) == 0) + ret = cf_util_get_int(child, &pattern->submatch_idx); + else if (strcasecmp("ExcludeRegex", child->key) == 0) + ret = cf_util_get_string(child, &pattern->excluderegex); + else if (strcasecmp("IsMandatory", child->key) == 0) + ret = cf_util_get_boolean(child, &pattern->is_mandatory); + else if (strcasecmp(LOGPARSER_PLUGIN_INST_STR, child->key) == 0) + ret = logparser_config_msg_item_type(child, &user_data, + MSG_ITEM_PLUGIN_INST); + else if (strcasecmp(LOGPARSER_TYPE_STR, child->key) == 0) + ret = logparser_config_msg_item_type(child, &user_data, MSG_ITEM_TYPE); + else if (strcasecmp(LOGPARSER_TYPE_INST_STR, child->key) == 0) + ret = + logparser_config_msg_item_type(child, &user_data, MSG_ITEM_TYPE_INST); + else if (strcasecmp(LOGPARSER_SEVERITY_STR, child->key) == 0) + ret = + logparser_config_msg_item_type(child, &user_data, MSG_ITEM_SEVERITY); + else { + ERROR(PLUGIN_NAME ": Invalid configuration option \"%s\".", child->key); + goto free_user_data; + } + if (ret != 0) { + ERROR(PLUGIN_NAME ": Error getting %s option", child->key); + goto free_user_data; + } + } + + if (user_data != NULL) { + pattern->user_data = (void *)user_data; + pattern->free_user_data = logparser_free_user_data; + } + + parser->patterns = ptr; + parser->patterns_len++; + + return 0; + +free_user_data: + if (pattern->user_data != NULL) + pattern->free_user_data(pattern->user_data); +free_ptr: + sfree(ptr); + return -1; +} + +static int logparser_config_message(const oconfig_item_t *ci, char *filename, + bool first_read) { + char *msg_name = NULL; + char *severity = NULL; + int ret; + log_parser_t *ptr; + + ret = cf_util_get_string(ci, &msg_name); + if (ret != 0) { + ERROR(PLUGIN_NAME ": Error getting message name"); + return -1; + } + + ptr = realloc(logparser_ctx.parsers, + sizeof(*ptr) * (logparser_ctx.parsers_len + 1)); + + if (ptr == NULL) { + ERROR(PLUGIN_NAME ": Error reallocating memory for message parsers."); + goto error; + } + + logparser_ctx.parsers = ptr; + + log_parser_t *parser = ptr + logparser_ctx.parsers_len; + + memset(parser, 0, sizeof(*parser)); + parser->name = msg_name; + parser->first_read = first_read; + parser->filename = filename; + parser->def_severity = NOTIF_OKAY; + + for (int i = 0; i < ci->children_num; i++) { + oconfig_item_t *child = ci->children + i; + if (strcasecmp("Match", child->key) == 0) + ret = logparser_config_match(child, parser); + else if (strcasecmp("DefaultPluginInstance", child->key) == 0) + ret = cf_util_get_string(child, &parser->def_plugin_inst); + else if (strcasecmp("DefaultType", child->key) == 0) + ret = cf_util_get_string(child, &parser->def_type); + else if (strcasecmp("DefaultTypeInstance", child->key) == 0) + ret = cf_util_get_string(child, &parser->def_type_inst); + else if (strcasecmp("DefaultSeverity", child->key) == 0) { + ret = cf_util_get_string(child, &severity); + if (strcasecmp(LOGPARSER_SEV_OK_STR, severity) == 0) + parser->def_severity = NOTIF_OKAY; + else if (strcasecmp(LOGPARSER_SEV_WARN_STR, severity) == 0) + parser->def_severity = NOTIF_WARNING; + else if (strcasecmp(LOGPARSER_SEV_FAIL_STR, severity) == 0) + parser->def_severity = NOTIF_FAILURE; + else { + ERROR(PLUGIN_NAME ": Invalid severity value: \"%s\".", severity); + sfree(severity); + goto error; + } + sfree(severity); + } else { + ERROR(PLUGIN_NAME ": Invalid configuration option \"%s\".", child->key); + goto error; + } + if (ret != 0) { + ERROR(PLUGIN_NAME ": Error getting %s option", child->key); + goto error; + } + } + logparser_ctx.parsers_len++; + + return 0; + +error: + sfree(msg_name); + return -1; +} + +static int logparser_config_logfile(oconfig_item_t *ci) { + char *filename = NULL; + bool first_read = false; // First full read + int ret = 0; + + ret = cf_util_get_string(ci, &filename); + if (ret != 0) { + ERROR(PLUGIN_NAME ": Error getting filename"); + return -1; + } + + for (int i = 0; i < ci->children_num; i++) { + oconfig_item_t *child = ci->children + i; + if (strcasecmp("FirstFullRead", child->key) == 0) + ret = cf_util_get_boolean(child, &first_read); + else if (strcasecmp("Message", child->key) == 0) + ret = logparser_config_message(child, filename, first_read); + else { + ERROR(PLUGIN_NAME ": Invalid configuration option \"%s\".", child->key); + goto error; + } + + if (ret != 0) { + ERROR(PLUGIN_NAME ": Error getting %s option", child->key); + goto error; + } + } + + return 0; + +error: + sfree(filename); + return -1; +} + +static int logparser_validate_config(void) { + for (size_t i = 0; i < logparser_ctx.parsers_len; i++) { + log_parser_t *parser = logparser_ctx.parsers + i; + + if (strlen(parser->filename) < 1) { + ERROR(PLUGIN_NAME ": Log filename in \"%s\" message can't be empty", + parser->name); + return -1; + } + + if (parser->def_plugin_inst != NULL && + strlen(parser->def_plugin_inst) < 1) { + ERROR(PLUGIN_NAME + ": DefaultPluginInstance in \"%s\" message can't be empty", + parser->name); + return -1; + } + + if (parser->def_type != NULL && strlen(parser->def_type) < 1) { + ERROR(PLUGIN_NAME ": DefaultType in \"%s\" message can't be empty", + parser->name); + return -1; + } + + if (parser->def_type_inst != NULL && strlen(parser->def_type_inst) < 1) { + ERROR(PLUGIN_NAME + ": DefaultTypeInstance in \"%s\" message can't be empty", + parser->name); + return -1; + } + + if (parser->patterns_len < 2) { + ERROR(PLUGIN_NAME ": Message \"%s\" should have at least 2 matches", + parser->name); + return -1; + } + + if (parser->patterns[START_IDX].is_mandatory != 1) { + ERROR(PLUGIN_NAME + ": Start match \"%s\" in message \"%s\" can't be optional", + parser->patterns[START_IDX].name, parser->name); + return -1; + } + + if (parser->patterns[STOP_IDX].is_mandatory != 1) { + ERROR(PLUGIN_NAME + ": Stop match \"%s\" in message \"%s\" can't be optional", + parser->patterns[STOP_IDX].name, parser->name); + return -1; + } + + for (int j = 0; j < parser->patterns_len; j++) { + message_pattern_t *pattern = parser->patterns + j; + + if (pattern->regex == NULL) { + ERROR(PLUGIN_NAME + ": Regex must be set (message: \"%s\", match: \"%s\")", + parser->name, pattern->name); + return -1; + } else if (strlen(pattern->regex) < 1) { + ERROR(PLUGIN_NAME + ": Regex can't be empty (message: \"%s\", match: \"%s\")", + parser->name, pattern->name); + return -1; + } + + if (pattern->excluderegex != NULL && strlen(pattern->excluderegex) < 1) { + ERROR(PLUGIN_NAME + ": ExcludeRegex can't be empty (message: \"%s\", match: \"%s\")", + parser->name, pattern->name); + return -1; + } + + if (pattern->submatch_idx < -1) { + ERROR(PLUGIN_NAME ": SubmatchIdx must be in range [-1..n]"); + return -1; + } + + if (pattern->user_data != NULL && pattern->submatch_idx == -1) + WARNING(PLUGIN_NAME ": Options [PluginInstance, Type, TypeInstance, " + "Severity] are omitted when SubmatchIdx is set to " + "-1 (message: \"%s\", match: \"%s\")", + parser->name, pattern->name); + } + } + + return 0; +} + +static int logparser_config(oconfig_item_t *ci) { + int ret; + + for (int i = 0; i < ci->children_num; i++) { + oconfig_item_t *child = ci->children + i; + + if (strcasecmp("Logfile", child->key) == 0) { + ret = logparser_config_logfile(child); + + if (ret != 0) { + return -1; + } + } + } + + return logparser_validate_config(); +} + +#if COLLECT_DEBUG +static void logparser_print_config(void) { + const char *severity_desc[5] = {NULL, LOGPARSER_SEV_FAIL_STR, + LOGPARSER_SEV_WARN_STR, NULL, + LOGPARSER_SEV_OK_STR}; + const char *item_type_desc[MAX_FIELDS] = { + LOGPARSER_PLUGIN_INST_STR, LOGPARSER_TYPE_STR, LOGPARSER_TYPE_INST_STR, + LOGPARSER_SEVERITY_STR}; + const char *bool_str[2] = {"False", "True"}; + + DEBUG(PLUGIN_NAME ": ==========LOGPARSER CONFIG============="); + DEBUG(PLUGIN_NAME ": Message configs count: %zu", logparser_ctx.parsers_len); + + for (size_t i = 0; i < logparser_ctx.parsers_len; i++) { + log_parser_t *parser = logparser_ctx.parsers + i; + DEBUG(PLUGIN_NAME ": Message: \"%s\"", parser->name); + DEBUG(PLUGIN_NAME ": File: \"%s\"", parser->filename); + if (parser->def_plugin_inst != NULL) + DEBUG(PLUGIN_NAME ": DefaultPluginInstance: \"%s\"", + parser->def_plugin_inst); + if (parser->def_type != NULL) + DEBUG(PLUGIN_NAME ": DefaultType: \"%s\"", parser->def_type); + if (parser->def_type_inst != NULL) + DEBUG(PLUGIN_NAME ": DefaultTypeInstance: \"%s\"", + parser->def_type_inst); + DEBUG(PLUGIN_NAME ": DefaultSeverity: %s", + severity_desc[parser->def_severity]); + DEBUG(PLUGIN_NAME ": Match configs count: %zu", parser->patterns_len); + + for (size_t j = 0; j < parser->patterns_len; j++) { + message_pattern_t *pattern = parser->patterns + j; + DEBUG(PLUGIN_NAME ": Match: \"%s\"", pattern->name); + DEBUG(PLUGIN_NAME ": Regex: \"%s\"", pattern->regex); + if (pattern->excluderegex != NULL) + DEBUG(PLUGIN_NAME ": ExcludeRegex: \"%s\"", pattern->excluderegex); + DEBUG(PLUGIN_NAME ": SubmatchIdx: %d", pattern->submatch_idx); + DEBUG(PLUGIN_NAME ": IsMandatory: %s", + bool_str[pattern->is_mandatory]); + if (pattern->user_data != NULL) { + message_item_user_data_t *ud = + (message_item_user_data_t *)pattern->user_data; + for (size_t k = 0; k < ud->infos_len; k++) { + if (ud->infos[k].type == MSG_ITEM_SEVERITY) + DEBUG(PLUGIN_NAME ": Severity: %s", + severity_desc[ud->infos[k].val.severity]); + else { + if (ud->infos[k].val.str_override != NULL) + DEBUG(PLUGIN_NAME ": %s: \"%s\"", + item_type_desc[ud->infos[k].type], + ud->infos[k].val.str_override); + else + DEBUG(PLUGIN_NAME ": %s: %s", + item_type_desc[ud->infos[k].type], bool_str[1]); + } + } + } + } + } + DEBUG(PLUGIN_NAME ": ======================================="); +} +#endif + +static int logparser_init(void) { +#if COLLECT_DEBUG + logparser_print_config(); +#endif + + for (size_t i = 0; i < logparser_ctx.parsers_len; i++) { + log_parser_t *parser = logparser_ctx.parsers + i; + parser->job = message_parser_init(parser->filename, START_IDX, STOP_IDX, + parser->patterns, parser->patterns_len); + if (parser->job == NULL) { + ERROR(PLUGIN_NAME ": Failed to initialize %s parser.", + logparser_ctx.parsers[i].name); + logparser_shutdown(); + return -1; + } + } + + return 0; +} + +static void logparser_dispatch_notification(notification_t *n) { + sstrncpy(n->host, hostname_g, sizeof(n->host)); + plugin_dispatch_notification(n); + if (n->meta != NULL) + plugin_notification_meta_free(n->meta); +} + +static void logparser_process_msg(log_parser_t *parser, message_t *msg, + unsigned int max_items) { + notification_t n = {.severity = parser->def_severity, + .time = cdtime(), + .plugin = PLUGIN_NAME, + .meta = NULL}; + + /* Writing default values if set */ + if (parser->def_plugin_inst != NULL) + sstrncpy(n.plugin_instance, parser->def_plugin_inst, + sizeof(n.plugin_instance)); + if (parser->def_type != NULL) + sstrncpy(n.type, parser->def_type, sizeof(n.type)); + if (parser->def_type_inst != NULL) + sstrncpy(n.type_instance, parser->def_type_inst, sizeof(n.type_instance)); + + for (int i = 0; i < max_items; i++) { + message_item_t *item = msg->message_items + i; + if (!item->value[0]) + break; + + DEBUG(PLUGIN_NAME ": [%02d] %s:%s", i, item->name, item->value); + + if (item->user_data != NULL) { + message_item_user_data_t *user_data = + (message_item_user_data_t *)item->user_data; + + for (size_t i = 0; i < user_data->infos_len; i++) { + char *ptr = NULL; + size_t size; + switch (user_data->infos[i].type) { + case MSG_ITEM_SEVERITY: + n.severity = user_data->infos[i].val.severity; + break; + case MSG_ITEM_PLUGIN_INST: + ptr = n.plugin_instance; + size = sizeof(n.plugin_instance); + break; + case MSG_ITEM_TYPE: + ptr = n.type; + size = sizeof(n.type); + break; + case MSG_ITEM_TYPE_INST: + ptr = n.type_instance; + size = sizeof(n.type_instance); + break; + default: + ERROR(PLUGIN_NAME ": Message item has wrong type!"); + return; + } + + if (user_data->infos[i].type != MSG_ITEM_SEVERITY) { + if (user_data->infos[i].val.str_override != NULL) + sstrncpy(ptr, user_data->infos[i].val.str_override, size); + else + sstrncpy(ptr, item->value, size); + } + } + } + + if (plugin_notification_meta_add_string(&n, item->name, item->value)) + ERROR(PLUGIN_NAME ": Failed to add notification meta data %s:%s", + item->name, item->value); + } + + logparser_dispatch_notification(&n); +} + +static int logparser_parser_read(log_parser_t *parser) { + message_t *messages_storage; + unsigned int max_item_num; + int msg_num = + message_parser_read(parser->job, &messages_storage, parser->first_read); + + if (msg_num < 0) { + notification_t n = {.severity = NOTIF_FAILURE, + .time = cdtime(), + .message = "Failed to read from log file", + .plugin = PLUGIN_NAME, + .meta = NULL}; + logparser_dispatch_notification(&n); + return -1; + } + + max_item_num = STATIC_ARRAY_SIZE(messages_storage[0].message_items); + + DEBUG(PLUGIN_NAME ": read %d messages, %s", msg_num, parser->name); + + for (int i = 0; i < msg_num; i++) { + message_t *msg = messages_storage + i; + logparser_process_msg(parser, msg, max_item_num); + } + return 0; +} + +static int logparser_read(__attribute__((unused)) user_data_t *ud) { + int ret = 0; + + for (size_t i = 0; i < logparser_ctx.parsers_len; i++) { + log_parser_t *parser = logparser_ctx.parsers + i; + ret = logparser_parser_read(parser); + if (parser->first_read) + parser->first_read = false; + + if (ret < 0) { + ERROR(PLUGIN_NAME ": Failed to parse %s messages from %s", parser->name, + parser->filename); + break; + } + } + + return ret; +} + +static int logparser_shutdown(void) { + if (logparser_ctx.parsers == NULL) + return 0; + + for (size_t i = 0; i < logparser_ctx.parsers_len; i++) { + log_parser_t *parser = logparser_ctx.parsers + i; + + if (parser->job != NULL) + message_parser_cleanup(parser->job); + + for (size_t j = 0; j < parser->patterns_len; j++) + if (parser->patterns[j].free_user_data != NULL) + parser->patterns[j].free_user_data(parser->patterns[j].user_data); + + sfree(parser->patterns); + sfree(parser->filename); + sfree(parser->def_plugin_inst); + sfree(parser->def_type); + sfree(parser->def_type_inst); + } + + sfree(logparser_ctx.parsers); + + return 0; +} + +void module_register(void) { + plugin_register_complex_config(PLUGIN_NAME, logparser_config); + plugin_register_init(PLUGIN_NAME, logparser_init); + plugin_register_complex_read(NULL, PLUGIN_NAME, logparser_read, 0, NULL); + plugin_register_shutdown(PLUGIN_NAME, logparser_shutdown); +} diff --git a/src/utils/tail/tail.c b/src/utils/tail/tail.c index d87254095..088248587 100644 --- a/src/utils/tail/tail.c +++ b/src/utils/tail/tail.c @@ -41,7 +41,7 @@ struct cu_tail_s { struct stat stat; }; -static int cu_tail_reopen(cu_tail_t *obj, _Bool force_rewind) { +static int cu_tail_reopen(cu_tail_t *obj, bool force_rewind) { int seek_end = 0; struct stat stat_buf = {0}; @@ -72,7 +72,7 @@ static int cu_tail_reopen(cu_tail_t *obj, _Bool force_rewind) { * if we re-open the same file again or the file opened is the first at all * or the first after an error */ if ((obj->stat.st_ino == 0) || (obj->stat.st_ino == stat_buf.st_ino)) - seek_end = force_rewind ? 0 : 1; + seek_end = force_rewind ? false : true; FILE *fh = fopen(obj->file, "r"); if (fh == NULL) { @@ -124,7 +124,7 @@ int cu_tail_destroy(cu_tail_t *obj) { return 0; } /* int cu_tail_destroy */ -int cu_tail_readline(cu_tail_t *obj, char *buf, int buflen, _Bool force_rewind) { +int cu_tail_readline(cu_tail_t *obj, char *buf, int buflen, bool force_rewind) { int status; if (buflen < 1) { @@ -187,7 +187,7 @@ int cu_tail_readline(cu_tail_t *obj, char *buf, int buflen, _Bool force_rewind) } /* int cu_tail_readline */ int cu_tail_read(cu_tail_t *obj, char *buf, int buflen, tailfunc_t *callback, - void *data, _Bool force_rewind) { + void *data, bool force_rewind) { int status; while (42) { diff --git a/src/utils/tail/tail.h b/src/utils/tail/tail.h index b3aa07231..76f12ccf0 100644 --- a/src/utils/tail/tail.h +++ b/src/utils/tail/tail.h @@ -73,7 +73,7 @@ int cu_tail_destroy(cu_tail_t *obj); * * Returns 0 when successful and non-zero otherwise. */ -int cu_tail_readline(cu_tail_t *obj, char *buf, int buflen, _Bool force_rewind); +int cu_tail_readline(cu_tail_t *obj, char *buf, int buflen, bool force_rewind); /* * cu_tail_readline @@ -83,6 +83,6 @@ int cu_tail_readline(cu_tail_t *obj, char *buf, int buflen, _Bool force_rewind); * Returns 0 when successful and non-zero otherwise. */ int cu_tail_read(cu_tail_t *obj, char *buf, int buflen, tailfunc_t *callback, - void *data, _Bool force_rewind); + void *data, bool force_rewind); #endif /* UTILS_TAIL_H */ diff --git a/src/utils_message_parser.c b/src/utils_message_parser.c index dc051f36f..1f48e2af9 100644 --- a/src/utils_message_parser.c +++ b/src/utils_message_parser.c @@ -2,7 +2,7 @@ * collectd - src/utils_message_parser.c * MIT License * - * Copyright(c) 2017 Intel Corporation. All rights reserved. + * Copyright(c) 2017-2018 Intel Corporation. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -24,6 +24,7 @@ * * Authors: * Krzysztof Matczak + * Marcin Mozejko */ #include "collectd.h" @@ -38,76 +39,78 @@ #define MSG_STOR_INIT_LEN 64 #define MSG_STOR_INC_STEP 10 -typedef struct checked_match_t { - parser_job_data *parser_job; - message_pattern msg_pattern; +typedef struct checked_match_s { + parser_job_data_t *parser_job; + message_pattern_t msg_pattern; int msg_pattern_idx; -} checked_match; +} checked_match_t; -struct parser_job_data_t { +struct parser_job_data_s { const char *filename; unsigned int start_idx; unsigned int stop_idx; cu_tail_match_t *tm; - message *messages_storage; + message_t *messages_storage; size_t messages_max_len; int message_idx; unsigned int message_item_idx; unsigned int messages_completed; - message_pattern *message_patterns; + message_pattern_t *message_patterns; size_t message_patterns_len; - int (*resize_message_buffer)(parser_job_data *self, size_t); - int (*start_message_assembly)(parser_job_data *self); - void (*end_message_assembly)(parser_job_data *self); - void (*message_item_assembly)(parser_job_data *self, checked_match *cm, + int (*resize_message_buffer)(parser_job_data_t *self, size_t); + int (*start_message_assembly)(parser_job_data_t *self); + void (*end_message_assembly)(parser_job_data_t *self); + void (*message_item_assembly)(parser_job_data_t *self, checked_match_t *cm, char *const *matches); }; -static void message_item_assembly(parser_job_data *self, checked_match *cm, +static void message_item_assembly(parser_job_data_t *self, checked_match_t *cm, char *const *matches) { - message_item *msg_it = &(self->messages_storage[self->message_idx] - .message_items[self->message_item_idx]); - sstrncpy(msg_it->name, (cm->msg_pattern).name, sizeof(msg_it->name)); + message_item_t *msg_it = &self->messages_storage[self->message_idx] + .message_items[self->message_item_idx]; + sstrncpy(msg_it->name, cm->msg_pattern.name, sizeof(msg_it->name)); sstrncpy(msg_it->value, matches[cm->msg_pattern.submatch_idx], sizeof(msg_it->value)); + msg_it->user_data = cm->msg_pattern.user_data; + msg_it->free_user_data = cm->msg_pattern.free_user_data; ++(self->message_item_idx); } -static int start_message_assembly(parser_job_data *self) { +static int start_message_assembly(parser_job_data_t *self) { /* Remove previous message assembly if unfinished */ if (self->message_idx >= 0 && - self->messages_storage[self->message_idx].started == 1 && - self->messages_storage[self->message_idx].completed == 0) { + self->messages_storage[self->message_idx].started == true && + self->messages_storage[self->message_idx].completed == false) { DEBUG(UTIL_NAME ": Removing unfinished assembly of previous message"); - self->messages_storage[self->message_idx] = (message){0}; + self->messages_storage[self->message_idx] = (message_t){{{{0}}}}; self->message_item_idx = 0; } else ++(self->message_idx); /* Resize messages buffer if needed */ if (self->message_idx >= self->messages_max_len) { - INFO(UTIL_NAME ": Exceeded message buffer size: %lu", + INFO(UTIL_NAME ": Exceeded message buffer size: %zu", self->messages_max_len); - if (self->resize_message_buffer(self, self->messages_max_len + - MSG_STOR_INC_STEP) != 0) { - ERROR(UTIL_NAME ": Insufficient message buffer size: %lu. Remaining " + if (self->resize_message_buffer( + self, self->messages_max_len + MSG_STOR_INC_STEP) != 0) { + ERROR(UTIL_NAME ": Insufficient message buffer size: %zu. Remaining " "messages for this read will be skipped", self->messages_max_len); self->message_idx = self->messages_max_len; return -1; } } - self->messages_storage[self->message_idx] = (message){0}; + self->messages_storage[self->message_idx] = (message_t){{{{0}}}}; self->message_item_idx = 0; - self->messages_storage[self->message_idx].started = 1; - self->messages_storage[self->message_idx].completed = 0; + self->messages_storage[self->message_idx].started = true; + self->messages_storage[self->message_idx].completed = false; return 0; } -static int resize_message_buffer(parser_job_data *self, size_t new_size) { - INFO(UTIL_NAME ": Resizing message buffer size to %lu", new_size); +static int resize_message_buffer(parser_job_data_t *self, size_t new_size) { + INFO(UTIL_NAME ": Resizing message buffer size to %zu", new_size); void *new_storage = realloc(self->messages_storage, - new_size * sizeof(*(self->messages_storage))); + new_size * sizeof(*self->messages_storage)); if (new_storage == NULL) { ERROR(UTIL_NAME ": Error while reallocating message buffer"); return -1; @@ -118,11 +121,11 @@ static int resize_message_buffer(parser_job_data *self, size_t new_size) { unsigned int unused_idx = self->message_idx < 0 ? 0 : self->message_idx; memset(self->messages_storage + unused_idx, 0, (self->messages_max_len - unused_idx) * - sizeof(*(self->messages_storage))); + sizeof(*self->messages_storage)); return 0; } -static void end_message_assembly(parser_job_data *self) { +static void end_message_assembly(parser_job_data_t *self) { /* Checking mandatory items */ for (size_t i = 0; i < self->message_patterns_len; i++) { if (self->message_patterns[i].is_mandatory && @@ -132,14 +135,14 @@ static void end_message_assembly(parser_job_data *self) { UTIL_NAME ": Mandatory message item pattern %s not found. Message discarded", self->message_patterns[i].regex); - self->messages_storage[self->message_idx] = (message){0}; + self->messages_storage[self->message_idx] = (message_t){{{{0}}}}; self->message_item_idx = 0; if (self->message_idx > 0) --(self->message_idx); return; } } - self->messages_storage[self->message_idx].completed = 1; + self->messages_storage[self->message_idx].completed = true; ++self->messages_completed; self->message_item_idx = 0; } @@ -150,11 +153,11 @@ static int message_assembler(const char *row, char *const *matches, ERROR(UTIL_NAME ": Invalid user_data pointer"); return -1; } - checked_match *cm = (checked_match *)user_data; - parser_job_data *parser_job = cm->parser_job; + checked_match_t *cm = (checked_match_t *)user_data; + parser_job_data_t *parser_job = cm->parser_job; if (cm->msg_pattern.submatch_idx < -1 || - cm->msg_pattern.submatch_idx >= (int) matches_num) { + cm->msg_pattern.submatch_idx >= (int)matches_num) { ERROR(UTIL_NAME ": Invalid target submatch index: %d", cm->msg_pattern.submatch_idx); return -1; @@ -169,7 +172,7 @@ static int message_assembler(const char *row, char *const *matches, /* Every matched start pattern resets current message items and starts * assembling new messages */ - if (strcmp((cm->msg_pattern).regex, + if (strcmp(cm->msg_pattern.regex, parser_job->message_patterns[parser_job->start_idx].regex) == 0) { DEBUG(UTIL_NAME ": Found beginning pattern"); if (parser_job->start_message_assembly(parser_job) != 0) @@ -178,8 +181,8 @@ static int message_assembler(const char *row, char *const *matches, /* Ignoring message items without corresponding start item or * after completion */ if (parser_job->message_idx < 0 || - parser_job->messages_storage[parser_job->message_idx].started == 0 || - parser_job->messages_storage[parser_job->message_idx].completed == 1) { + parser_job->messages_storage[parser_job->message_idx].started == false || + parser_job->messages_storage[parser_job->message_idx].completed == true) { DEBUG(UTIL_NAME ": Dropping item with no corresponding start element"); return 0; } @@ -192,7 +195,7 @@ static int message_assembler(const char *row, char *const *matches, .matched_patterns_check[cm->msg_pattern_idx] = 1; /* Handle message ending */ - if (strcmp((cm->msg_pattern).regex, + if (strcmp(cm->msg_pattern.regex, parser_job->message_patterns[parser_job->stop_idx].regex) == 0) { DEBUG(UTIL_NAME ": Found ending pattern"); parser_job->end_message_assembly(parser_job); @@ -200,12 +203,12 @@ static int message_assembler(const char *row, char *const *matches, return 0; } -parser_job_data *message_parser_init(const char *filename, - unsigned int start_idx, - unsigned int stop_idx, - message_pattern message_patterns[], - size_t message_patterns_len) { - parser_job_data *parser_job = calloc(1, sizeof(*parser_job)); +parser_job_data_t *message_parser_init(const char *filename, + unsigned int start_idx, + unsigned int stop_idx, + message_pattern_t message_patterns[], + size_t message_patterns_len) { + parser_job_data_t *parser_job = calloc(1, sizeof(*parser_job)); if (parser_job == NULL) { ERROR(UTIL_NAME ": Error allocating parser_job"); return NULL; @@ -221,35 +224,35 @@ parser_job_data *message_parser_init(const char *filename, parser_job->message_idx = -1; parser_job->messages_completed = 0; parser_job->message_patterns = - calloc(message_patterns_len, sizeof(*(parser_job->message_patterns))); + calloc(message_patterns_len, sizeof(*parser_job->message_patterns)); if (parser_job->message_patterns == NULL) { ERROR(UTIL_NAME ": Error allocating message_patterns"); - return NULL; + goto free_parser_job; } - parser_job->messages_storage = calloc( - parser_job->messages_max_len, sizeof(*(parser_job->messages_storage))); + parser_job->messages_storage = calloc(parser_job->messages_max_len, + sizeof(*parser_job->messages_storage)); if (parser_job->messages_storage == NULL) { ERROR(UTIL_NAME ": Error allocating messages_storage"); - return NULL; + goto free_msg_patterns; } /* Crete own copy of regex patterns */ memcpy(parser_job->message_patterns, message_patterns, - sizeof(*(parser_job->message_patterns)) * message_patterns_len); + sizeof(*parser_job->message_patterns) * message_patterns_len); parser_job->message_patterns_len = message_patterns_len; /* Init tail match */ parser_job->tm = tail_match_create(parser_job->filename); if (parser_job->tm == NULL) { ERROR(UTIL_NAME ": Error creating tail match"); - return NULL; + goto free_msg_storage; } for (size_t i = 0; i < message_patterns_len; i++) { /* Create current_match container for passing regex info * to callback function */ - checked_match *current_match = calloc(1, sizeof(*current_match)); + checked_match_t *current_match = calloc(1, sizeof(*current_match)); if (current_match == NULL) { ERROR(UTIL_NAME ": Error allocating current_match"); - return NULL; + goto free_tail_match; } current_match->parser_job = parser_job; current_match->msg_pattern = message_patterns[i]; @@ -260,38 +263,50 @@ parser_job_data *message_parser_init(const char *filename, message_assembler, current_match, free); if (m == NULL) { ERROR(UTIL_NAME ": Error creating match callback"); - return NULL; + goto free_tail_match; } if (tail_match_add_match(parser_job->tm, m, 0, 0, 0) != 0) { ERROR(UTIL_NAME ": Error adding match callback"); - return NULL; + goto free_tail_match; } } return parser_job; + +free_tail_match: + tail_match_destroy(parser_job->tm); +free_msg_storage: + sfree(parser_job->messages_storage); +free_msg_patterns: + sfree(parser_job->message_patterns); +free_parser_job: + sfree(parser_job); + + return NULL; } -int message_parser_read(parser_job_data *parser_job, message **messages_storage, - _Bool force_rewind) { +int message_parser_read(parser_job_data_t *parser_job, + message_t **messages_storage, bool force_rewind) { if (parser_job == NULL) { ERROR(UTIL_NAME ": Invalid parser_job pointer"); return -1; } parser_job->messages_completed = 0; - _Bool incomplete_msg_found = 0; + bool incomplete_msg_found = false; /* Finish incomplete message assembly in this read */ if (parser_job->message_idx >= 0 && parser_job->messages_storage[parser_job->message_idx].started && !(parser_job->messages_storage[parser_job->message_idx].completed)) { INFO(UTIL_NAME ": Found incomplete message from previous read."); - incomplete_msg_found = 1; - message tmp_message = parser_job->messages_storage[parser_job->message_idx]; + incomplete_msg_found = true; + message_t tmp_message = + parser_job->messages_storage[parser_job->message_idx]; int tmp_message_item_idx = parser_job->message_item_idx; memset(parser_job->messages_storage, 0, parser_job->messages_max_len * - sizeof(*(parser_job->messages_storage))); + sizeof(*parser_job->messages_storage)); memcpy(parser_job->messages_storage, &tmp_message, - sizeof(*(parser_job->messages_storage))); + sizeof(*parser_job->messages_storage)); parser_job->message_item_idx = tmp_message_item_idx; parser_job->message_idx = 0; } @@ -299,7 +314,7 @@ int message_parser_read(parser_job_data *parser_job, message **messages_storage, else if (parser_job->message_idx >= 0) { memset(parser_job->messages_storage, 0, parser_job->messages_max_len * - sizeof(*(parser_job->messages_storage))); + sizeof(*parser_job->messages_storage)); parser_job->message_item_idx = 0; parser_job->message_idx = -1; } @@ -319,7 +334,7 @@ int message_parser_read(parser_job_data *parser_job, message **messages_storage, return parser_job->messages_completed; } -void message_parser_cleanup(parser_job_data *parser_job) { +void message_parser_cleanup(parser_job_data_t *parser_job) { if (parser_job == NULL) { ERROR(UTIL_NAME ": Invalid parser_job pointer"); return; diff --git a/src/utils_message_parser.h b/src/utils_message_parser.h index 2c85366d8..cd7e192cf 100644 --- a/src/utils_message_parser.h +++ b/src/utils_message_parser.h @@ -2,7 +2,7 @@ * collectd - src/utils_message_parser.h * MIT License * - * Copyright(c) 2017 Intel Corporation. All rights reserved. + * Copyright(c) 2017-2018 Intel Corporation. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -24,6 +24,7 @@ * * Authors: * Krzysztof Matczak + * Marcin Mozejko */ #ifndef UTILS_MESSAGE_PARSER_H @@ -31,7 +32,7 @@ #include "utils_tail_match.h" -typedef struct message_item_pattern_t { +typedef struct message_pattern_s { /* User defined name for message item */ char *name; /* Regular expression for finding out specific message item. The match result @@ -48,22 +49,28 @@ typedef struct message_item_pattern_t { /* Regular expression that excludes string from further processing */ char *excluderegex; /* Flag indicating if this item is mandatory for message validation */ - _Bool is_mandatory; -} message_pattern; + bool is_mandatory; + /* Pointer to optional user data */ + void *user_data; + /* Freeing function */ + void (*free_user_data)(void *data); +} message_pattern_t; -typedef struct message_item_t { +typedef struct message_item_s { char name[32]; char value[64]; -} message_item; + void *user_data; + void (*free_user_data)(void *data); +} message_item_t; -typedef struct message_t { - message_item message_items[32]; +typedef struct message_s { + message_item_t message_items[32]; int matched_patterns_check[32]; - _Bool started; - _Bool completed; -} message; + bool started; + bool completed; +} message_t; -typedef struct parser_job_data_t parser_job_data; +typedef struct parser_job_data_s parser_job_data_t; /* * NAME @@ -101,11 +108,11 @@ typedef struct parser_job_data_t parser_job_data; * RETURN VALUE * Returns NULL upon failure, pointer to new parser job otherwise. */ -parser_job_data *message_parser_init(const char *filename, - unsigned int start_idx, - unsigned int stop_idx, - message_pattern message_patterns[], - size_t message_patterns_len); +parser_job_data_t *message_parser_init(const char *filename, + unsigned int start_idx, + unsigned int stop_idx, + message_pattern_t message_patterns[], + size_t message_patterns_len); /* * NAME @@ -127,8 +134,8 @@ parser_job_data *message_parser_init(const char *filename, * Returns -1 upon failure, number of messages collected from last read * otherwise. */ -int message_parser_read(parser_job_data *parser_job, message **messages_storage, - _Bool force_rewind); +int message_parser_read(parser_job_data_t *parser_job, + message_t **messages_storage, bool force_rewind); /* * NAME @@ -141,6 +148,6 @@ int message_parser_read(parser_job_data *parser_job, message **messages_storage, * `parser_job' Pointer to parser job to be cleaned up. * */ -void message_parser_cleanup(parser_job_data *parser_job); +void message_parser_cleanup(parser_job_data_t *parser_job); #endif /* UTILS_MESSAGE_PARSER_H */ diff --git a/src/utils_message_parser_test.c b/src/utils_message_parser_test.c new file mode 100644 index 000000000..e63e69471 --- /dev/null +++ b/src/utils_message_parser_test.c @@ -0,0 +1,451 @@ +/** + * collectd - src/utils_message_parser_test.c + * + * Copyright(c) 2018 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * Authors: + * Marcin Mozejko + **/ + +#include "testing.h" +#include "utils_message_parser.c" + +#define TEST_PATTERN_NAME "test_pattern_name" +#define TEST_REGEX "test_regex" +#define TEST_EX_REGEX "test_ex_regex" +#define TEST_MSG_ITEM_NAME "test_msg_item_name" +#define TEST_MSG_ITEM_VAL "test_msg_item_value" +#define TEST_FILENAME "test_filename" +#define TEST_PATTERNS_LEN 4 + +static int start_message_assembly_mock_error(parser_job_data_t *self) { + return -1; +} + +static int start_message_assembly_mock_success(parser_job_data_t *self) { + return 0; +} + +static void message_item_assembly_mock(parser_job_data_t *self, + checked_match_t *cm, + char *const *matches) { + return; +} + +static void end_message_assembly_mock(parser_job_data_t *self) { return; } + +DEF_TEST(msg_item_assembly) { + parser_job_data_t *job = calloc(1, sizeof(*job)); + checked_match_t *cm = calloc(1, sizeof(*cm)); + char *const matches[] = {"test_match0", "test_match1"}; + + job->messages_storage = calloc(1, sizeof(*job->messages_storage)); + job->message_idx = 0; + + cm->parser_job = job; + cm->msg_pattern.name = TEST_PATTERN_NAME; + cm->msg_pattern.regex = TEST_REGEX; + cm->msg_pattern.submatch_idx = 1; + cm->msg_pattern.excluderegex = TEST_EX_REGEX; + cm->msg_pattern.is_mandatory = false; + cm->msg_pattern_idx = 0; + + message_item_t *msg_it = &(job->messages_storage[job->message_idx] + .message_items[job->message_item_idx]); + + message_item_assembly(job, cm, matches); + + EXPECT_EQ_STR(msg_it->name, TEST_PATTERN_NAME); + EXPECT_EQ_STR(msg_it->value, "test_match1"); + + sfree(cm); + sfree(job->messages_storage); + sfree(job); + + return 0; +} + +DEF_TEST(start_msg_item_assembly_1) { + parser_job_data_t *job = calloc(1, sizeof(*job)); + + job->messages_max_len = MSG_STOR_INIT_LEN; + job->message_idx = 0; + job->message_item_idx = 1; + job->messages_storage = + calloc(job->messages_max_len, sizeof(*job->messages_storage)); + job->messages_storage[0].started = true; + job->messages_storage[0].completed = false; + + sstrncpy(job->messages_storage[0].message_items[0].name, TEST_MSG_ITEM_NAME, + sizeof(TEST_MSG_ITEM_NAME)); + sstrncpy(job->messages_storage[0].message_items[0].value, TEST_MSG_ITEM_VAL, + sizeof(TEST_MSG_ITEM_VAL)); + + int ret = start_message_assembly(job); + + EXPECT_EQ_STR("", job->messages_storage[0].message_items[0].name); + EXPECT_EQ_STR("", job->messages_storage[0].message_items[0].value); + EXPECT_EQ_INT(0, job->message_item_idx); + EXPECT_EQ_INT(0, ret); + sfree(job->messages_storage); + sfree(job); + + return 0; +} + +DEF_TEST(start_msg_item_assembly_2) { + parser_job_data_t *job = calloc(1, sizeof(*job)); + + job->messages_max_len = MSG_STOR_INIT_LEN; + job->message_idx = 0; + job->message_item_idx = 1; + job->messages_storage = + calloc(job->messages_max_len, sizeof(*job->messages_storage)); + job->messages_storage[0].started = true; + job->messages_storage[0].completed = true; + + sstrncpy(job->messages_storage[0].message_items[0].name, TEST_MSG_ITEM_NAME, + sizeof(TEST_MSG_ITEM_NAME)); + sstrncpy(job->messages_storage[0].message_items[0].value, TEST_MSG_ITEM_VAL, + sizeof(TEST_MSG_ITEM_VAL)); + + int ret = start_message_assembly(job); + + EXPECT_EQ_STR("", job->messages_storage[1].message_items[0].name); + EXPECT_EQ_STR("", job->messages_storage[1].message_items[0].value); + EXPECT_EQ_STR(TEST_MSG_ITEM_NAME, + job->messages_storage[0].message_items[0].name); + EXPECT_EQ_STR(TEST_MSG_ITEM_VAL, + job->messages_storage[0].message_items[0].value); + + EXPECT_EQ_INT(0, job->message_item_idx); + EXPECT_EQ_INT(1, job->message_idx); + EXPECT_EQ_INT(0, ret); + sfree(job->messages_storage); + sfree(job); + + return 0; +} + +DEF_TEST(start_msg_item_assembly_3) { + parser_job_data_t *job = calloc(1, sizeof(*job)); + + job->messages_max_len = 1; + job->message_idx = 0; + job->message_item_idx = 1; + job->messages_storage = + calloc(job->messages_max_len, sizeof(*job->messages_storage)); + job->messages_storage[0].started = true; + job->messages_storage[0].completed = true; + job->resize_message_buffer = resize_message_buffer; + + sstrncpy(job->messages_storage[0].message_items[0].name, TEST_MSG_ITEM_NAME, + sizeof(TEST_MSG_ITEM_NAME)); + sstrncpy(job->messages_storage[0].message_items[0].value, TEST_MSG_ITEM_VAL, + sizeof(TEST_MSG_ITEM_VAL)); + + int ret = start_message_assembly(job); + + EXPECT_EQ_STR(TEST_MSG_ITEM_NAME, + job->messages_storage[0].message_items[0].name); + EXPECT_EQ_STR(TEST_MSG_ITEM_VAL, + job->messages_storage[0].message_items[0].value); + EXPECT_EQ_INT(0, job->message_item_idx); + EXPECT_EQ_INT(1 + MSG_STOR_INC_STEP, job->messages_max_len); + EXPECT_EQ_INT(0, ret); + sfree(job->messages_storage); + sfree(job); + + return 0; +} + +DEF_TEST(resize_msg_buffer) { + const u_int new_size = 5; + parser_job_data_t *job = calloc(1, sizeof(*job)); + job->messages_storage = calloc(1, sizeof(*job->messages_storage)); + job->messages_max_len = 1; + int ret = resize_message_buffer(job, new_size); + EXPECT_EQ_INT(0, ret); + EXPECT_EQ_INT(new_size, job->messages_max_len); + sfree(job->messages_storage); + sfree(job); + + return 0; +} + +DEF_TEST(end_msg_assembly_1) { + parser_job_data_t *job = calloc(1, sizeof(*job)); + + job->message_idx = 0; + job->messages_storage = calloc(1, sizeof(*job->messages_storage)); + job->messages_max_len = 1; + + end_message_assembly(job); + + EXPECT_EQ_INT(1, job->messages_storage[job->message_idx].completed); + EXPECT_EQ_INT(1, job->messages_completed); + EXPECT_EQ_INT(0, job->message_item_idx); + + sfree(job->messages_storage); + sfree(job); + + return 0; +} + +DEF_TEST(end_msg_assembly_2) { + parser_job_data_t *job = calloc(1, sizeof(*job)); + + job->message_idx = 1; + job->message_item_idx = 1; + job->messages_storage = calloc(2, sizeof(*job->messages_storage)); + job->messages_max_len = 1; + job->message_patterns_len = 1; + job->message_patterns = calloc(1, sizeof(*job->message_patterns)); + job->message_patterns[0].is_mandatory = true; + job->message_patterns[0].regex = TEST_REGEX; + + end_message_assembly(job); + + EXPECT_EQ_INT(0, job->messages_storage[job->message_idx].completed); + EXPECT_EQ_INT(0, job->messages_completed); + EXPECT_EQ_INT(0, job->message_item_idx); + EXPECT_EQ_INT(0, job->message_idx); + + sfree(job->messages_storage); + sfree(job->message_patterns); + sfree(job); + + return 0; +} + +DEF_TEST(msg_assembler_1) { + int ret = message_assembler(NULL, NULL, 0, NULL); + + EXPECT_EQ_INT(-1, ret); + + return 0; +} + +DEF_TEST(msg_assembler_2) { + + checked_match_t *cm = calloc(1, sizeof(*cm)); + cm->msg_pattern.submatch_idx = 1; + + int ret = message_assembler(NULL, NULL, 1, cm); + + EXPECT_EQ_INT(-1, ret); + + sfree(cm); + + return 0; +} + +DEF_TEST(msg_assembler_3) { + parser_job_data_t *job = calloc(1, sizeof(*job)); + checked_match_t *cm = calloc(1, sizeof(*cm)); + + job->messages_storage = calloc(1, sizeof(*job->messages_storage)); + job->message_patterns = calloc(1, sizeof(*job->message_patterns)); + job->message_patterns_len = 1; + job->message_item_idx = 32; + job->message_idx = 0; + job->end_message_assembly = end_message_assembly; + cm->msg_pattern.submatch_idx = 1; + cm->parser_job = job; + + int ret = message_assembler(NULL, NULL, 3, cm); + + EXPECT_EQ_INT(-1, ret); + + sfree(cm); + sfree(job->messages_storage); + sfree(job->message_patterns); + sfree(job); + + return 0; +} + +DEF_TEST(msg_assembler_4) { + parser_job_data_t *job = calloc(1, sizeof(*job)); + checked_match_t *cm = calloc(1, sizeof(*cm)); + + cm->parser_job = job; + cm->msg_pattern_idx = 0; + cm->msg_pattern.submatch_idx = 0; + cm->msg_pattern.regex = TEST_REGEX; + job->messages_storage = calloc(1, sizeof(*job->messages_storage)); + job->message_patterns = calloc(1, sizeof(*job->message_patterns)); + job->message_patterns_len = 1; + job->message_patterns[0].regex = cm->msg_pattern.regex; + job->message_item_idx = 0; + job->message_idx = 0; + job->start_message_assembly = start_message_assembly_mock_error; + + int ret = message_assembler(NULL, NULL, 3, cm); + EXPECT_EQ_INT(-1, ret); + + job->start_message_assembly = start_message_assembly_mock_success; + job->message_idx = -1; + job->messages_storage[0].started = true; + job->messages_storage[0].completed = false; + ret = message_assembler(NULL, NULL, 3, cm); + EXPECT_EQ_INT(0, ret); + + job->message_idx = 0; + job->messages_storage[0].started = false; + job->messages_storage[0].completed = false; + ret = message_assembler(NULL, NULL, 3, cm); + EXPECT_EQ_INT(0, ret); + + job->message_idx = 0; + job->messages_storage[0].started = true; + job->messages_storage[0].completed = true; + ret = message_assembler(NULL, NULL, 3, cm); + EXPECT_EQ_INT(0, ret); + + job->messages_storage[0].completed = false; + job->message_item_assembly = message_item_assembly_mock; + job->end_message_assembly = end_message_assembly_mock; + ret = message_assembler(NULL, NULL, 3, cm); + + EXPECT_EQ_INT(1, job->messages_storage[0].matched_patterns_check[0]); + EXPECT_EQ_INT(0, ret); + + sfree(cm); + sfree(job->messages_storage); + sfree(job->message_patterns); + sfree(job); + + return 0; +} + +DEF_TEST(msg_parser_init) { + message_pattern_t patterns[TEST_PATTERNS_LEN] = { + {"pattern_1", "test_regex_1", 0, "", 1}, + {"pattern_2", "test_regex_2", 0, "", 0}, + {"pattern_3", "test_regex_3", 0, "", 0}, + {"pattern_4", "test_regex_4", 0, "", 1} + + }; + unsigned int start_idx = 0; + unsigned int stop_idx = 1; + const char *filename = TEST_FILENAME; + parser_job_data_t *job = NULL; + + job = message_parser_init(filename, start_idx, stop_idx, patterns, + TEST_PATTERNS_LEN); + + OK(job != NULL); + OK(job->resize_message_buffer == resize_message_buffer); + OK(job->start_message_assembly == start_message_assembly); + OK(job->end_message_assembly == end_message_assembly); + OK(job->message_item_assembly == message_item_assembly); + OK(job->messages_max_len == MSG_STOR_INIT_LEN); + OK(job->filename == filename); + OK(job->start_idx == start_idx); + OK(job->stop_idx == stop_idx); + OK(job->message_idx == -1); + OK(job->messages_completed == 0); + OK(memcmp(patterns, job->message_patterns, + sizeof(message_pattern_t) * TEST_PATTERNS_LEN) == 0); + OK(job->message_patterns_len == TEST_PATTERNS_LEN); + + sfree(job->messages_storage); + sfree(job->message_patterns); + tail_match_destroy(job->tm); + sfree(job); + + return 0; +} + +DEF_TEST(msg_parser_read_1) { + parser_job_data_t *job = NULL; + int ret = message_parser_read(job, NULL, 0); + + EXPECT_EQ_INT(-1, ret); + + return 0; +} + +DEF_TEST(msg_parser_read_2) { + + message_pattern_t patterns[TEST_PATTERNS_LEN] = { + {"pattern_1", "test_regex_1", 0, "", 1}, + {"pattern_2", "test_regex_2", 0, "", 0}, + {"pattern_3", "test_regex_3", 0, "", 0}, + {"pattern_4", "test_regex_4", 0, "", 1} + + }; + + unsigned int start_idx = 0; + unsigned int stop_idx = 1; + const char *filename = TEST_FILENAME; + parser_job_data_t *job = calloc(1, sizeof(*job)); + + job->resize_message_buffer = resize_message_buffer; + job->start_message_assembly = start_message_assembly; + job->end_message_assembly = end_message_assembly; + job->message_item_assembly = message_item_assembly; + job->messages_max_len = MSG_STOR_INIT_LEN; + job->filename = filename; + job->start_idx = start_idx; + job->stop_idx = stop_idx; + job->message_idx = -1; + job->messages_completed = 0; + job->message_patterns = patterns; + job->message_patterns_len = TEST_PATTERNS_LEN; + job->tm = tail_match_create(job->filename); + + int ret = message_parser_read(job, NULL, 0); + + EXPECT_EQ_INT(-1, ret); + + tail_match_destroy(job->tm); + sfree(job); + + return 0; +} + +int main(void) { + /* message_item_assembly */ + RUN_TEST(msg_item_assembly); + /* start_message_item_assembly */ + RUN_TEST(start_msg_item_assembly_1); + RUN_TEST(start_msg_item_assembly_2); + RUN_TEST(start_msg_item_assembly_3); + /* resize_message_buffer */ + RUN_TEST(resize_msg_buffer); + /* end_message assembly */ + RUN_TEST(end_msg_assembly_1); + RUN_TEST(end_msg_assembly_2); + /* message_assembler */ + RUN_TEST(msg_assembler_1); + RUN_TEST(msg_assembler_2); + RUN_TEST(msg_assembler_3); + RUN_TEST(msg_assembler_4); + /* message_parser_init */ + RUN_TEST(msg_parser_init); + /* message_parser_read */ + RUN_TEST(msg_parser_read_1); + RUN_TEST(msg_parser_read_2); + + END_TEST; +} diff --git a/src/utils_tail_match.c b/src/utils_tail_match.c index a0220abdd..25714c168 100644 --- a/src/utils_tail_match.c +++ b/src/utils_tail_match.c @@ -306,7 +306,7 @@ out: return status; } /* int tail_match_add_match_simple */ -int tail_match_read(cu_tail_match_t *obj, _Bool force_rewind) { +int tail_match_read(cu_tail_match_t *obj, bool force_rewind) { char buffer[4096]; int status; diff --git a/src/utils_tail_match.h b/src/utils_tail_match.h index 0f510294d..9770fd7f6 100644 --- a/src/utils_tail_match.h +++ b/src/utils_tail_match.h @@ -135,6 +135,6 @@ int tail_match_add_match_simple(cu_tail_match_t *obj, const char *regex, * RETURN VALUE * Zero on success, nonzero on failure. */ -int tail_match_read(cu_tail_match_t *obj, _Bool force_rewind); +int tail_match_read(cu_tail_match_t *obj, bool force_rewind); #endif /* UTILS_TAIL_MATCH_H */ -- 2.47.2