From: Eric Leblond Date: Sat, 7 Feb 2015 12:10:23 +0000 (+0100) Subject: output-json: add redis support X-Git-Tag: suricata-3.0RC1~56 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=eef5678e5e4ae210e3a0137cad7650ccd798369b;p=thirdparty%2Fsuricata.git output-json: add redis support This patch adds redis support to JSON output. --- diff --git a/configure.ac b/configure.ac index 97ce3ab174..9e13938304 100644 --- a/configure.ac +++ b/configure.ac @@ -1703,6 +1703,46 @@ LDFLAGS="${LDFLAGS} -pie" fi +# libhiredis + AC_ARG_ENABLE(hiredis, + AS_HELP_STRING([--enable-hiredis],[Enable Redis support]), + [ enable_hiredis="yes"], + [ enable_hiredis="no"]) + AC_ARG_WITH(libhiredis_includes, + [ --with-libhiredis-includes=DIR libhiredis include directory], + [with_libhiredis_includes="$withval"],[with_libhiredis_includes="no"]) + AC_ARG_WITH(libhiredis_libraries, + [ --with-libhiredis-libraries=DIR libhiredis library directory], + [with_libhiredis_libraries="$withval"],[with_libhiredis_libraries="no"]) + + if test "$enable_hiredis" = "yes"; then + if test "$with_libhiredis_includes" != "no"; then + CPPFLAGS="${CPPFLAGS} -I${with_libhiredis_includes}" + fi + + AC_CHECK_HEADER("hiredis/hiredis.h",HIREDIS="yes",HIREDIS="no") + if test "$HIREDIS" = "yes"; then + if test "$with_libhiredis_libraries" != "no"; then + LDFLAGS="${LDFLAGS} -L${with_libhiredis_libraries}" + fi + AC_CHECK_LIB(hiredis, redisConnect,, HIREDIS="no") + fi + if test "$HIREDIS" = "no"; then + echo + echo " ERROR! libhiredis library not found, go get it" + echo " from https://github.com/redis/hiredis or your distribution:" + echo + echo " Ubuntu: apt-get install libhiredis-dev" + echo " Fedora: yum install libhiredis-devel" + echo + exit 1 + fi + if test "$HIREDIS" = "yes"; then + AC_DEFINE([HAVE_LIBHIREDIS],[1],[libhiredis available]) + enable_hiredis="yes" + fi + fi + # get cache line size AC_PATH_PROG(HAVE_GETCONF_CMD, getconf, "no") if test "$HAVE_GETCONF_CMD" != "no"; then @@ -1810,6 +1850,7 @@ SURICATA_BUILD_CONF="Suricata Configuration: libnss support: ${enable_nss} libnspr support: ${enable_nspr} libjansson support: ${enable_jansson} + hiredis support: ${enable_hiredis} Prelude support: ${enable_prelude} PCRE jit: ${pcre_jit_available} LUA support: ${enable_lua} diff --git a/src/output-json.c b/src/output-json.c index a8e5456351..eb937c4ef5 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -110,6 +110,9 @@ void OutputJsonRegisterTests (void) #else /* implied we do have JSON support */ #include +#if HAVE_LIBHIREDIS +#include +#endif #define DEFAULT_LOG_FILENAME "eve.json" #define DEFAULT_ALERT_SYSLOG_FACILITY_STR "local0" @@ -379,6 +382,28 @@ int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer) file_ctx->Write((const char *)MEMBUFFER_BUFFER(buffer), MEMBUFFER_OFFSET(buffer), file_ctx); } +#if HAVE_LIBHIREDIS + else if (file_ctx->type == LOGFILE_TYPE_REDIS) { + /* FIXME go async here */ + redisReply *reply = redisCommand(file_ctx->redis, "%s %s %s", + file_ctx->redis_setup.command, + file_ctx->redis_setup.key, + js_s); + switch (reply->type) { + case REDIS_REPLY_ERROR: + SCLogWarning(SC_WARN_NO_UNITTESTS, "Redis error: %s", reply->str); + break; + case REDIS_REPLY_INTEGER: + SCLogDebug("Redis integer %lld", reply->integer); + break; + default: + SCLogError(SC_ERR_INVALID_VALUE, + "Redis default triggered with %d", reply->type); + break; + } + freeReplyObject(reply); + } +#endif SCMutexUnlock(&file_ctx->fp_mutex); free(js_s); return 0; @@ -477,6 +502,14 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf) json_ctx->json_out = LOGFILE_TYPE_UNIX_DGRAM; } else if (strcmp(output_s, "unix_stream") == 0) { json_ctx->json_out = LOGFILE_TYPE_UNIX_STREAM; + } else if (strcmp(output_s, "redis") == 0) { +#if HAVE_LIBHIREDIS + json_ctx->json_out = LOGFILE_TYPE_REDIS; +#else + SCLogError(SC_ERR_INVALID_ARGUMENT, + "redis JSON output option is not compiled"); + exit(EXIT_FAILURE); +#endif } else { SCLogError(SC_ERR_INVALID_ARGUMENT, "Invalid JSON output option: %s", output_s); @@ -549,6 +582,58 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf) openlog(ident, LOG_PID|LOG_NDELAY, facility); } +#if HAVE_LIBHIREDIS + else if (json_ctx->json_out == LOGFILE_TYPE_REDIS) { + ConfNode *redis_node = ConfNodeLookupChild(conf, "redis"); + const char *redis_server = NULL; + const char *redis_port = NULL; + const char *redis_mode = NULL; + const char *redis_key = NULL; + + if (redis_node) { + redis_server = ConfNodeLookupChildValue(redis_node, "server"); + redis_port = ConfNodeLookupChildValue(redis_node, "port"); + redis_mode = ConfNodeLookupChildValue(redis_node, "mode"); + redis_key = ConfNodeLookupChildValue(redis_node, "key"); + } + if (!redis_server) { + redis_server = "127.0.0.1"; + SCLogInfo("Using default redis server (127.0.0.1)"); + } + if (!redis_port) + redis_port = "6379"; + if (!redis_mode) + redis_mode = "list"; + if (!redis_key) + redis_key = "suricata"; + json_ctx->file_ctx->redis_setup.key = SCStrdup(redis_key); + + if (!json_ctx->file_ctx->redis_setup.key) { + SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate redis key name"); + exit(EXIT_FAILURE); + } + + if (!strcmp(redis_mode, "list")) { + json_ctx->file_ctx->redis_setup.command = SCStrdup("LPUSH"); + if (!json_ctx->file_ctx->redis_setup.command) { + SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate redis key command"); + exit(EXIT_FAILURE); + } + } else { + json_ctx->file_ctx->redis_setup.command = SCStrdup("PUBLISH"); + if (!json_ctx->file_ctx->redis_setup.command) { + SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate redis key command"); + exit(EXIT_FAILURE); + } + } + redisContext *c = redisConnect(redis_server, atoi(redis_port)); + if (c != NULL && c->err) { + SCLogError(SC_ERR_SOCKET, "Error connecting to redis server: %s\n", c->errstr); + exit(EXIT_FAILURE); + } + json_ctx->file_ctx->redis = c; + } +#endif const char *sensor_id_s = ConfNodeLookupChildValue(conf, "sensor-id"); if (sensor_id_s != NULL) { diff --git a/src/output-json.h b/src/output-json.h index 15c2829555..89e11d86ce 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -40,7 +40,6 @@ TmEcode OutputJSON(json_t *js, void *data, uint64_t *count); int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer); OutputCtx *OutputJsonInitCtx(ConfNode *); - enum JsonFormat { COMPACT, INDENT }; /* diff --git a/src/util-logopenfile.c b/src/util-logopenfile.c index b25c4a8270..b22988ae84 100644 --- a/src/util-logopenfile.c +++ b/src/util-logopenfile.c @@ -367,6 +367,12 @@ int LogFileFreeCtx(LogFileCtx *lf_ctx) SCMutexUnlock(&lf_ctx->fp_mutex); } +#ifdef HAVE_LIBHIREDIS + if (lf_ctx->type == LOGFILE_TYPE_REDIS && lf_ctx->redis) { + redisFree(lf_ctx->redis); + } +#endif + SCMutexDestroy(&lf_ctx->fp_mutex); if (lf_ctx->prefix != NULL) diff --git a/src/util-logopenfile.h b/src/util-logopenfile.h index d345475d22..b5ac9823de 100644 --- a/src/util-logopenfile.h +++ b/src/util-logopenfile.h @@ -27,6 +27,8 @@ #include "conf.h" /* ConfNode */ #include "tm-modules.h" /* LogFileCtx */ +#include "hiredis/hiredis.h" + typedef struct { uint16_t fileno; } PcieFile; @@ -34,15 +36,31 @@ typedef struct { enum LogFileType { LOGFILE_TYPE_FILE, LOGFILE_TYPE_SYSLOG, LOGFILE_TYPE_UNIX_DGRAM, - LOGFILE_TYPE_UNIX_STREAM }; + LOGFILE_TYPE_UNIX_STREAM, + LOGFILE_TYPE_REDIS }; + +enum RedisMode { REDIS_LIST, REDIS_CHANNEL }; + +typedef struct RedisSetup_ { + enum RedisMode mode; + char *command; + char *key; +} RedisSetup; /** Global structure for Output Context */ typedef struct LogFileCtx_ { union { FILE *fp; PcieFile *pcie_fp; +#ifdef HAVE_LIBHIREDIS + redisContext *redis; +#endif }; +#ifdef HAVE_LIBHIREDIS + RedisSetup redis_setup; +#endif + int (*Write)(const char *buffer, int buffer_len, struct LogFileCtx_ *fp); void (*Close)(struct LogFileCtx_ *fp); diff --git a/suricata.yaml.in b/suricata.yaml.in index 9ab40098fd..30921751a9 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -92,7 +92,7 @@ outputs: # Extensible Event Format (nicknamed EVE) event log in JSON format - eve-log: enabled: yes - filetype: regular #regular|syslog|unix_dgram|unix_stream + filetype: regular #regular|syslog|unix_dgram|unix_stream|redis filename: eve.json #prefix: "@cee: " # prefix to prepend to each log entry # the following are valid when type: syslog above @@ -100,6 +100,11 @@ outputs: #facility: local5 #level: Info ## possible levels: Emergency, Alert, Critical, ## Error, Warning, Notice, Info, Debug + #redis: + # server: 127.0.0.1 + # port: 6379 + # mode: list ## possible values: list (default), channel + # key: suricata ## key or channel to use (default to suricata) types: - alert: # payload: yes # enable dumping payload in Base64