From: Victor Julien Date: Tue, 16 Feb 2016 15:50:48 +0000 (+0100) Subject: eve: fix mishandling of big messages X-Git-Tag: suricata-3.0.1RC1~140 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8bb1cf08efd8ae54bc7d363f350a523709b71c32;p=thirdparty%2Fsuricata.git eve: fix mishandling of big messages When the string representation of a JSON message grew bigger than 64k, the JSON record would just be truncated. This lead to errors in the parser(s) of the JSON stream. This patch changes the buffer logic to grow the buffer on demand. --- diff --git a/src/log-stats.c b/src/log-stats.c index 8507deb301..ad6ee54720 100644 --- a/src/log-stats.c +++ b/src/log-stats.c @@ -118,7 +118,7 @@ int LogStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *st) /* since we can have many threads, the buffer might not be big enough. * Expand if necessary. */ - if (MEMBUFFER_OFFSET(aft->buffer) + len > MEMBUFFER_SIZE(aft->buffer)) { + if (MEMBUFFER_OFFSET(aft->buffer) + len >= MEMBUFFER_SIZE(aft->buffer)) { MemBufferExpand(&aft->buffer, OUTPUT_BUFFER_SIZE); } @@ -144,7 +144,7 @@ int LogStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *st) /* since we can have many threads, the buffer might not be big enough. * Expand if necessary. */ - if (MEMBUFFER_OFFSET(aft->buffer) + len > MEMBUFFER_SIZE(aft->buffer)) { + if (MEMBUFFER_OFFSET(aft->buffer) + len >= MEMBUFFER_SIZE(aft->buffer)) { MemBufferExpand(&aft->buffer, OUTPUT_BUFFER_SIZE); } diff --git a/src/output-json-alert.c b/src/output-json-alert.c index 32ec6c8498..f7281f49a4 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -362,7 +362,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) } } - OutputJSONBuffer(js, aft->file_ctx, aft->json_buffer); + OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer); json_object_del(js, "alert"); } json_object_clear(js); @@ -373,7 +373,6 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) static int AlertJsonDecoderEvent(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) { - MemBuffer *buffer = (MemBuffer *)aft->json_buffer; int i; char timebuf[64]; json_t *js; @@ -384,7 +383,7 @@ static int AlertJsonDecoderEvent(ThreadVars *tv, JsonAlertLogThread *aft, const CreateIsoTimeString(&p->ts, timebuf, sizeof(timebuf)); for (i = 0; i < p->alerts.cnt; i++) { - MemBufferReset(buffer); + MemBufferReset(aft->json_buffer); const PacketAlert *pa = &p->alerts.alerts[i]; if (unlikely(pa->s == NULL)) { @@ -436,7 +435,7 @@ static int AlertJsonDecoderEvent(ThreadVars *tv, JsonAlertLogThread *aft, const /* alert */ json_object_set_new(js, "alert", ajs); - OutputJSONBuffer(js, aft->file_ctx, buffer); + OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer); json_object_clear(js); json_decref(js); } diff --git a/src/output-json-dns.c b/src/output-json-dns.c index 89c11b3eef..c44be89a43 100644 --- a/src/output-json-dns.c +++ b/src/output-json-dns.c @@ -73,8 +73,6 @@ typedef struct LogDnsLogThread_ { static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx, uint64_t tx_id, DNSQueryEntry *entry) { - MemBuffer *buffer = (MemBuffer *)aft->buffer; - SCLogDebug("got a DNS request and now logging !!"); json_t *djs = json_object(); @@ -83,7 +81,7 @@ static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx, } /* reset */ - MemBufferReset(buffer); + MemBufferReset(aft->buffer); /* type */ json_object_set_new(djs, "type", json_string("query")); @@ -109,13 +107,12 @@ static void LogQuery(LogDnsLogThread *aft, json_t *js, DNSTransaction *tx, /* dns */ json_object_set_new(js, "dns", djs); - OutputJSONBuffer(js, aft->dnslog_ctx->file_ctx, buffer); + OutputJSONBuffer(js, aft->dnslog_ctx->file_ctx, &aft->buffer); json_object_del(js, "dns"); } static void OutputAnswer(LogDnsLogThread *aft, json_t *djs, DNSTransaction *tx, DNSAnswerEntry *entry) { - MemBuffer *buffer = (MemBuffer *)aft->buffer; json_t *js = json_object(); if (js == NULL) return; @@ -179,9 +176,9 @@ static void OutputAnswer(LogDnsLogThread *aft, json_t *djs, DNSTransaction *tx, } /* reset */ - MemBufferReset(buffer); + MemBufferReset(aft->buffer); json_object_set_new(djs, "dns", js); - OutputJSONBuffer(djs, aft->dnslog_ctx->file_ctx, buffer); + OutputJSONBuffer(djs, aft->dnslog_ctx->file_ctx, &aft->buffer); json_object_del(djs, "dns"); return; @@ -189,7 +186,6 @@ static void OutputAnswer(LogDnsLogThread *aft, json_t *djs, DNSTransaction *tx, static void OutputFailure(LogDnsLogThread *aft, json_t *djs, DNSTransaction *tx, DNSQueryEntry *entry) { - MemBuffer *buffer = (MemBuffer *)aft->buffer; json_t *js = json_object(); if (js == NULL) return; @@ -214,9 +210,9 @@ static void OutputFailure(LogDnsLogThread *aft, json_t *djs, DNSTransaction *tx, } /* reset */ - MemBufferReset(buffer); + MemBufferReset(aft->buffer); json_object_set_new(djs, "dns", js); - OutputJSONBuffer(djs, aft->dnslog_ctx->file_ctx, buffer); + OutputJSONBuffer(djs, aft->dnslog_ctx->file_ctx, &aft->buffer); json_object_del(djs, "dns"); return; diff --git a/src/output-json-drop.c b/src/output-json-drop.c index 68c14d9b5d..234332c898 100644 --- a/src/output-json-drop.c +++ b/src/output-json-drop.c @@ -85,7 +85,6 @@ typedef struct JsonDropLogThread_ { static int DropLogJSON (JsonDropLogThread *aft, const Packet *p) { uint16_t proto = 0; - MemBuffer *buffer = (MemBuffer *)aft->buffer; json_t *js = CreateJSONHeader((Packet *)p, 0, "drop");//TODO const if (unlikely(js == NULL)) return TM_ECODE_OK; @@ -97,7 +96,7 @@ static int DropLogJSON (JsonDropLogThread *aft, const Packet *p) } /* reset */ - MemBufferReset(buffer); + MemBufferReset(aft->buffer); if (PKT_IS_IPV4(p)) { json_object_set_new(djs, "len", json_integer(IPV4_GET_IPLEN(p))); @@ -168,7 +167,7 @@ static int DropLogJSON (JsonDropLogThread *aft, const Packet *p) } } - OutputJSONBuffer(js, aft->drop_ctx->file_ctx, buffer); + OutputJSONBuffer(js, aft->drop_ctx->file_ctx, &aft->buffer); json_object_del(js, "drop"); json_object_clear(js); json_decref(js); diff --git a/src/output-json-file.c b/src/output-json-file.c index c3932cac3c..12e416ac73 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -83,14 +83,13 @@ typedef struct JsonFileLogThread_ { */ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff) { - MemBuffer *buffer = (MemBuffer *)aft->buffer; json_t *js = CreateJSONHeader((Packet *)p, 0, "fileinfo"); //TODO const json_t *hjs = NULL; if (unlikely(js == NULL)) return; /* reset */ - MemBufferReset(buffer); + MemBufferReset(aft->buffer); switch (p->flow->alproto) { case ALPROTO_HTTP: @@ -158,7 +157,7 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F /* originally just 'file', but due to bug 1127 naming it fileinfo */ json_object_set_new(js, "fileinfo", fjs); - OutputJSONBuffer(js, aft->filelog_ctx->file_ctx, buffer); + OutputJSONBuffer(js, aft->filelog_ctx->file_ctx, &aft->buffer); json_object_del(js, "fileinfo"); switch (p->flow->alproto) { diff --git a/src/output-json-flow.c b/src/output-json-flow.c index f89c32df70..372bafa44c 100644 --- a/src/output-json-flow.c +++ b/src/output-json-flow.c @@ -313,10 +313,9 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) { SCEnter(); JsonFlowLogThread *jhl = (JsonFlowLogThread *)thread_data; - MemBuffer *buffer = (MemBuffer *)jhl->buffer; /* reset */ - MemBufferReset(buffer); + MemBufferReset(jhl->buffer); json_t *js = CreateJSONHeaderFromFlow(f, "flow"); //TODO const if (unlikely(js == NULL)) @@ -324,7 +323,7 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) JsonFlowLogJSON(jhl, js, f); - OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, buffer); + OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer); json_object_del(js, "http"); json_object_clear(js); diff --git a/src/output-json-http.c b/src/output-json-http.c index 6873987305..ddd5dfd681 100644 --- a/src/output-json-http.c +++ b/src/output-json-http.c @@ -371,7 +371,6 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl htp_tx_t *tx = txptr; JsonHttpLogThread *jhl = (JsonHttpLogThread *)thread_data; - MemBuffer *buffer = (MemBuffer *)jhl->buffer; json_t *js = CreateJSONHeaderWithTxId((Packet *)p, 1, "http", tx_id); //TODO const if (unlikely(js == NULL)) @@ -380,11 +379,11 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl SCLogDebug("got a HTTP request and now logging !!"); /* reset */ - MemBufferReset(buffer); + MemBufferReset(jhl->buffer); JsonHttpLogJSON(jhl, js, tx, tx_id); - OutputJSONBuffer(js, jhl->httplog_ctx->file_ctx, buffer); + OutputJSONBuffer(js, jhl->httplog_ctx->file_ctx, &jhl->buffer); json_object_del(js, "http"); json_object_clear(js); diff --git a/src/output-json-netflow.c b/src/output-json-netflow.c index 1b1fa8c0bd..2adb368894 100644 --- a/src/output-json-netflow.c +++ b/src/output-json-netflow.c @@ -289,26 +289,25 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) { SCEnter(); JsonNetFlowLogThread *jhl = (JsonNetFlowLogThread *)thread_data; - MemBuffer *buffer = (MemBuffer *)jhl->buffer; /* reset */ - MemBufferReset(buffer); + MemBufferReset(jhl->buffer); json_t *js = CreateJSONHeaderFromFlow(f, "netflow", 0); //TODO const if (unlikely(js == NULL)) return TM_ECODE_OK; JsonNetFlowLogJSONToServer(jhl, js, f); - OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, buffer); + OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer); json_object_del(js, "netflow"); json_object_clear(js); json_decref(js); /* reset */ - MemBufferReset(buffer); + MemBufferReset(jhl->buffer); js = CreateJSONHeaderFromFlow(f, "netflow", 1); //TODO const if (unlikely(js == NULL)) return TM_ECODE_OK; JsonNetFlowLogJSONToClient(jhl, js, f); - OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, buffer); + OutputJSONBuffer(js, jhl->flowlog_ctx->file_ctx, &jhl->buffer); json_object_del(js, "netflow"); json_object_clear(js); json_decref(js); diff --git a/src/output-json-smtp.c b/src/output-json-smtp.c index 617b7247f3..364d4ff431 100644 --- a/src/output-json-smtp.c +++ b/src/output-json-smtp.c @@ -87,7 +87,6 @@ static int JsonSmtpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl { SCEnter(); JsonEmailLogThread *jhl = (JsonEmailLogThread *)thread_data; - MemBuffer *buffer = (MemBuffer *)jhl->buffer; json_t *sjs; json_t *js = CreateJSONHeaderWithTxId((Packet *)p, 1, "smtp", tx_id); @@ -95,7 +94,7 @@ static int JsonSmtpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl return TM_ECODE_OK; /* reset */ - MemBufferReset(buffer); + MemBufferReset(jhl->buffer); sjs = JsonSmtpDataLogger(f, state, tx, tx_id); if (sjs) { @@ -103,7 +102,7 @@ static int JsonSmtpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl } if (JsonEmailLogJson(jhl, js, p, f, state, tx, tx_id) == TM_ECODE_OK) { - OutputJSONBuffer(js, jhl->emaillog_ctx->file_ctx, buffer); + OutputJSONBuffer(js, jhl->emaillog_ctx->file_ctx, &jhl->buffer); } json_object_del(js, "email"); if (sjs) { diff --git a/src/output-json-ssh.c b/src/output-json-ssh.c index 3dc4d10f84..55e77a7ead 100644 --- a/src/output-json-ssh.c +++ b/src/output-json-ssh.c @@ -93,7 +93,6 @@ void JsonSshLogJSON(json_t *tjs, SshState *ssh_state) static int JsonSshLogger(ThreadVars *tv, void *thread_data, const Packet *p) { JsonSshLogThread *aft = (JsonSshLogThread *)thread_data; - MemBuffer *buffer = (MemBuffer *)aft->buffer; OutputSshCtx *ssh_ctx = aft->sshlog_ctx; if (unlikely(p->flow == NULL)) { @@ -125,13 +124,13 @@ static int JsonSshLogger(ThreadVars *tv, void *thread_data, const Packet *p) } /* reset */ - MemBufferReset(buffer); + MemBufferReset(aft->buffer); JsonSshLogJSON(tjs, ssh_state); json_object_set_new(js, "ssh", tjs); - OutputJSONBuffer(js, ssh_ctx->file_ctx, buffer); + OutputJSONBuffer(js, ssh_ctx->file_ctx, &aft->buffer); json_object_clear(js); json_decref(js); diff --git a/src/output-json-stats.c b/src/output-json-stats.c index a22f9d6b96..3fd37ad3d0 100644 --- a/src/output-json-stats.c +++ b/src/output-json-stats.c @@ -95,7 +95,6 @@ static int JsonStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable * { SCEnter(); JsonStatsLogThread *aft = (JsonStatsLogThread *)thread_data; - MemBuffer *buffer = (MemBuffer *)aft->buffer; const char delta_suffix[] = "_delta"; struct timeval tval; @@ -185,8 +184,8 @@ static int JsonStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable * json_object_set_new(js, "stats", js_stats); - OutputJSONBuffer(js, aft->statslog_ctx->file_ctx, buffer); - MemBufferReset(buffer); + OutputJSONBuffer(js, aft->statslog_ctx->file_ctx, &aft->buffer); + MemBufferReset(aft->buffer); json_object_clear(js_stats); json_object_del(js, "stats"); diff --git a/src/output-json-template.c b/src/output-json-template.c index d360e6746c..8a00d88beb 100644 --- a/src/output-json-template.c +++ b/src/output-json-template.c @@ -57,7 +57,6 @@ static int JsonTemplateLogger(ThreadVars *tv, void *thread_data, { TemplateTransaction *templatetx = tx; LogTemplateLogThread *thread = thread_data; - MemBuffer *buffer = thread->buffer; json_t *js, *templatejs; SCLogNotice("Logging template transaction %"PRIu64".", templatetx->tx_id); @@ -91,8 +90,8 @@ static int JsonTemplateLogger(ThreadVars *tv, void *thread_data, json_object_set_new(js, "template", templatejs); - MemBufferReset(buffer); - OutputJSONBuffer(js, thread->templatelog_ctx->file_ctx, buffer); + MemBufferReset(thread->buffer); + OutputJSONBuffer(js, thread->templatelog_ctx->file_ctx, &thread->buffer); json_decref(js); return TM_ECODE_OK; diff --git a/src/output-json-tls.c b/src/output-json-tls.c index d8b8078f97..3cedfd203a 100644 --- a/src/output-json-tls.c +++ b/src/output-json-tls.c @@ -129,7 +129,6 @@ void JsonTlsLogJSONExtended(json_t *tjs, SSLState * state) static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p) { JsonTlsLogThread *aft = (JsonTlsLogThread *)thread_data; - MemBuffer *buffer = (MemBuffer *)aft->buffer; OutputTlsCtx *tls_ctx = aft->tlslog_ctx; if (unlikely(p->flow == NULL)) { @@ -161,7 +160,7 @@ static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p) } /* reset */ - MemBufferReset(buffer); + MemBufferReset(aft->buffer); JsonTlsLogJSONBasic(tjs, ssl_state); @@ -171,7 +170,7 @@ static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p) json_object_set_new(js, "tls", tjs); - OutputJSONBuffer(js, tls_ctx->file_ctx, buffer); + OutputJSONBuffer(js, tls_ctx->file_ctx, &aft->buffer); json_object_clear(js); json_decref(js); diff --git a/src/output-json.c b/src/output-json.c index b9c2b886e6..7661725d52 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -117,7 +117,7 @@ void OutputJsonRegisterTests (void) #define DEFAULT_ALERT_SYSLOG_LEVEL LOG_INFO #define MODULE_NAME "OutputJSON" -#define OUTPUT_BUFFER_SIZE 65535 +#define OUTPUT_BUFFER_SIZE 65536 TmEcode OutputJson (ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); TmEcode OutputJsonThreadInit(ThreadVars *, void *, void **); @@ -335,31 +335,38 @@ json_t *CreateJSONHeaderWithTxId(Packet *p, int direction_sensitive, char *event return js; } +/* helper struct for the callback */ +typedef struct MemBufferWrapper_ { + MemBuffer **buffer; +} MemBufferWrapper; + static int MemBufferCallback(const char *str, size_t size, void *data) { - MemBuffer *memb = data; -#if 0 // can't expand, need a MemBuffer ** - /* since we can have many threads, the buffer might not be big enough. - * * Expand if necessary. */ - if (MEMBUFFER_OFFSET(memb) + size > MEMBUFFER_SIZE(memb)) { - MemBufferExpand(&memb, OUTPUT_BUFFER_SIZE); + MemBufferWrapper *wrapper = data; + MemBuffer **memb = wrapper->buffer; + + if (MEMBUFFER_OFFSET(*memb) + size >= MEMBUFFER_SIZE(*memb)) { + MemBufferExpand(memb, OUTPUT_BUFFER_SIZE); } -#endif - MemBufferWriteRaw(memb, str, size); + + MemBufferWriteRaw((*memb), str, size); return 0; } -int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer) +int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer) { if (file_ctx->sensor_name) { json_object_set_new(js, "host", json_string(file_ctx->sensor_name)); } - if (file_ctx->prefix) - MemBufferWriteRaw(buffer, file_ctx->prefix, file_ctx->prefix_len); + if (file_ctx->prefix) { + MemBufferWriteRaw((*buffer), file_ctx->prefix, file_ctx->prefix_len); + } + + MemBufferWrapper wrapper = { .buffer = buffer }; - int r = json_dump_callback(js, MemBufferCallback, buffer, + int r = json_dump_callback(js, MemBufferCallback, &wrapper, JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII| #ifdef JSON_ESCAPE_SLASH JSON_ESCAPE_SLASH @@ -370,7 +377,7 @@ int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer) if (r != 0) return TM_ECODE_OK; - LogFileWrite(file_ctx, buffer); + LogFileWrite(file_ctx, *buffer); return 0; } diff --git a/src/output-json.h b/src/output-json.h index b6aec6c05f..752f14992d 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -38,7 +38,7 @@ void JsonTcpFlags(uint8_t flags, json_t *js); json_t *CreateJSONHeader(Packet *p, int direction_sensative, char *event_type); json_t *CreateJSONHeaderWithTxId(Packet *p, int direction_sensitive, char *event_type, uint32_t tx_id); TmEcode OutputJSON(json_t *js, void *data, uint64_t *count); -int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer); +int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer); OutputCtx *OutputJsonInitCtx(ConfNode *); enum JsonFormat { COMPACT, INDENT }; diff --git a/src/util-buffer.h b/src/util-buffer.h index 58d5098a28..cc7616ccfb 100644 --- a/src/util-buffer.h +++ b/src/util-buffer.h @@ -135,7 +135,7 @@ void MemBufferFree(MemBuffer *buffer); \ if (((raw_buffer_len) >= (dst)->size - (dst)->offset)) { \ SCLogDebug("Truncating data write since it exceeded buffer limit of " \ - "- %"PRIu32"\n", (dst)->size); \ + "- %"PRIu32, (dst)->size); \ write_len = ((dst)->size - (dst)->offset) - 1; \ } else { \ write_len = (raw_buffer_len); \