output-json: drop eve records that are too long

In the situation where the mem buffer cannot be expanded to the
requested size, drop the log message.

For each JSON log context, a warning will be emitted once with a partial
bit of the log record being dropped to identify what event types may be
leading to large log records.

This also fixes the call to MemBufferExpand which is supposed be
passed the amount to expand by, not the new size required.

Ticket: #7300

diff --git a/src/output-json.c b/src/output-json.c
index 18376fd428a5b7bcd025ac39af58e8587974392a..4b42b6802d0565f92394c13cc994523fce9f35e5 100644 (file)
--- a/src/output-json.c
+++ b/src/output-json.c
@@ -980,8 +980,22 @@ int OutputJsonBuilderBuffer(
 
     size_t jslen = jb_len(js);
     DEBUG_VALIDATE_BUG_ON(jb_len(js) > UINT32_MAX);
-    if (MEMBUFFER_OFFSET(*buffer) + jslen >= MEMBUFFER_SIZE(*buffer)) {
-        MemBufferExpand(buffer, (uint32_t)jslen);
+    size_t remaining = MEMBUFFER_SIZE(*buffer) - MEMBUFFER_OFFSET(*buffer);
+    if (jslen >= remaining) {
+        size_t expand_by = jslen + 1 - remaining;
+        if (MemBufferExpand(buffer, (uint32_t)expand_by) < 0) {
+            if (!ctx->too_large_warning) {
+                /* Log a warning once, and include enough of the log
+                 * message to hopefully identify the event_type. */
+                char partial[120];
+                size_t partial_len = MIN(sizeof(partial), jslen);
+                memcpy(partial, jb_ptr(js), partial_len - 1);
+                partial[partial_len - 1] = '\0';
+                SCLogWarning("Formatted JSON EVE record too large, will be dropped: %s", partial);
+                ctx->too_large_warning = true;
+            }
+            return 0;
+        }
     }
 
     MemBufferWriteRaw((*buffer), jb_ptr(js), (uint32_t)jslen);

diff --git a/src/output-json.h b/src/output-json.h
index 89597e616a0f51a90c37d65f7a26574a93cbe667..205ed445d4d6720bb7f62cb6e2b41fcacc7cab6b 100644 (file)
--- a/src/output-json.h
+++ b/src/output-json.h
@@ -90,6 +90,7 @@ typedef struct OutputJsonThreadCtx_ {
     OutputJsonCtx *ctx;
     LogFileCtx *file_ctx;
     MemBuffer *buffer;
+    bool too_large_warning;
 } OutputJsonThreadCtx;
 
 json_t *SCJsonString(const char *val);