output-json: drop eve records that are too long

In the situation where the mem buffer cannot be expanded to the
requested size, drop the log message.

For each JSON log context, a warning will be emitted once with a partial
bit of the log record being dropped to identify what event types may be
leading to large log records.

This also fixes the call to MemBufferExpand which is supposed be
passed the amount to expand by, not the new size required.

Ticket: #7300
(cherry picked from commit d39e42728a5b84d9cefbd4329034064d71c4e268)

diff --git a/src/output-json.c b/src/output-json.c
index 1d9f8b2146923e9f33baeee22b0750588e6328cb..00d99fc6972cf0ea664239bcd4e7d333babec957 100644 (file)
--- a/src/output-json.c
+++ b/src/output-json.c
@@ -987,8 +987,23 @@ int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx)
     }
 
     size_t jslen = jb_len(js);
-    if (MEMBUFFER_OFFSET(*buffer) + jslen >= MEMBUFFER_SIZE(*buffer)) {
-        MemBufferExpand(buffer, jslen);
+    DEBUG_VALIDATE_BUG_ON(jb_len(js) > UINT32_MAX);
+    size_t remaining = MEMBUFFER_SIZE(*buffer) - MEMBUFFER_OFFSET(*buffer);
+    if (jslen >= remaining) {
+        size_t expand_by = jslen + 1 - remaining;
+        if (MemBufferExpand(buffer, (uint32_t)expand_by) < 0) {
+            if (!ctx->too_large_warning) {
+                /* Log a warning once, and include enough of the log
+                 * message to hopefully identify the event_type. */
+                char partial[120];
+                size_t partial_len = MIN(sizeof(partial), jslen);
+                memcpy(partial, jb_ptr(js), partial_len - 1);
+                partial[partial_len - 1] = '\0';
+                SCLogWarning("Formatted JSON EVE record too large, will be dropped: %s", partial);
+                ctx->too_large_warning = true;
+            }
+            return 0;
+        }
     }
 
     MemBufferWriteRaw((*buffer), jb_ptr(js), jslen);

diff --git a/src/output-json.h b/src/output-json.h
index 74d07bb8a2c8dd720c1ca73caf2567fc2b1f3daa..dca6e88e700a09526577becbc7d084e4171c129e 100644 (file)
--- a/src/output-json.h
+++ b/src/output-json.h
@@ -90,6 +90,7 @@ typedef struct OutputJsonThreadCtx_ {
     OutputJsonCtx *ctx;
     LogFileCtx *file_ctx;
     MemBuffer *buffer;
+    bool too_large_warning;
 } OutputJsonThreadCtx;
 
 json_t *SCJsonString(const char *val);