detect: log app-layer metadata in alert with single tx

Ticket: 7199

Uses a config parameter detect.guess-applayer-tx to enable
this behavior (off by default)

This feature is requested for use cases with signatures not
using app-layer keywords but still targetting application
layer transactions, such as pass/drop rule combination,
or lua usage.

This overrides the previous behavior of checking if the signature
has a content match, by checking if there is only one live
transaction, in addition to the config parameter being set.

(cherry picked from commit f2c37763149f16da17326cb313750052e5a2117d)

diff --git a/doc/userguide/output/eve/eve-json-output.rst b/doc/userguide/output/eve/eve-json-output.rst
index 364a80418d3998545de68ab66d9a6286a88bd3bd..f6ff28c8ae463f308878986638b9daa5018e032e 100644 (file)
--- a/doc/userguide/output/eve/eve-json-output.rst
+++ b/doc/userguide/output/eve/eve-json-output.rst
@@ -62,6 +62,21 @@ Alerts are event records for rule matches. They can be amended with
 metadata, such as the application layer record (HTTP, DNS, etc) an
 alert was generated for, and elements of the rule.
 
+The alert is amended with application layer metadata for signatures
+using application layer keywords. It is also the case for protocols
+over UDP as each single packet is expected to contain a PDU.
+
+For other signatures, the option ``guess-applayer-tx``
+can be used to force the detect engine to tie a transaction
+to an alert.
+This transaction is not guaranteed to be the relevant one,
+depending on your use case and how you define relevant here.
+If there are multiple live transactions, none will get
+picked up.
+The alert event will have ``"tx_guessed": true`` to recognize
+these alerts.
+
+
 Metadata::
 
         - alert:

diff --git a/doc/userguide/upgrade.rst b/doc/userguide/upgrade.rst
index 09c18d96b08b9d259d7b3095521048236d0b2325..816d7e22db2158e9586f5a0eb4dcaddc692fd068 100644 (file)
--- a/doc/userguide/upgrade.rst
+++ b/doc/userguide/upgrade.rst
@@ -59,6 +59,10 @@ Upgrading to 7.0.8
             7.0 releases. It will not be provided in
             Suricata 8. Please fix any rules that depend on this
             behavior.
+- Application layer metadata is logged with alerts by default **only for rules that
+  use application layer keywords**. For other rules, the configuration parameter
+  ``detect.guess-applayer-tx`` can be used to force the detect engine to find a
+  transaction, which is not guaranteed to be the one you expect.
 
 Upgrading 6.0 to 7.0
 --------------------

diff --git a/etc/schema.json b/etc/schema.json
index 8683bb4816bf212d1e6a3271ee0a05423f96d1fa..488cf3d511e7ab171341773c47e6fb5286170a6d 100644 (file)
--- a/etc/schema.json
+++ b/etc/schema.json
@@ -112,6 +112,10 @@
         "tx_id": {
             "type": "integer"
         },
+        "tx_guessed": {
+            "description": "the signature that triggered this alert didn't tie to a transaction, so the transaction (and metadata) logged is a forced estimation and may not be the one you expect",
+            "type": "boolean"
+        },
         "files": {
             "type": "array",
             "minItems": 1,

diff --git a/src/decode.h b/src/decode.h
index da41726e806b1022225e63b1f3c65dc7f3ba91ea..103e42c6fc1d129bd849f07dc513f6854f007346 100644 (file)
--- a/src/decode.h
+++ b/src/decode.h
@@ -281,6 +281,8 @@ typedef struct PacketAlert_ {
 #define PACKET_ALERT_RATE_FILTER_MODIFIED   0x10
 /** alert is in a frame, frame_id set */
 #define PACKET_ALERT_FLAG_FRAME 0x20
+/** alert in a tx was forced */
+#define PACKET_ALERT_FLAG_TX_GUESSED 0x040
 
 extern uint16_t packet_alert_max;
 #define PACKET_ALERT_MAX 15

diff --git a/src/detect-engine.c b/src/detect-engine.c
index f1c47b0fa356a28ada73087475242423e6026f73..a7b6ee1f26c4a3168b80b4bcc08b1296f1e66444 100644 (file)
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -2922,7 +2922,14 @@ static int DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx)
     SCLogDebug("de_ctx->inspection_recursion_limit: %d",
                de_ctx->inspection_recursion_limit);
 
-    /* parse port grouping whitelisting settings */
+    int guess_applayer = 0;
+    if ((ConfGetBool("detect.guess-applayer-tx", &guess_applayer)) == 1) {
+        if (guess_applayer == 1) {
+            de_ctx->guess_applayer = true;
+        }
+    }
+
+    /* parse port grouping priority settings */
 
     const char *ports = NULL;
     (void)ConfGet("detect.grouping.tcp-whitelist", &ports);

diff --git a/src/detect.c b/src/detect.c
index e02977bd01126dc901ce29e02b1c8f4897330455..f3cf0e0ba373c38b9c8975f67a0e2249682098fe 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -818,16 +818,19 @@ static inline void DetectRulePacketRules(
         DetectRunPostMatch(tv, det_ctx, p, s);
 
         uint64_t txid = PACKET_ALERT_NOTX;
-        if ((alert_flags & PACKET_ALERT_FLAG_STREAM_MATCH) ||
-                (s->alproto != ALPROTO_UNKNOWN && pflow->proto == IPPROTO_UDP)) {
-            // if there is a stream match (TCP), or
-            // a UDP specific app-layer signature,
-            // try to use the good tx for the packet direction
-            if (pflow->alstate) {
-                uint8_t dir =
-                        (p->flowflags & FLOW_PKT_TOCLIENT) ? STREAM_TOCLIENT : STREAM_TOSERVER;
-                txid = AppLayerParserGetTransactionInspectId(pflow->alparser, dir);
+        if (pflow && pflow->alstate) {
+            uint8_t dir = (p->flowflags & FLOW_PKT_TOCLIENT) ? STREAM_TOCLIENT : STREAM_TOSERVER;
+            txid = AppLayerParserGetTransactionInspectId(pflow->alparser, dir);
+            if ((s->alproto != ALPROTO_UNKNOWN && pflow->proto == IPPROTO_UDP) ||
+                    (de_ctx->guess_applayer &&
+                            AppLayerParserGetTxCnt(pflow, pflow->alstate) == txid + 1)) {
+                // if there is a UDP specific app-layer signature,
+                // or only one live transaction
+                // try to use the good tx for the packet direction
                 alert_flags |= PACKET_ALERT_FLAG_TX;
+                if (pflow->proto != IPPROTO_UDP) {
+                    alert_flags |= PACKET_ALERT_FLAG_TX_GUESSED;
+                }
             }
         }
         AlertQueueAppend(det_ctx, s, p, txid, alert_flags);

diff --git a/src/detect.h b/src/detect.h
index f147d0fda8e71744ad8b6d3356738badb6141725..2379d325382930f4a3ef2c03d9038976631af9d7 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -885,6 +885,9 @@ typedef struct DetectEngineCtx_ {
     /* maximum recursion depth for content inspection */
     int inspection_recursion_limit;
 
+    /* force app-layer tx finding for alerts with signatures not having app-layer keywords */
+    bool guess_applayer;
+
     /* registration id for per thread ctx for the filemagic/file.magic keywords */
     int filemagic_thread_ctx_id;
 

diff --git a/src/output-json-alert.c b/src/output-json-alert.c
index 4005c48800c28b6f7dfd346f1674d655ee9460c1..72127e5d0f14a17be2f320fc99213da13ae9e1e0 100644 (file)
--- a/src/output-json-alert.c
+++ b/src/output-json-alert.c
@@ -378,6 +378,9 @@ void AlertJsonHeader(void *ctx, const Packet *p, const PacketAlert *pa, JsonBuil
     if (pa->flags & PACKET_ALERT_FLAG_TX) {
         jb_set_uint(js, "tx_id", pa->tx_id);
     }
+    if (pa->flags & PACKET_ALERT_FLAG_TX_GUESSED) {
+        jb_set_bool(js, "tx_guessed", true);
+    }
 
     jb_open_object(js, "alert");
 

diff --git a/suricata.yaml.in b/suricata.yaml.in
index ba9005dc3e4ce37c667ca055f85c6362895c179b..504bfd0b66b85a96708fdd2e839d63022e2e6887 100644 (file)
--- a/suricata.yaml.in
+++ b/suricata.yaml.in
@@ -1677,6 +1677,11 @@ detect:
     toserver-groups: 25
   sgh-mpm-context: auto
   inspection-recursion-limit: 3000
+  # try to tie an app-layer transaction for rules without app-layer keywords
+  # if there is only one live transaction for the flow
+  # allows to log app-layer metadata in alert
+  # but the transaction may not be the relevant one.
+  # guess-applayer-tx: no
   # If set to yes, the loading of signatures will be made after the capture
   # is started. This will limit the downtime in IPS mode.
   #delayed-detect: yes