detect: absent keyword to test absence of sticky buffer

Ticket: 2224

It takes an argument to match only if the buffer is absent,
or it can still match if the buffer is present, but we test
the absence of some content.

For multi buffers, absent matches if there are 0 buffers.

For file keywords, absent matches if there is no file.

diff --git a/doc/userguide/rules/payload-keywords.rst b/doc/userguide/rules/payload-keywords.rst
index 2e92b70462696560a34a6cb72779194cb35a62c6..780ad111f7dc82e5426e5fc2b652d2e76e2cb533 100644 (file)
--- a/doc/userguide/rules/payload-keywords.rst
+++ b/doc/userguide/rules/payload-keywords.rst
@@ -274,6 +274,27 @@ You can also use the negation (!) before isdataat.
 
 .. image:: payload-keywords/isdataat1.png
 
+absent
+------
+
+The keyword ``absent`` checks that a sticky buffer does not exist.
+It can be used without any argument to match only on absent buffer :
+
+Example of ``absent`` in a rule:
+
+.. container:: example-rule
+
+   alert http any any -> any any (msg:"HTTP request without referer";  :example-rule-emphasis:`http.referer; absent;` sid:1; rev:1;)
+
+
+It can take an argument "or_else" to match on absent buffer or on what comes next such as negated content, for instance :
+
+.. container:: example-rule
+
+   alert http any any -> any any (msg:"HTTP request without referer";  :example-rule-emphasis:`http.referer; absent: or_else;` content: !"abc"; sid:1; rev:1;)
+
+For files (ie ``file.data``), absent means there are no files in the transaction.
+
 bsize
 -----
 

diff --git a/src/detect-engine-analyzer.c b/src/detect-engine-analyzer.c
index d852792fd3da24869992897b7d919f710e334b84..dae6f9f2f27065a5b46fbac2168ddf10c547fa43 100644 (file)
--- a/src/detect-engine-analyzer.c
+++ b/src/detect-engine-analyzer.c
@@ -39,6 +39,7 @@
 #include "detect-pcre.h"
 #include "detect-bytejump.h"
 #include "detect-bytetest.h"
+#include "detect-isdataat.h"
 #include "detect-flow.h"
 #include "detect-tcp-flags.h"
 #include "detect-tcp-ack.h"
@@ -855,6 +856,14 @@ static void DumpMatches(RuleAnalyzer *ctx, JsonBuilder *js, const SigMatchData *
                 jb_close(js);
                 break;
             }
+            case DETECT_ABSENT: {
+                const DetectAbsentData *dad = (const DetectAbsentData *)smd->ctx;
+                jb_open_object(js, "absent");
+                jb_set_bool(js, "or_else", dad->or_else);
+                jb_close(js);
+                break;
+            }
+
             case DETECT_IPOPTS: {
                 const DetectIpOptsData *cd = (const DetectIpOptsData *)smd->ctx;
 

diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c
index d4dab42816d5b9ba7da942d6a272e6ccd529ef31..e43e693b21519daa969c70d24fbb2deadf81c923 100644 (file)
--- a/src/detect-engine-content-inspection.c
+++ b/src/detect-engine-content-inspection.c
@@ -382,6 +382,13 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
                     prev_offset);
         } while(1);
 
+    } else if (smd->type == DETECT_ABSENT) {
+        const DetectAbsentData *id = (DetectAbsentData *)smd->ctx;
+        if (!id->or_else) {
+            // we match only on absent buffer
+            goto no_match;
+        }
+        goto match;
     } else if (smd->type == DETECT_ISDATAAT) {
         SCLogDebug("inspecting isdataat");
 
@@ -646,8 +653,7 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
             goto match;
         }
         goto no_match_discontinue;
-    }
-    else if (smd->type == DETECT_LUA) {
+    } else if (smd->type == DETECT_LUA) {
         SCLogDebug("lua starting");
 
         if (DetectLuaMatchBuffer(det_ctx, s, smd, buffer, buffer_len,
@@ -758,6 +764,24 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh
         return false;
 }
 
+bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd)
+{
+    // we will match on NULL buffers there is one absent
+    bool absent_data = false;
+    while (1) {
+        if (smd->type == DETECT_ABSENT) {
+            absent_data = true;
+            break;
+        }
+        if (smd->is_last) {
+            break;
+        }
+        // smd does not get reused after this loop
+        smd++;
+    }
+    return absent_data;
+}
+
 #ifdef UNITTESTS
 #include "tests/detect-engine-content-inspection.c"
 #endif

diff --git a/src/detect-engine-content-inspection.h b/src/detect-engine-content-inspection.h
index 2c253b77ad3d326c8128a6f7d04087748234ffcf..fe53086badd998eb6afabb1b7e93538469e43555 100644 (file)
--- a/src/detect-engine-content-inspection.h
+++ b/src/detect-engine-content-inspection.h
@@ -68,6 +68,12 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh
         const Signature *s, const SigMatchData *smd, Packet *p, Flow *f, const InspectionBuffer *b,
         const enum DetectContentInspectionType inspection_mode);
 
+/** \brief tells if we should match on absent buffer, because
+ *  there is an absent keyword being used
+ *  \param smd array of content inspection matches
+ *  \retval bool true to match on absent buffer, false otherwise */
+bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd);
+
 void DetectEngineContentInspectionRegisterTests(void);
 
 #endif /* SURICATA_DETECT_ENGINE_CONTENT_INSPECTION_H */

diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
index 5e8687e34686336f16a70de41c7738fbff1f8397..1c9984ea9541b1c12d5a870ff65eb5d53fad04f9 100644 (file)
--- a/src/detect-engine-mpm.c
+++ b/src/detect-engine-mpm.c
@@ -1125,6 +1125,9 @@ void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s)
         }
 
         for (SigMatch *sm = s->init_data->buffers[x].head; sm != NULL; sm = sm->next) {
+            // a buffer with absent keyword cannot be used as fast_pattern
+            if (sm->type == DETECT_ABSENT)
+                break;
             if (sm->type != DETECT_CONTENT)
                 continue;
 

diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h
index db4cd957af9d8a7b563041ffb51c623650a8c18d..e5f550be7547a1b62fdcf286eb6bb587b652f907 100644 (file)
--- a/src/detect-engine-register.h
+++ b/src/detect-engine-register.h
@@ -93,6 +93,7 @@ enum DetectKeywordId {
     DETECT_LUA,
     DETECT_ISDATAAT,
     DETECT_AL_URILEN,
+    DETECT_ABSENT,
     /* end of content inspection */
 
     DETECT_METADATA,

diff --git a/src/detect-engine.c b/src/detect-engine.c
index 77c25a1cf3a90fcd5f17213f976ba8dbe0fcb191..f4195739e2c11897199cb491d6cf58d027901b1e 100644 (file)
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -666,6 +666,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
     new_engine->sm_list = t->sm_list;
     new_engine->sm_list_base = t->sm_list_base;
     new_engine->smd = smd;
+    new_engine->match_on_null = DetectContentInspectionMatchOnAbsentBuffer(smd);
     new_engine->progress = t->progress;
     new_engine->v2 = t->v2;
     SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback,
@@ -2172,6 +2173,9 @@ uint8_t DetectEngineInspectBufferGeneric(DetectEngineCtx *de_ctx, DetectEngineTh
     const InspectionBuffer *buffer = engine->v2.GetData(det_ctx, transforms,
             f, flags, txv, list_id);
     if (unlikely(buffer == NULL)) {
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
         return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH :
                      DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     }
@@ -2233,6 +2237,14 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx,
         }
         local_id++;
     } while (1);
+    if (local_id == 0) {
+        // That means we did not get even one buffer value from the multi-buffer
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+    }
     return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 }
 

diff --git a/src/detect-file-data.c b/src/detect-file-data.c
index a721c08c7cf9d514469e049f2c428d94c5ea6897..d976b51c00b49491804f8c96d946b011d410ca46 100644 (file)
--- a/src/detect-file-data.c
+++ b/src/detect-file-data.c
@@ -403,6 +403,14 @@ uint8_t DetectEngineInspectFiledata(DetectEngineCtx *de_ctx, DetectEngineThreadC
     if (ffc == NULL) {
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
     }
+    if (ffc->head == NULL) {
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+        return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+    }
 
     int local_file_id = 0;
     File *file = ffc->head;

diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c
index f23434d8666ece33df1c938f0222e41a14f4f842..0f8d94a7b5b16e5d817a0492eaa6400eb7b71ce2 100644 (file)
--- a/src/detect-filemagic.c
+++ b/src/detect-filemagic.c
@@ -307,7 +307,15 @@ static uint8_t DetectEngineInspectFilemagic(DetectEngineCtx *de_ctx, DetectEngin
 
     AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags);
     FileContainer *ffc = files.fc;
-    if (ffc == NULL) {
+    if (ffc == NULL || ffc->head == NULL) {
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+        if (ffc != NULL) {
+            return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+        }
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
     }
 

diff --git a/src/detect-filename.c b/src/detect-filename.c
index f75fdbd680fe10e37e3bc9cc9acb46eb98038a62..ef144cf44086179ddf4348e6a86e38aaa1959744 100644 (file)
--- a/src/detect-filename.c
+++ b/src/detect-filename.c
@@ -244,7 +244,15 @@ static uint8_t DetectEngineInspectFilename(DetectEngineCtx *de_ctx, DetectEngine
 
     AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags);
     FileContainer *ffc = files.fc;
-    if (ffc == NULL) {
+    if (ffc == NULL || ffc->head == NULL) {
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+        if (ffc != NULL) {
+            return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+        }
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
     }
 

diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c
index 5e5604ea594d1666aa1dc2cd67f4ba89eb3d6081..7747b61b858bcbc0cc6b39af0c989bb3513cfb1f 100644 (file)
--- a/src/detect-http-client-body.c
+++ b/src/detect-http-client-body.c
@@ -312,6 +312,9 @@ static uint8_t DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx,
     const InspectionBuffer *buffer = HttpRequestBodyGetDataCallback(
             det_ctx, engine->v2.transforms, f, flags, txv, engine->sm_list, engine->sm_list_base);
     if (buffer == NULL || buffer->inspect == NULL) {
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
         return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     }
 

diff --git a/src/detect-http-header.c b/src/detect-http-header.c
index 0a1a8ecadeed043ee4fff8d99b3092693efb18a6..22a487583ae7bd1708618836bcd4f6ed305cc3a0 100644 (file)
--- a/src/detect-http-header.c
+++ b/src/detect-http-header.c
@@ -191,6 +191,9 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx,
         uint8_t *rawdata = GetBufferForTX(txv, det_ctx, f, flags, &rawdata_len);
         if (rawdata_len == 0) {
             SCLogDebug("no data");
+            if (engine->match_on_null && eof) {
+                return DETECT_ENGINE_INSPECT_SIG_MATCH;
+            }
             goto end;
         }
         /* setup buffer and apply transforms */

diff --git a/src/detect-isdataat.c b/src/detect-isdataat.c
index 7b4d629ad3a112dc040d0e9a2e88c6a2161c8ecf..6bc4e961c07ad03acdce8b63cb871539b92ba757 100644 (file)
--- a/src/detect-isdataat.c
+++ b/src/detect-isdataat.c
@@ -35,6 +35,7 @@
 
 #include "detect-isdataat.h"
 #include "detect-content.h"
+#include "detect-bytetest.h"
 #include "detect-uricontent.h"
 #include "detect-engine-build.h"
 
@@ -56,11 +57,99 @@ static DetectParseRegex parse_regex;
 int DetectIsdataatSetup (DetectEngineCtx *, Signature *, const char *);
 #ifdef UNITTESTS
 static void DetectIsdataatRegisterTests(void);
+static void DetectAbsentRegisterTests(void);
 #endif
 void DetectIsdataatFree(DetectEngineCtx *, void *);
 
 static int DetectEndsWithSetup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr);
 
+static void DetectAbsentFree(DetectEngineCtx *de_ctx, void *ptr)
+{
+    SCFree(ptr);
+}
+
+static int DetectAbsentSetup(DetectEngineCtx *de_ctx, Signature *s, const char *optstr)
+{
+    if (s->init_data->list == DETECT_SM_LIST_NOTSET) {
+        SCLogError("no buffer for absent keyword");
+        return -1;
+    }
+
+    if (DetectBufferGetActiveList(de_ctx, s) == -1)
+        return -1;
+
+    bool or_else;
+    if (optstr == NULL) {
+        or_else = false;
+    } else if (strcmp(optstr, "or_else") == 0) {
+        or_else = true;
+    } else {
+        SCLogError("unhandled value for absent keyword: %s", optstr);
+        return -1;
+    }
+    if (s->init_data->curbuf == NULL || s->init_data->list != (int)s->init_data->curbuf->id) {
+        SCLogError("unspected buffer for absent keyword");
+        return -1;
+    }
+    const DetectBufferType *b = DetectEngineBufferTypeGetById(de_ctx, s->init_data->list);
+    if (!b || b->frame) {
+        SCLogError("absent does not work with frames");
+        return -1;
+    }
+    if (s->init_data->curbuf->tail != NULL) {
+        SCLogError("absent must come first right after buffer");
+        return -1;
+    }
+    DetectAbsentData *dad = SCMalloc(sizeof(DetectAbsentData));
+    if (unlikely(dad == NULL))
+        return -1;
+
+    dad->or_else = or_else;
+
+    if (SigMatchAppendSMToList(de_ctx, s, DETECT_ABSENT, (SigMatchCtx *)dad, s->init_data->list) ==
+            NULL) {
+        DetectAbsentFree(de_ctx, dad);
+        return -1;
+    }
+    return 0;
+}
+
+bool DetectAbsentValidateContentCallback(Signature *s, const SignatureInitDataBuffer *b)
+{
+    bool has_other = false;
+    bool only_absent = false;
+    bool has_absent = false;
+    for (const SigMatch *sm = b->head; sm != NULL; sm = sm->next) {
+        if (sm->type == DETECT_ABSENT) {
+            has_absent = true;
+            const DetectAbsentData *dad = (const DetectAbsentData *)sm->ctx;
+            if (!dad->or_else) {
+                only_absent = true;
+            }
+        } else {
+            has_other = true;
+            if (sm->type == DETECT_CONTENT) {
+                const DetectContentData *cd = (DetectContentData *)sm->ctx;
+                if (has_absent && (cd->flags & DETECT_CONTENT_FAST_PATTERN)) {
+                    SCLogError("signature can't have absent and fast_pattern on the same buffer");
+                    return false;
+                }
+            }
+        }
+    }
+
+    if (only_absent && has_other) {
+        SCLogError("signature can't have a buffer tested absent and tested with other keywords "
+                   "such as content");
+        return false;
+    } else if (has_absent && !only_absent && !has_other) {
+        SCLogError(
+                "signature with absent: or_else expects other keywords to test on such as content");
+        return false;
+    }
+    return true;
+}
+
 /**
  * \brief Registration function for isdataat: keyword
  */
@@ -82,6 +171,16 @@ void DetectIsdataatRegister(void)
     sigmatch_table[DETECT_ENDS_WITH].Setup = DetectEndsWithSetup;
     sigmatch_table[DETECT_ENDS_WITH].flags = SIGMATCH_NOOPT;
 
+    sigmatch_table[DETECT_ABSENT].name = "absent";
+    sigmatch_table[DETECT_ABSENT].desc = "test if the buffer is absent";
+    sigmatch_table[DETECT_ABSENT].url = "/rules/payload-keywords.html#absent";
+    sigmatch_table[DETECT_ABSENT].Setup = DetectAbsentSetup;
+    sigmatch_table[DETECT_ABSENT].Free = DetectAbsentFree;
+    sigmatch_table[DETECT_ABSENT].flags = SIGMATCH_OPTIONAL_OPT;
+#ifdef UNITTESTS
+    sigmatch_table[DETECT_ABSENT].RegisterTests = DetectAbsentRegisterTests;
+#endif
+
     DetectSetupParseRegexes(PARSE_REGEX, &parse_regex);
 }
 
@@ -584,4 +683,41 @@ void DetectIsdataatRegisterTests(void)
     UtRegisterTest("DetectIsdataatTestPacket02", DetectIsdataatTestPacket02);
     UtRegisterTest("DetectIsdataatTestPacket03", DetectIsdataatTestPacket03);
 }
+
+static int DetectAbsentTestParse01(void)
+{
+    DetectEngineCtx *de_ctx = DetectEngineCtxInit();
+    FAIL_IF(de_ctx == NULL);
+    de_ctx->flags |= DE_QUIET;
+
+    Signature *s = DetectEngineAppendSig(de_ctx,
+            "alert http any any -> any any "
+            "(msg:\"invalid absent only with negated content\"; http.user_agent; "
+            "absent; content:!\"one\"; sid:2;)");
+    FAIL_IF(s != NULL);
+    s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any "
+                                      "(msg:\"invalid absent\"; http.user_agent; "
+                                      "content:!\"one\"; absent; sid:2;)");
+    FAIL_IF(s != NULL);
+    s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any "
+                                      "(msg:\"invalid absent\"; http.user_agent; "
+                                      "content:\"one\"; absent: or_else; sid:2;)");
+    FAIL_IF(s != NULL);
+    s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any "
+                                      "(msg:\"absent without sticky buffer\"; "
+                                      "content:!\"one\"; absent: or_else; sid:2;)");
+    FAIL_IF(s != NULL);
+    s = DetectEngineAppendSig(de_ctx,
+            "alert websocket any any -> any any "
+            "(msg:\"absent with frame\"; "
+            "frame: websocket.pdu; absent: or_else; content:!\"one\"; sid:2;)");
+    FAIL_IF(s != NULL);
+    DetectEngineCtxFree(de_ctx);
+    PASS;
+}
+
+void DetectAbsentRegisterTests(void)
+{
+    UtRegisterTest("DetectAbsentTestParse01", DetectAbsentTestParse01);
+}
 #endif

diff --git a/src/detect-isdataat.h b/src/detect-isdataat.h
index 01ea2e304f42172f57cdcf4a1454669c12748643..c0cff884838532bc83aecd2306a8ea477c6ad62a 100644 (file)
--- a/src/detect-isdataat.h
+++ b/src/detect-isdataat.h
@@ -34,7 +34,14 @@ typedef struct DetectIsdataatData_ {
     uint8_t flags; /* isdataat options*/
 } DetectIsdataatData;
 
+typedef struct DetectAbsentData_ {
+    /** absent or try to match with other keywords (false means only absent) */
+    bool or_else;
+} DetectAbsentData;
+
 /* prototypes */
 void DetectIsdataatRegister (void);
 
+bool DetectAbsentValidateContentCallback(Signature *s, const SignatureInitDataBuffer *);
+
 #endif /* SURICATA_DETECT_ISDATAAT_H */

diff --git a/src/detect-parse.c b/src/detect-parse.c
index 984501c1dd8a138a4e2d09f0545c19a44ebb9e7a..3b03dfb92b361620160561a245e5a63841afeeac 100644 (file)
--- a/src/detect-parse.c
+++ b/src/detect-parse.c
@@ -35,6 +35,7 @@
 
 #include "detect-content.h"
 #include "detect-bsize.h"
+#include "detect-isdataat.h"
 #include "detect-pcre.h"
 #include "detect-uricontent.h"
 #include "detect-reference.h"
@@ -1971,6 +1972,9 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
         if (!DetectBsizeValidateContentCallback(s, b)) {
             SCReturnInt(0);
         }
+        if (!DetectAbsentValidateContentCallback(s, b)) {
+            SCReturnInt(0);
+        }
     }
 
     int ts_excl = 0;

diff --git a/src/detect.h b/src/detect.h
index 4c22f509a3f3edc48b97798623cdaf0a75c396da..e88b5540ac797df98dc91983d75d1aadb0a03eb4 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -432,6 +432,8 @@ typedef struct DetectEngineAppInspectionEngine_ {
     uint8_t id;     /**< per sig id used in state keeping */
     bool mpm;
     bool stream;
+    /** will match on a NULL buffer (so an absent buffer) */
+    bool match_on_null;
     uint16_t sm_list;
     uint16_t sm_list_base; /**< base buffer being transformed */
     int16_t progress;