From: Jeff Lucovsky <jeff@lucovsky.org>
Date: Sun, 31 Mar 2019 12:36:58 +0000 (-0700)
Subject: detect/http: Use v2 inspect and mpm engines
X-Git-Tag: suricata-5.0.0-beta1~72
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=80be07a534942b720f6718651b585c2634bb6b57;p=thirdparty%2Fsuricata.git

detect/http: Use v2 inspect and mpm engines

This changeset updates the http stub detect logic to use the v2 inspect
and npm engines.
---

diff --git a/src/detect-http-accept-enc.c b/src/detect-http-accept-enc.c
index 702706e18e..277e7142a2 100644
--- a/src/detect-http-accept-enc.c
+++ b/src/detect-http-accept-enc.c
@@ -30,7 +30,8 @@
  * Implements http_accept_enc sticky buffer
  */
 
-#define KEYWORD_NAME "http_accept_enc"
+#define KEYWORD_NAME_LEGACY "http_accept_enc"
+#define KEYWORD_NAME "http.accept_enc"
 #define KEYWORD_DOC "http-keywords.html#http-accept-enc"
 #define BUFFER_NAME "http_accept_enc"
 #define BUFFER_DESC "http accept encoding header"
diff --git a/src/detect-http-accept-lang.c b/src/detect-http-accept-lang.c
index 27cc822755..c83847e69a 100644
--- a/src/detect-http-accept-lang.c
+++ b/src/detect-http-accept-lang.c
@@ -30,7 +30,8 @@
  * Implements http_accept_lang sticky buffer
  */
 
-#define KEYWORD_NAME "http_accept_lang"
+#define KEYWORD_NAME_LEGACY "http_accept_lang"
+#define KEYWORD_NAME "http.accept_lang"
 #define KEYWORD_DOC "http-keywords.html#http-accept-lang"
 #define BUFFER_NAME "http_accept_lang"
 #define BUFFER_DESC "http accept language header"
diff --git a/src/detect-http-accept.c b/src/detect-http-accept.c
index c52b95a865..db6bec4455 100644
--- a/src/detect-http-accept.c
+++ b/src/detect-http-accept.c
@@ -30,7 +30,8 @@
  * Implements support http_header_*
  */
 
-#define KEYWORD_NAME "http_accept"
+#define KEYWORD_NAME_LEGACY "http_accept"
+#define KEYWORD_NAME "http.accept"
 #define KEYWORD_DOC "http-keywords.html#http-accept"
 #define BUFFER_NAME "http_accept"
 #define BUFFER_DESC "http accept header"
diff --git a/src/detect-http-connection.c b/src/detect-http-connection.c
index 01184894a4..2995898df7 100644
--- a/src/detect-http-connection.c
+++ b/src/detect-http-connection.c
@@ -30,7 +30,8 @@
  * Implements the http_connection sticky buffer
  */
 
-#define KEYWORD_NAME "http_connection"
+#define KEYWORD_NAME_LEGACY "http_connection"
+#define KEYWORD_NAME "http.connection"
 #define KEYWORD_DOC "http-keywords.html#http-connection"
 #define BUFFER_NAME "http_connection"
 #define BUFFER_DESC "http connection header"
diff --git a/src/detect-http-content-len.c b/src/detect-http-content-len.c
index 48dd031790..b9203e2c73 100644
--- a/src/detect-http-content-len.c
+++ b/src/detect-http-content-len.c
@@ -30,7 +30,8 @@
  * Implements http_content_len sticky buffer
  */
 
-#define KEYWORD_NAME "http_content_len"
+#define KEYWORD_NAME_LEGACY "http_content_len"
+#define KEYWORD_NAME "http.content_len"
 #define KEYWORD_DOC "http-keywords.html#http-content-len"
 #define BUFFER_NAME "http_content_len"
 #define BUFFER_DESC "http content length header"
diff --git a/src/detect-http-content-type.c b/src/detect-http-content-type.c
index c740267f62..66d5b95675 100644
--- a/src/detect-http-content-type.c
+++ b/src/detect-http-content-type.c
@@ -30,7 +30,8 @@
  * Implement http_content_type sticky buffer
  */
 
-#define KEYWORD_NAME "http_content_type"
+#define KEYWORD_NAME_LEGACY "http_content_type"
+#define KEYWORD_NAME "http.content_type"
 #define KEYWORD_DOC "http-keywords.html#http-content-type"
 #define BUFFER_NAME "http_content_type"
 #define BUFFER_DESC "http content type header"
diff --git a/src/detect-http-headers-stub.h b/src/detect-http-headers-stub.h
index c76574411f..fb1f0b15c8 100644
--- a/src/detect-http-headers-stub.h
+++ b/src/detect-http-headers-stub.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2017 Open Information Security Foundation
+/* Copyright (C) 2007-2019 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -32,6 +32,7 @@
 #include "flow.h"
 #include "app-layer.h"
 #include "app-layer-parser.h"
+#include "app-layer-protos.h"
 #include "app-layer-htp.h"
 
 #include "detect.h"
@@ -49,305 +50,121 @@
 static int g_buffer_id = 0;
 
 #ifdef KEYWORD_TOSERVER
-/** \brief HTTP Headers Mpm prefilter callback
- *
- *  \param det_ctx detection engine thread ctx
- *  \param p packet to inspect
- *  \param f flow to inspect
- *  \param txv tx to inspect
- *  \param pectx inspection context
- */
-static void PrefilterTxHttpRequestHeader(DetectEngineThreadCtx *det_ctx,
-        const void *pectx,
-        Packet *p, Flow *f, void *txv,
-        const uint64_t idx, const uint8_t flags)
+static InspectionBuffer *GetRequestData(DetectEngineThreadCtx *det_ctx,
+        const DetectEngineTransforms *transforms, Flow *_f,
+        const uint8_t _flow_flags, void *txv, const int list_id)
 {
     SCEnter();
-    const MpmCtx *mpm_ctx = (MpmCtx *)pectx;
-    htp_tx_t *tx = (htp_tx_t *)txv;
 
-    if (tx->request_headers == NULL)
-        return;
-
-    htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
-                                                      HEADER_NAME);
-    if (h == NULL || h->value == NULL) {
-        SCLogDebug("HTTP %s header not present in this request", HEADER_NAME);
-        return;
-    }
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
+    if (buffer->inspect == NULL) {
+        htp_tx_t *tx = (htp_tx_t *)txv;
 
-    const uint32_t buffer_len = bstr_len(h->value);
-    const uint8_t *buffer = bstr_ptr(h->value);
+        if (tx->request_headers == NULL)
+            return NULL;
 
-    if (buffer_len >= mpm_ctx->minlen) {
-        (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
-                &det_ctx->mtcu, &det_ctx->pmq, buffer, buffer_len);
-    }
-}
-#if 0
-static void PrefilterTxHttpRequestTrailers(DetectEngineThreadCtx *det_ctx,
-        const void *pectx,
-        Packet *p, Flow *f, void *txv,
-        const uint64_t idx, const uint8_t flags)
-{
-    SCEnter();
+        htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
+                                                          HEADER_NAME);
+        if (h == NULL || h->value == NULL) {
+            SCLogDebug("HTTP %s header not present in this request",
+                       HEADER_NAME);
+            return NULL;
+        }
 
-    const MpmCtx *mpm_ctx = (MpmCtx *)pectx;
-    htp_tx_t *tx = (htp_tx_t *)txv;
+        const uint32_t data_len = bstr_len(h->value);
+        const uint8_t *data = bstr_ptr(h->value);
 
-    if (tx->request_headers == NULL)
-        return;
-    const HtpTxUserData *htud = (const HtpTxUserData *)htp_tx_get_user_data(tx);
-    /* if the request wasn't flagged as having a trailer, we skip */
-    if (htud && !htud->request_has_trailers)
-        return;
-
-    HtpState *htp_state = f->alstate;
-    uint32_t buffer_len = 0;
-    const uint8_t *buffer = DetectEngineHHDGetBufferForTX(tx, idx,
-                                                    NULL, det_ctx,
-                                                    f, htp_state,
-                                                    flags,
-                                                    &buffer_len);
-
-    if (buffer_len >= mpm_ctx->minlen) {
-        (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
-                &det_ctx->mtcu, &det_ctx->pmq, buffer, buffer_len);
+        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferApplyTransforms(buffer, transforms);
     }
-}
-#endif
-static int PrefilterTxHttpRequestHeaderRegister(DetectEngineCtx *de_ctx,
-        SigGroupHead *sgh, MpmCtx *mpm_ctx)
-{
-    SCEnter();
 
-    int r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxHttpRequestHeader,
-        ALPROTO_HTTP, HTP_REQUEST_HEADERS,
-        mpm_ctx, NULL, KEYWORD_NAME " (request)");
-    return r;
-#if 0
-    if (r != 0)
-        return r;
-    return PrefilterAppendTxEngine(sgh, PrefilterTxHttpRequestTrailers,
-        ALPROTO_HTTP, HTP_REQUEST_TRAILER,
-        mpm_ctx, NULL, "http_header (request)");
-#endif
+    return buffer;
 }
-#endif
-
-#ifdef KEYWORD_TOCLIENT
-/** \brief HTTP Headers Mpm prefilter callback
- *
- *  \param det_ctx detection engine thread ctx
- *  \param p packet to inspect
- *  \param f flow to inspect
- *  \param txv tx to inspect
- *  \param pectx inspection context
- */
-static void PrefilterTxHttpResponseHeader(DetectEngineThreadCtx *det_ctx,
-        const void *pectx,
-        Packet *p, Flow *f, void *txv,
-        const uint64_t idx, const uint8_t flags)
-{
-    SCEnter();
-    const MpmCtx *mpm_ctx = (MpmCtx *)pectx;
-    htp_tx_t *tx = (htp_tx_t *)txv;
 
-    if (tx->response_headers == NULL)
-        return;
-
-    htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
-                                                      HEADER_NAME);
-    if (h == NULL || h->value == NULL) {
-        SCLogDebug("HTTP %s header not present in this request", HEADER_NAME);
-        return;
-    }
-
-    const uint32_t buffer_len = bstr_len(h->value);
-    const uint8_t *buffer = bstr_ptr(h->value);
-
-    if (buffer_len >= mpm_ctx->minlen) {
-        (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
-                &det_ctx->mtcu, &det_ctx->pmq, buffer, buffer_len);
-    }
-}
-#if 0
-static void PrefilterTxHttpResponseTrailers(DetectEngineThreadCtx *det_ctx,
-        const void *pectx,
-        Packet *p, Flow *f, void *txv,
-        const uint64_t idx, const uint8_t flags)
-{
-    SCEnter();
-
-    const MpmCtx *mpm_ctx = (MpmCtx *)pectx;
-    htp_tx_t *tx = (htp_tx_t *)txv;
-
-    if (tx->response_headers == NULL)
-        return;
-    const HtpTxUserData *htud = (const HtpTxUserData *)htp_tx_get_user_data(tx);
-    /* if the request wasn't flagged as having a trailer, we skip */
-    if (htud && !htud->response_has_trailers)
-        return;
-
-    HtpState *htp_state = f->alstate;
-    uint32_t buffer_len = 0;
-    const uint8_t *buffer = DetectEngineHHDGetBufferForTX(tx, idx,
-                                                    NULL, det_ctx,
-                                                    f, htp_state,
-                                                    flags,
-                                                    &buffer_len);
-
-    if (buffer_len >= mpm_ctx->minlen) {
-        (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
-                &det_ctx->mtcu, &det_ctx->pmq, buffer, buffer_len);
-    }
-}
 #endif
-static int PrefilterTxHttpResponseHeaderRegister(DetectEngineCtx *de_ctx,
-        SigGroupHead *sgh, MpmCtx *mpm_ctx)
+#ifdef KEYWORD_TOCLIENT
+static InspectionBuffer *GetResponseData(DetectEngineThreadCtx *det_ctx,
+        const DetectEngineTransforms *transforms, Flow *_f,
+        const uint8_t _flow_flags, void *txv, const int list_id)
 {
     SCEnter();
 
-    int r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxHttpResponseHeader,
-        ALPROTO_HTTP, HTP_RESPONSE_HEADERS,
-        mpm_ctx, NULL, KEYWORD_NAME " (response)");
-    return r;
-#if 0
-    if (r != 0)
-        return r;
-    return PrefilterAppendTxEngine(sgh, PrefilterTxHttpRequestTrailers,
-        ALPROTO_HTTP, HTP_REQUEST_TRAILER,
-        mpm_ctx, NULL, "http_header (request)");
-#endif
-}
-#endif
-
-#ifdef KEYWORD_TOSERVER
-static int InspectEngineHttpRequestHeader(ThreadVars *tv,
-        DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
-        const Signature *s, const SigMatchData *smd,
-        Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
-{
-    htp_tx_t *tx = (htp_tx_t *)txv;
-    if (tx->request_headers == NULL)
-        goto end;
-
-    htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
-                                                      HEADER_NAME);
-    if (h == NULL || h->value == NULL) {
-        SCLogDebug("HTTP UA header not present in this request");
-        goto end;
-    }
-
-    const uint32_t buffer_len = bstr_len(h->value);
-    uint8_t *buffer = bstr_ptr(h->value);
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
+    if (buffer->inspect == NULL) {
+        htp_tx_t *tx = (htp_tx_t *)txv;
 
-    if (buffer_len == 0)
-        goto end;
+        if (tx->response_headers == NULL)
+            return NULL;
 
-    det_ctx->buffer_offset = 0;
-    det_ctx->discontinue_matching = 0;
-    det_ctx->inspection_recursion_counter = 0;
-    int r = DetectEngineContentInspection(de_ctx, det_ctx, s, smd,
-                                          f,
-                                          buffer, buffer_len,
-                                          0, DETECT_CI_FLAGS_SINGLE,
-                                          DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE, NULL);
-    if (r == 1)
-        return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
+                                                          HEADER_NAME);
+        if (h == NULL || h->value == NULL) {
+            SCLogDebug("HTTP %s header not present in this request",
+                       HEADER_NAME);
+            return NULL;
+        }
 
- end:
-    if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_REQUEST_HEADERS)
-        return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
-    else
-        return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
-}
-#endif
-#ifdef KEYWORD_TOCLIENT
-static int InspectEngineHttpResponseHeader(ThreadVars *tv,
-        DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
-        const Signature *s, const SigMatchData *smd,
-        Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
-{
-    htp_tx_t *tx = (htp_tx_t *)txv;
-    if (tx->response_headers == NULL)
-        goto end;
+        const uint32_t data_len = bstr_len(h->value);
+        const uint8_t *data = bstr_ptr(h->value);
 
-    htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
-                                                      HEADER_NAME);
-    if (h == NULL || h->value == NULL) {
-        SCLogDebug("HTTP header not present in this request");
-        goto end;
+        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferApplyTransforms(buffer, transforms);
     }
 
-    const uint32_t buffer_len = bstr_len(h->value);
-    uint8_t *buffer = bstr_ptr(h->value);
-
-    if (buffer_len == 0)
-        goto end;
-
-    det_ctx->buffer_offset = 0;
-    det_ctx->discontinue_matching = 0;
-    det_ctx->inspection_recursion_counter = 0;
-    int r = DetectEngineContentInspection(de_ctx, det_ctx, s, smd,
-                                          f,
-                                          buffer, buffer_len,
-                                          0, DETECT_CI_FLAGS_SINGLE,
-                                          DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE, NULL);
-    if (r == 1)
-        return DETECT_ENGINE_INSPECT_SIG_MATCH;
-
- end:
-    if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_HEADERS)
-        return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
-    else
-        return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+    return buffer;
 }
 #endif
 
 /**
- * \brief The setup function for the http_header keyword for a signature.
+ * \brief this function setup the http.header keyword used in the rule
  *
- * \param de_ctx Pointer to the detection engine context.
- * \param s      Pointer to signature for the current Signature being parsed
- *               from the rules.
- * \param m      Pointer to the head of the SigMatchs for the current rule
- *               being parsed.
- * \param arg    Pointer to the string holding the keyword value.
+ * \param de_ctx   Pointer to the Detection Engine Context
+ * \param s        Pointer to the Signature to which the current keyword belongs
+ * \param str      Should hold an empty string always
  *
- * \retval  0 On success.
- * \retval -1 On failure.
+ * \retval 0       On success
  */
-static int DetectHttpHeadersSetup(DetectEngineCtx *de_ctx, Signature *s, const char *arg)
+static int DetectHttpHeadersSetupSticky(DetectEngineCtx *de_ctx, Signature *s, const char *str)
 {
-    s->init_data->list = g_buffer_id;
+    if (DetectBufferSetActiveList(s, g_buffer_id) < 0)
+        return -1;
+
+    if (DetectSignatureSetAppProto(s, ALPROTO_HTTP) < 0)
+        return -1;
+
     return 0;
 }
 
 static void DetectHttpHeadersRegisterStub(void)
 {
     sigmatch_table[KEYWORD_ID].name = KEYWORD_NAME;
+#ifdef KEYWORD_NAME_LEGACY
+    sigmatch_table[KEYWORD_ID].alias = KEYWORD_NAME_LEGACY;
+#endif
     sigmatch_table[KEYWORD_ID].desc = KEYWORD_NAME " sticky buffer for the " BUFFER_DESC;
     sigmatch_table[KEYWORD_ID].url = DOC_URL DOC_VERSION "/rules/" KEYWORD_DOC;
-    sigmatch_table[KEYWORD_ID].Setup = DetectHttpHeadersSetup;
-    sigmatch_table[KEYWORD_ID].flags |= SIGMATCH_NOOPT;
+    sigmatch_table[KEYWORD_ID].Setup = DetectHttpHeadersSetupSticky;
+    sigmatch_table[KEYWORD_ID].flags |= SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER;
+
 #ifdef KEYWORD_TOSERVER
-    DetectAppLayerMpmRegister(BUFFER_NAME, SIG_FLAG_TOSERVER, 2,
-            PrefilterTxHttpRequestHeaderRegister);
+    DetectAppLayerMpmRegister2(BUFFER_NAME, SIG_FLAG_TOSERVER, 2,
+            PrefilterGenericMpmRegister, GetRequestData,
+            ALPROTO_HTTP, HTP_REQUEST_HEADERS);
 #endif
 #ifdef KEYWORD_TOCLIENT
-    DetectAppLayerMpmRegister(BUFFER_NAME, SIG_FLAG_TOCLIENT, 2,
-            PrefilterTxHttpResponseHeaderRegister);
+    DetectAppLayerMpmRegister2(BUFFER_NAME, SIG_FLAG_TOCLIENT, 2,
+            PrefilterGenericMpmRegister, GetResponseData,
+            ALPROTO_HTTP, HTP_RESPONSE_HEADERS);
 #endif
 #ifdef KEYWORD_TOSERVER
-    DetectAppLayerInspectEngineRegister(BUFFER_NAME,
+    DetectAppLayerInspectEngineRegister2(BUFFER_NAME,
             ALPROTO_HTTP, SIG_FLAG_TOSERVER, HTP_REQUEST_HEADERS,
-            InspectEngineHttpRequestHeader);
+            DetectEngineInspectBufferGeneric, GetRequestData);
 #endif
 #ifdef KEYWORD_TOCLIENT
-    DetectAppLayerInspectEngineRegister(BUFFER_NAME,
+    DetectAppLayerInspectEngineRegister2(BUFFER_NAME,
             ALPROTO_HTTP, SIG_FLAG_TOCLIENT, HTP_RESPONSE_HEADERS,
-            InspectEngineHttpResponseHeader);
+            DetectEngineInspectBufferGeneric, GetResponseData);
 #endif
 
     DetectBufferTypeSetDescriptionByName(BUFFER_NAME, BUFFER_DESC);
diff --git a/src/detect-http-referer.c b/src/detect-http-referer.c
index 7c9132b45a..5ade2071a3 100644
--- a/src/detect-http-referer.c
+++ b/src/detect-http-referer.c
@@ -30,7 +30,8 @@
  * Implements support http_referer sticky buffer
  */
 
-#define KEYWORD_NAME "http_referer"
+#define KEYWORD_NAME_LEGACY "http_referer"
+#define KEYWORD_NAME "http.referer"
 #define KEYWORD_DOC "http-keywords.html#http-referer"
 #define BUFFER_NAME "http_referer"
 #define BUFFER_DESC "http referer header"