http: strip 'proxy' part of http_uri

Strip the 'proxy' parts from the normalized uri as inspected by http_uri,
urilen, pcre /U and others.

  In a request line like:
    GET http://suricata-ids.org/blah/ HTTP/1.1
  the normalized URI will now be:
    /blah/

This doesn't affect http_raw_uri. So matching the hostname, etc is still
possible through this keyword.

Additionally, a new per HTTP 'personality' option was added to change
this behavior: "uri-include-all":

  uri-include-all: <true|false>
    Include all parts of the URI. By default the
    'scheme', username/password, hostname and port
    are excluded. Setting this option to true adds
    all of them to the normalized uri as inspected
    by http_uri, urilen, pcre with /U and the other
    keywords that inspect the normalized uri.
    Note that this does not affect http_raw_uri.

So adding uri-include-all:true to all personalities in the yaml will
restore the old default behavior.

Ticket 1008.

diff --git a/src/app-layer-htp-libhtp.c b/src/app-layer-htp-libhtp.c
index 120a61e0d6be40a84dd97e1f6cf4d97512536314..41ab1c8b8842d2655fe3a83b8ccc03f81bdeda8a 100644 (file)
--- a/src/app-layer-htp-libhtp.c
+++ b/src/app-layer-htp-libhtp.c
@@ -102,8 +102,11 @@ int64_t SC_htp_parse_content_length(bstr *b)
  *        Keep an eye out on the tx->parsed_uri struct and how the parameters
  *        in it are generated, just in case some modifications are made to
  *        them in the future.
+ *
+ * \param uri_include_all boolean to indicate if scheme, username/password,
+                          hostname and port should be part of the buffer
  */
-bstr *SCHTPGenerateNormalizedUri(htp_tx_t *tx, htp_uri_t *uri)
+bstr *SCHTPGenerateNormalizedUri(htp_tx_t *tx, htp_uri_t *uri, int uri_include_all)
 {
     if (uri == NULL)
         return NULL;
@@ -111,32 +114,34 @@ bstr *SCHTPGenerateNormalizedUri(htp_tx_t *tx, htp_uri_t *uri)
     // On the first pass determine the length of the final string
     size_t len = 0;
 
-    if (uri->scheme != NULL) {
-        len += bstr_len(uri->scheme);
-        len += 3; // "://"
-    }
-
-    if ((uri->username != NULL) || (uri->password != NULL)) {
-        if (uri->username != NULL) {
-            len += bstr_len(uri->username);
+    if (uri_include_all) {
+        if (uri->scheme != NULL) {
+            len += bstr_len(uri->scheme);
+            len += 3; // "://"
         }
 
-        len += 1; // ":"
+        if ((uri->username != NULL) || (uri->password != NULL)) {
+            if (uri->username != NULL) {
+                len += bstr_len(uri->username);
+            }
 
-        if (uri->password != NULL) {
-            len += bstr_len(uri->password);
-        }
+            len += 1; // ":"
 
-        len += 1; // "@"
-    }
+            if (uri->password != NULL) {
+                len += bstr_len(uri->password);
+            }
 
-    if (uri->hostname != NULL) {
-        len += bstr_len(uri->hostname);
-    }
+            len += 1; // "@"
+        }
 
-    if (uri->port != NULL) {
-        len += 1; // ":"
-        len += bstr_len(uri->port);
+        if (uri->hostname != NULL) {
+            len += bstr_len(uri->hostname);
+        }
+
+        if (uri->port != NULL) {
+            len += 1; // ":"
+            len += bstr_len(uri->port);
+        }
     }
 
     if (uri->path != NULL) {
@@ -159,32 +164,34 @@ bstr *SCHTPGenerateNormalizedUri(htp_tx_t *tx, htp_uri_t *uri)
         return NULL;
     }
 
-    if (uri->scheme != NULL) {
-        bstr_add_noex(r, uri->scheme);
-        bstr_add_c_noex(r, "://");
-    }
-
-    if ((uri->username != NULL) || (uri->password != NULL)) {
-        if (uri->username != NULL) {
-            bstr_add_noex(r, uri->username);
+    if (uri_include_all) {
+        if (uri->scheme != NULL) {
+            bstr_add_noex(r, uri->scheme);
+            bstr_add_c_noex(r, "://");
         }
 
-        bstr_add_c(r, ":");
+        if ((uri->username != NULL) || (uri->password != NULL)) {
+            if (uri->username != NULL) {
+                bstr_add_noex(r, uri->username);
+            }
 
-        if (uri->password != NULL) {
-            bstr_add_noex(r, uri->password);
-        }
+            bstr_add_c(r, ":");
 
-        bstr_add_c_noex(r, "@");
-    }
+            if (uri->password != NULL) {
+                bstr_add_noex(r, uri->password);
+            }
 
-    if (uri->hostname != NULL) {
-        bstr_add_noex(r, uri->hostname);
-    }
+            bstr_add_c_noex(r, "@");
+        }
+
+        if (uri->hostname != NULL) {
+            bstr_add_noex(r, uri->hostname);
+        }
 
-    if (uri->port != NULL) {
-        bstr_add_c(r, ":");
-        bstr_add_noex(r, uri->port);
+        if (uri->port != NULL) {
+            bstr_add_c(r, ":");
+            bstr_add_noex(r, uri->port);
+        }
     }
 
     if (uri->path != NULL) {

diff --git a/src/app-layer-htp-libhtp.h b/src/app-layer-htp-libhtp.h
index 5008f3211f7a9d4ca9cd719695eece88619bb45d..4c4eb3cd9bb9dfed5908f6861128e4ce137f616d 100644 (file)
--- a/src/app-layer-htp-libhtp.h
+++ b/src/app-layer-htp-libhtp.h
@@ -45,7 +45,7 @@
 #include "suricata.h"
 #include "suricata-common.h"
 
-bstr *SCHTPGenerateNormalizedUri(htp_tx_t *tx, htp_uri_t *uri);
+bstr *SCHTPGenerateNormalizedUri(htp_tx_t *tx, htp_uri_t *uri, int uri_include_all);
 int64_t SC_htp_parse_content_length(bstr *b);
 
 #endif /* __APP_LAYER_HTP_LIBHTP__H__ */

diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
index 1fe43e08f208f6ce810dfa18f6f096e9dc8ccf4a..1b8ded5b22df2e5bd669169f1d6b2121b523adbf 100644 (file)
--- a/src/app-layer-htp.c
+++ b/src/app-layer-htp.c
@@ -1996,8 +1996,10 @@ static int HTPCallbackRequestLine(htp_tx_t *tx)
 {
     HtpTxUserData *tx_ud;
     bstr *request_uri_normalized;
+    HtpState *hstate = htp_connp_get_user_data(tx->connp);
+    HTPCfgRec *cfg = hstate->cfg;
 
-    request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri);
+    request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri, cfg->uri_include_all);
     if (request_uri_normalized == NULL)
         return HTP_OK;
 
@@ -2011,7 +2013,6 @@ static int HTPCallbackRequestLine(htp_tx_t *tx)
     htp_tx_set_user_data(tx, tx_ud);
 
     if (tx->flags) {
-        HtpState *hstate = htp_connp_get_user_data(tx->connp);
         HTPErrorCheckTxRequestFlags(hstate, tx);
     }
     return HTP_OK;
@@ -2104,6 +2105,7 @@ static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
  */
 static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
 {
+    cfg_prec->uri_include_all = FALSE;
     cfg_prec->request_body_limit = HTP_CONFIG_DEFAULT_REQUEST_BODY_LIMIT;
     cfg_prec->response_body_limit = HTP_CONFIG_DEFAULT_RESPONSE_BODY_LIMIT;
     cfg_prec->request_inspect_min_size = HTP_CONFIG_DEFAULT_REQUEST_INSPECT_MIN_SIZE;
@@ -2320,6 +2322,10 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
             htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
                                                 HTP_DECODER_URL_PATH,
                                                 ConfValIsTrue(p->val));
+        } else if (strcasecmp("uri-include-all", p->name) == 0) {
+            cfg_prec->uri_include_all = ConfValIsTrue(p->val);
+            SCLogDebug("uri-include-all %s",
+                    cfg_prec->uri_include_all ? "enabled" : "disabled");
         } else if (strcasecmp("query-plusspace-decode", p->name) == 0) {
             htp_config_set_plusspace_decode(cfg_prec->cfg,
                                                 HTP_DECODER_URLENCODED,
@@ -2386,6 +2392,8 @@ void HTPConfigure(void)
         HTPCfgRec *htprec = cfglist.next = SCMalloc(sizeof(HTPCfgRec));
         if (NULL == htprec)
             exit(EXIT_FAILURE);
+        memset(htprec, 0x00, sizeof(*htprec));
+
         cfglist.next->next = nextrec;
         cfglist.next->cfg = htp_config_create();
         if (NULL == cfglist.next->cfg) {
@@ -5019,6 +5027,219 @@ end:
     return result;
 }
 
+/** \test Test 'proxy' URI normalization. Ticket 1008
+ */
+static int HTPParserDecodingTest08(void)
+{
+    int result = 0;
+    Flow *f = NULL;
+    uint8_t httpbuf1[] =
+        "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
+    uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
+    TcpSession ssn;
+
+    HtpState *htp_state =  NULL;
+    int r = 0;
+    char input[] = "\
+%YAML 1.1\n\
+---\n\
+libhtp:\n\
+\n\
+  default-config:\n\
+    personality: IDS\n\
+";
+
+    ConfCreateContextBackup();
+    ConfInit();
+    HtpConfigCreateBackup();
+    ConfYamlLoadString(input, strlen(input));
+    HTPConfigure();
+    char *addr = "4.3.2.1";
+    memset(&ssn, 0, sizeof(ssn));
+
+    f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
+    if (f == NULL)
+        goto end;
+    f->protoctx = &ssn;
+
+    StreamTcpInitConfig(TRUE);
+
+    uint32_t u;
+    for (u = 0; u < httplen1; u++) {
+        uint8_t flags = 0;
+
+        if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
+        else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
+        else flags = STREAM_TOSERVER;
+
+        SCMutexLock(&f->m);
+        r = AppLayerParse(NULL, f, ALPROTO_HTTP, flags, &httpbuf1[u], 1);
+        if (r != 0) {
+            printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
+                    " 0: ", u, r);
+            result = 0;
+            SCMutexUnlock(&f->m);
+            goto end;
+        }
+        SCMutexUnlock(&f->m);
+    }
+
+    htp_state = f->alstate;
+    if (htp_state == NULL) {
+        printf("no http state: ");
+        result = 0;
+        goto end;
+    }
+
+    uint8_t ref1[] = "/blah/";
+    size_t reflen = sizeof(ref1) - 1;
+
+    htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
+    if (tx == NULL)
+        goto end;
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
+            printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
+            goto end;
+        }
+
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
+        {
+            printf("normalized uri \"");
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
+            printf("\" != \"");
+            PrintRawUriFp(stdout, ref1, reflen);
+            printf("\": ");
+            goto end;
+        }
+    }
+
+    result = 1;
+
+end:
+    HTPFreeConfig();
+    ConfDeInit();
+    ConfRestoreContextBackup();
+    HtpConfigRestoreBackup();
+
+    StreamTcpFreeConfig(TRUE);
+    if (htp_state != NULL)
+        HTPStateFree(htp_state);
+    UTHFreeFlow(f);
+    return result;
+}
+
+/** \test Test 'proxy' URI normalization. Ticket 1008
+ */
+static int HTPParserDecodingTest09(void)
+{
+    int result = 0;
+    Flow *f = NULL;
+    uint8_t httpbuf1[] =
+        "GET http://suricata-ids.org/blah/ HTTP/1.1\r\nHost: suricata-ids.org\r\n\r\n";
+    uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
+    TcpSession ssn;
+
+    HtpState *htp_state =  NULL;
+    int r = 0;
+    char input[] = "\
+%YAML 1.1\n\
+---\n\
+libhtp:\n\
+\n\
+  default-config:\n\
+    personality: IDS\n\
+    uri-include-all: true\n\
+";
+
+    ConfCreateContextBackup();
+    ConfInit();
+    HtpConfigCreateBackup();
+    ConfYamlLoadString(input, strlen(input));
+    HTPConfigure();
+    char *addr = "4.3.2.1";
+    memset(&ssn, 0, sizeof(ssn));
+
+    f = UTHBuildFlow(AF_INET, "1.2.3.4", addr, 1024, 80);
+    if (f == NULL)
+        goto end;
+    f->protoctx = &ssn;
+
+    StreamTcpInitConfig(TRUE);
+
+    uint32_t u;
+    for (u = 0; u < httplen1; u++) {
+        uint8_t flags = 0;
+
+        if (u == 0) flags = STREAM_TOSERVER|STREAM_START;
+        else if (u == (httplen1 - 1)) flags = STREAM_TOSERVER|STREAM_EOF;
+        else flags = STREAM_TOSERVER;
+
+        SCMutexLock(&f->m);
+        r = AppLayerParse(NULL, f, ALPROTO_HTTP, flags, &httpbuf1[u], 1);
+        if (r != 0) {
+            printf("toserver chunk %" PRIu32 " returned %" PRId32 ", expected"
+                    " 0: ", u, r);
+            result = 0;
+            SCMutexUnlock(&f->m);
+            goto end;
+        }
+        SCMutexUnlock(&f->m);
+    }
+
+    htp_state = f->alstate;
+    if (htp_state == NULL) {
+        printf("no http state: ");
+        result = 0;
+        goto end;
+    }
+
+    uint8_t ref1[] = "http://suricata-ids.org/blah/";
+    size_t reflen = sizeof(ref1) - 1;
+
+    htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
+    if (tx == NULL)
+        goto end;
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
+            printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
+            goto end;
+        }
+
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
+        {
+            printf("normalized uri \"");
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
+            printf("\" != \"");
+            PrintRawUriFp(stdout, ref1, reflen);
+            printf("\": ");
+            goto end;
+        }
+    }
+
+    result = 1;
+
+end:
+    HTPFreeConfig();
+    ConfDeInit();
+    ConfRestoreContextBackup();
+    HtpConfigRestoreBackup();
+
+    StreamTcpFreeConfig(TRUE);
+    if (htp_state != NULL)
+        HTPStateFree(htp_state);
+    UTHFreeFlow(f);
+    return result;
+}
+
 /** \test BG box crash -- chunks are messed up. Observed for real. */
 static int HTPBodyReassemblyTest01(void)
 {
@@ -5329,6 +5550,8 @@ void HTPParserRegisterTests(void) {
     UtRegisterTest("HTPParserDecodingTest05", HTPParserDecodingTest05, 1);
     UtRegisterTest("HTPParserDecodingTest06", HTPParserDecodingTest06, 1);
     UtRegisterTest("HTPParserDecodingTest07", HTPParserDecodingTest07, 1);
+    UtRegisterTest("HTPParserDecodingTest08", HTPParserDecodingTest08, 1);
+    UtRegisterTest("HTPParserDecodingTest09", HTPParserDecodingTest09, 1);
 
     UtRegisterTest("HTPBodyReassemblyTest01", HTPBodyReassemblyTest01, 1);
 

diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h
index 7785bf01c98297cd7e004ebec3acc804d90a7d25..befbd379122069e6e933dd6592d86f0eaa8e8ed8 100644 (file)
--- a/src/app-layer-htp.h
+++ b/src/app-layer-htp.h
@@ -134,6 +134,8 @@ typedef struct HTPCfgRec_ {
     htp_cfg_t           *cfg;
     struct HTPCfgRec_   *next;
 
+    int                 uri_include_all; /**< use all info in uri (bool) */
+
     /** max size of the client body we inspect */
     uint32_t            request_body_limit;
     uint32_t            response_body_limit;

diff --git a/suricata.yaml.in b/suricata.yaml.in
index c9b3a0e8b510cfa0bdbf0e46a9055c4d72db9cb0..33bbc4bc3ecb15f9b41e34ba2b7b3107ecf08a49 100644 (file)
--- a/suricata.yaml.in
+++ b/suricata.yaml.in
@@ -1057,6 +1057,16 @@ app-layer:
       #   double-decode-path:     Double decode path section of the URI
       #   double-decode-query:    Double decode query section of the URI
       #
+      #   uri-include-all:        Include all parts of the URI. By default the
+      #                           'scheme', username/password, hostname and port
+      #                           are excluded. Setting this option to true adds
+      #                           all of them to the normalized uri as inspected
+      #                           by http_uri, urilen, pcre with /U and the other
+      #                           keywords that inspect the normalized uri.
+      #                           Note that this does not affect http_raw_uri.
+      #                           Also, note that including all was the default in
+      #                           1.4 and 2.0beta1.
+      #
       # Currently Available Personalities:
       #   Minimal
       #   Generic