Suricata upgrade to libhtp 0.5.x.

Remove the support for now unsupported personalities from libhtp -
TOMCAT_6_0, APACHE and APACHE_2_2.  We instead use the APACHE_2
personality.

diff --git a/configure.ac b/configure.ac
index 8077a785c754760ca8c93d7466d3aa27293eb33f..2bf93b2db60d80650dc0bf808bea56f4ab5d5f6b 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -1050,17 +1050,10 @@ AC_INIT(configure.ac)
             echo
             exit 1
         fi
-        PKG_CHECK_MODULES(LIBHTPMINVERSION, htp >= 0.2.3,[libhtp_minver_found="yes"],[libhtp_minver_found="no"])
+        PKG_CHECK_MODULES(LIBHTPMINVERSION, htp >= 0.5.X,[libhtp_minver_found="yes"],[libhtp_minver_found="no"])
         if test "$libhtp_minver_found" = "no"; then
             echo
-            echo "   ERROR! libhtp was found but is not the minimum version required >=0.2.3"
-            echo
-            exit 1
-        fi
-        PKG_CHECK_MODULES(LIBHTPMAXVERSION, htp <= 0.3.0,[libhtp_maxver_found="yes"],[libhtp_maxver_found="no"])
-        if test "$libhtp_maxver_found" = "no"; then
-            echo
-            echo "   ERROR! libhtp 0.3.x was found but only 0.2.x is supported"
+            echo "   ERROR! libhtp was found but is not the minimum version required >= 0.5.X"
             echo
             exit 1
         fi

diff --git a/src/Makefile.am b/src/Makefile.am
index ddf8d3740cceab3d27dd1beeb79873c7bb38db79..f0d16a00430d8a6d19be7ecd801040f52bddf799 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -24,6 +24,7 @@ app-layer-ftp.c app-layer-ftp.h \
 app-layer-htp-body.c app-layer-htp-body.h \
 app-layer-htp.c app-layer-htp.h \
 app-layer-htp-file.c app-layer-htp-file.h \
+app-layer-htp-libhtp.c app-layer-htp-libhtp.h \
 app-layer-parser.c app-layer-parser.h \
 app-layer-protos.c app-layer-protos.h \
 app-layer-smb2.c app-layer-smb2.h \

diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c
index 7e528323a88f3d366033190eb129f108a4137986..77b1b5526f03cc44b53d67a5136baa87df5073d1 100644 (file)
--- a/src/app-layer-htp-file.c
+++ b/src/app-layer-htp-file.c
@@ -348,9 +348,9 @@ static int HTPFileParserTest01(void) {
         goto end;
     }
 
-    if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0)
+    if (tx->request_method == NULL || memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0)
     {
-        printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method));
+        printf("expected method POST, got %s \n", bstr_util_strdup_to_c(tx->request_method));
         goto end;
     }
 
@@ -445,9 +445,9 @@ static int HTPFileParserTest02(void) {
         goto end;
     }
 
-    if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0)
+    if (tx->request_method == NULL || memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0)
     {
-        printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method));
+        printf("expected method POST, got %s \n", bstr_util_strdup_to_c(tx->request_method));
         goto end;
     }
 
@@ -568,9 +568,9 @@ static int HTPFileParserTest03(void) {
         goto end;
     }
 
-    if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0)
+    if (tx->request_method == NULL || memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0)
     {
-        printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method));
+        printf("expected method POST, got %s \n", bstr_util_strdup_to_c(tx->request_method));
         goto end;
     }
 
@@ -696,9 +696,9 @@ static int HTPFileParserTest04(void) {
         goto end;
     }
 
-    if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0)
+    if (tx->request_method == NULL || memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0)
     {
-        printf("expected method POST, got %s: ", bstr_tocstr(tx->request_method));
+        printf("expected method POST, got %s: ", bstr_util_strdup_to_c(tx->request_method));
         goto end;
     }
 
@@ -778,9 +778,9 @@ static int HTPFileParserTest05(void) {
         goto end;
     }
 
-    if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0)
+    if (tx->request_method == NULL || memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0)
     {
-        printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method));
+        printf("expected method POST, got %s \n", bstr_util_strdup_to_c(tx->request_method));
         goto end;
     }
 
@@ -892,9 +892,9 @@ static int HTPFileParserTest06(void) {
         goto end;
     }
 
-    if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0)
+    if (tx->request_method == NULL || memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0)
     {
-        printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method));
+        printf("expected method POST, got %s \n", bstr_util_strdup_to_c(tx->request_method));
         goto end;
     }
 
@@ -995,9 +995,9 @@ static int HTPFileParserTest07(void) {
         goto end;
     }
 
-    if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0)
+    if (tx->request_method == NULL || memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0)
     {
-        printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method));
+        printf("expected method POST, got %s \n", bstr_util_strdup_to_c(tx->request_method));
         goto end;
     }
 
@@ -1399,9 +1399,9 @@ static int HTPFileParserTest11(void) {
         goto end;
     }
 
-    if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0)
+    if (tx->request_method == NULL || memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0)
     {
-        printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method));
+        printf("expected method POST, got %s \n", bstr_util_strdup_to_c(tx->request_method));
         goto end;
     }
 

diff --git a/src/app-layer-htp-libhtp.c b/src/app-layer-htp-libhtp.c
new file mode 100644 (file)
index 0000000..120a61e
--- /dev/null
+++ b/src/app-layer-htp-libhtp.c
@@ -0,0 +1,209 @@
+/*
+ * We are using this file to hold APIs copied from libhtp 0.5.x.
+ */
+
+/***************************************************************************
+ * Copyright (c) 2009-2010 Open Information Security Foundation
+ * Copyright (c) 2010-2013 Qualys, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * - Neither the name of the Qualys, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ ***************************************************************************/
+
+/**
+ * Anoop Saldanha <anoopsaldanha@gmail.com>
+ */
+
+#include "suricata.h"
+#include "suricata-common.h"
+
+
+/**
+ * \brief A direct flick off libhtp-0.5.x htp_is_lws().
+ */
+static int SC_htp_is_lws(int c)
+{
+    if ((c == ' ') || (c == '\t')) return 1;
+    else return 0;
+}
+
+/**
+ * \brief A direct flick off libhtp-0.5.x htp_parse_positive_integer_whitespace().
+ */
+static int64_t SC_htp_parse_positive_integer_whitespace(unsigned char *data, size_t len, int base)
+{
+    if (len == 0) return -1003;
+
+    size_t last_pos;
+    size_t pos = 0;
+
+    // Ignore LWS before
+    while ((pos < len) && (SC_htp_is_lws(data[pos]))) pos++;
+    if (pos == len) return -1001;
+
+    int64_t r = bstr_util_mem_to_pint(data + pos, len - pos, base, &last_pos);
+    if (r < 0) return r;
+
+    // Move after the last digit
+    pos += last_pos;
+
+    // Ignore LWS after
+    while (pos < len) {
+        if (!SC_htp_is_lws(data[pos])) {
+            return -1002;
+        }
+
+        pos++;
+    }
+
+    return r;
+}
+
+/**
+ * \brief A direct flick off libhtp-0.5.x htp_parse_content_length()
+ */
+int64_t SC_htp_parse_content_length(bstr *b)
+{
+    return SC_htp_parse_positive_integer_whitespace((unsigned char *) bstr_ptr(b), bstr_len(b), 10);
+}
+
+/**
+ * \brief Generates the normalized uri.
+ *
+ *        Libhtp doesn't recreate the whole normalized uri and save it.
+ *        That duty has now been passed to us.  A lot of this code has been
+ *        copied from libhtp.
+ *
+ *        Keep an eye out on the tx->parsed_uri struct and how the parameters
+ *        in it are generated, just in case some modifications are made to
+ *        them in the future.
+ */
+bstr *SCHTPGenerateNormalizedUri(htp_tx_t *tx, htp_uri_t *uri)
+{
+    if (uri == NULL)
+        return NULL;
+
+    // On the first pass determine the length of the final string
+    size_t len = 0;
+
+    if (uri->scheme != NULL) {
+        len += bstr_len(uri->scheme);
+        len += 3; // "://"
+    }
+
+    if ((uri->username != NULL) || (uri->password != NULL)) {
+        if (uri->username != NULL) {
+            len += bstr_len(uri->username);
+        }
+
+        len += 1; // ":"
+
+        if (uri->password != NULL) {
+            len += bstr_len(uri->password);
+        }
+
+        len += 1; // "@"
+    }
+
+    if (uri->hostname != NULL) {
+        len += bstr_len(uri->hostname);
+    }
+
+    if (uri->port != NULL) {
+        len += 1; // ":"
+        len += bstr_len(uri->port);
+    }
+
+    if (uri->path != NULL) {
+        len += bstr_len(uri->path);
+    }
+
+    if (uri->query != NULL) {
+        len += 1; // "?"
+        len += bstr_len(uri->query);
+    }
+
+    if (uri->fragment != NULL) {
+        len += 1; // "#"
+        len += bstr_len(uri->fragment);
+    }
+
+    // On the second pass construct the string
+    bstr *r = bstr_alloc(len);
+    if (r == NULL) {
+        return NULL;
+    }
+
+    if (uri->scheme != NULL) {
+        bstr_add_noex(r, uri->scheme);
+        bstr_add_c_noex(r, "://");
+    }
+
+    if ((uri->username != NULL) || (uri->password != NULL)) {
+        if (uri->username != NULL) {
+            bstr_add_noex(r, uri->username);
+        }
+
+        bstr_add_c(r, ":");
+
+        if (uri->password != NULL) {
+            bstr_add_noex(r, uri->password);
+        }
+
+        bstr_add_c_noex(r, "@");
+    }
+
+    if (uri->hostname != NULL) {
+        bstr_add_noex(r, uri->hostname);
+    }
+
+    if (uri->port != NULL) {
+        bstr_add_c(r, ":");
+        bstr_add_noex(r, uri->port);
+    }
+
+    if (uri->path != NULL) {
+        bstr_add_noex(r, uri->path);
+    }
+
+    if (uri->query != NULL) {
+        bstr *query = bstr_dup(uri->query);
+        uint64_t flags = 0;
+        htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, query, &flags);
+        bstr_add_c_noex(r, "?");
+        bstr_add_noex(r, query);
+        bstr_free(query);
+    }
+
+    if (uri->fragment != NULL) {
+        bstr_add_c_noex(r, "#");
+        bstr_add_noex(r, uri->fragment);
+    }
+
+    return r;
+}

diff --git a/src/app-layer-htp-libhtp.h b/src/app-layer-htp-libhtp.h
new file mode 100644 (file)
index 0000000..5008f32
--- /dev/null
+++ b/src/app-layer-htp-libhtp.h
@@ -0,0 +1,51 @@
+/*
+ * We are using this file to hold APIs copied from libhtp 0.5.x.
+ */
+
+/***************************************************************************
+ * Copyright (c) 2009-2010 Open Information Security Foundation
+ * Copyright (c) 2010-2013 Qualys, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * - Neither the name of the Qualys, Inc. nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ ***************************************************************************/
+
+/**
+ * Anoop Saldanha <anoopsaldanha@gmail.com>
+ */
+
+#ifndef __APP_LAYER_HTP_LIBHTP__H__
+#define __APP_LAYER_HTP_LIBHTP__H__
+
+#include "suricata.h"
+#include "suricata-common.h"
+
+bstr *SCHTPGenerateNormalizedUri(htp_tx_t *tx, htp_uri_t *uri);
+int64_t SC_htp_parse_content_length(bstr *b);
+
+#endif /* __APP_LAYER_HTP_LIBHTP__H__ */

diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
index 6f5ce72d70395b259b9173820dadbbc082ced6c6..05a7f6c1f4fc2eff38e3cbf34f6d4cd719dbb4e2 100644 (file)
--- a/src/app-layer-htp.c
+++ b/src/app-layer-htp.c
@@ -54,6 +54,7 @@
 #include "app-layer-htp.h"
 #include "app-layer-htp-body.h"
 #include "app-layer-htp-file.h"
+#include "app-layer-htp-libhtp.h"
 
 #include "util-spm.h"
 #include "util-debug.h"
@@ -175,9 +176,7 @@ static const char *HTPLookupPersonalityString(int p)
         CASE_HTP_PERSONALITY_STRING(IIS_6_0);
         CASE_HTP_PERSONALITY_STRING(IIS_7_0);
         CASE_HTP_PERSONALITY_STRING(IIS_7_5);
-        CASE_HTP_PERSONALITY_STRING(TOMCAT_6_0);
-        CASE_HTP_PERSONALITY_STRING(APACHE);
-        CASE_HTP_PERSONALITY_STRING(APACHE_2_2);
+        CASE_HTP_PERSONALITY_STRING(APACHE_2);
     }
 
     return NULL;
@@ -205,9 +204,14 @@ static int HTPLookupPersonality(const char *str)
     IF_HTP_PERSONALITY_NUM(IIS_6_0);
     IF_HTP_PERSONALITY_NUM(IIS_7_0);
     IF_HTP_PERSONALITY_NUM(IIS_7_5);
-    IF_HTP_PERSONALITY_NUM(TOMCAT_6_0);
-    IF_HTP_PERSONALITY_NUM(APACHE);
-    IF_HTP_PERSONALITY_NUM(APACHE_2_2);
+    IF_HTP_PERSONALITY_NUM(APACHE_2);
+    if ((strcasecmp("TOMCAT_6_0", str) == 0) ||
+        (strcasecmp("APACHE", str) == 0) ||
+        (strcasecmp("APACHE_2_2", str) == 0)) {
+        SCLogError(SC_WARN_OPTION_OBSOLETE, "Personality %s no "
+                   "longer supported by libhtp.", str);
+        return -1;
+    }
 
     return -1;
 }
@@ -265,7 +269,7 @@ void HTPStateFree(void *state)
         uint64_t tx_id;
         uint64_t total_txs = HTPStateGetTxCnt(state);
         /* free the list of body chunks */
-        if (s->connp->conn != NULL) {
+        if (s->conn != NULL) {
             for (tx_id = 0; tx_id < total_txs; tx_id++) {
                 htp_tx_t *tx = HTPStateGetTx(s, tx_id);
                 if (tx != NULL) {
@@ -477,16 +481,16 @@ static int HTPHandleErrorGetId(const char *msg) {
  *  \param s state
  */
 static void HTPHandleError(HtpState *s) {
-    if (s == NULL || s->connp == NULL || s->connp->conn == NULL ||
-        s->connp->conn->messages == NULL) {
+    if (s == NULL || s->conn == NULL ||
+        s->conn->messages == NULL) {
         return;
     }
 
-    size_t size = list_size(s->connp->conn->messages);
+    size_t size = htp_list_size(s->conn->messages);
     size_t msg;
 
     for (msg = 0; msg < size; msg++) {
-        htp_log_t *log = list_get(s->connp->conn->messages, msg);
+        htp_log_t *log = htp_list_get(s->conn->messages, msg);
         if (log == NULL)
             continue;
 
@@ -515,16 +519,16 @@ static void HTPHandleError(HtpState *s) {
  *  \param s state
  */
 static void HTPHandleWarning(HtpState *s) {
-    if (s == NULL || s->connp == NULL || s->connp->conn == NULL ||
-        s->connp->conn->messages == NULL) {
+    if (s == NULL || s->conn == NULL ||
+        s->conn->messages == NULL) {
         return;
     }
 
-    size_t size = list_size(s->connp->conn->messages);
+    size_t size = htp_list_size(s->conn->messages);
     size_t msg;
 
     for (msg = 0; msg < size; msg++) {
-        htp_log_t *log = list_get(s->connp->conn->messages, msg);
+        htp_log_t *log = htp_list_get(s->conn->messages, msg);
         if (log == NULL)
             continue;
 
@@ -548,7 +552,7 @@ static void HTPHandleWarning(HtpState *s) {
  *  \param  input_len   Length in bytes of the received data
  *  \param  output      Pointer to the output (not used in this function)
  *
- *  \retval On success returns 1 or on failure returns -1
+ *  \retval On success returns 1 or on failure returns -1.
  */
 static int HTPHandleRequestData(Flow *f, void *htp_state,
                                 AppLayerParserState *pstate,
@@ -569,7 +573,7 @@ static int HTPHandleRequestData(Flow *f, void *htp_state,
      * be used by HTP library.  This is looked up via IP in the radix
      * tree.  Failing that, the default HTP config is used.
      */
-    if (NULL == hstate->connp ) {
+    if (NULL == hstate->conn) {
         HTPCfgRec *htp_cfg_rec = &cfglist;
         htp_cfg_t *htp = cfglist.cfg; /* Default to the global HTP config */
         SCRadixNode *cfgnode = NULL;
@@ -608,6 +612,8 @@ static int HTPHandleRequestData(Flow *f, void *htp_state,
             goto error;
         }
 
+        hstate->conn = htp_connp_get_connection(hstate->connp);
+
         htp_connp_set_user_data(hstate->connp, (void *)hstate);
         hstate->cfg = htp_cfg_rec;
 
@@ -617,14 +623,6 @@ static int HTPHandleRequestData(Flow *f, void *htp_state,
     /* the code block above should make sure connp is never NULL here */
     BUG_ON(hstate->connp == NULL);
 
-    if (hstate->connp->in_status == STREAM_STATE_ERROR) {
-        SCLogError(SC_ERR_ALPARSER, "Inbound parser is in error state, no"
-                " need to feed data to libhtp");
-        SCReturnInt(-1);
-    } else if (hstate->connp->in_status == STREAM_STATE_TUNNEL) {
-        SCReturnInt(0);
-    }
-
     /* Unset the body inspection (the callback should
      * reactivate it if necessary) */
     hstate->flags &=~ HTP_FLAG_NEW_BODY_SET;
@@ -644,7 +642,7 @@ static int HTPHandleRequestData(Flow *f, void *htp_state,
     r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
 
     switch(r) {
-        case STREAM_STATE_ERROR:
+        case HTP_STREAM_ERROR:
             HTPHandleError(hstate);
 
             hstate->flags |= HTP_FLAG_STATE_ERROR;
@@ -652,12 +650,14 @@ static int HTPHandleRequestData(Flow *f, void *htp_state,
             hstate->flags &= ~HTP_FLAG_NEW_BODY_SET;
             ret = -1;
             break;
-        case STREAM_STATE_DATA:
-        case STREAM_STATE_DATA_OTHER:
+        case HTP_STREAM_DATA:
+        case HTP_STREAM_DATA_OTHER:
             HTPHandleWarning(hstate);
 
             hstate->flags |= HTP_FLAG_STATE_DATA;
             break;
+        case HTP_STREAM_TUNNEL:
+            break;
         default:
             HTPHandleWarning(hstate);
             hstate->flags &= ~HTP_FLAG_STATE_DATA;
@@ -667,11 +667,7 @@ static int HTPHandleRequestData(Flow *f, void *htp_state,
     /* if the TCP connection is closed, then close the HTTP connection */
     if ((pstate->flags & APP_LAYER_PARSER_EOF) &&
         !(hstate->flags & HTP_FLAG_STATE_CLOSED_TS)) {
-        hstate->connp->in_status = STREAM_STATE_CLOSED;
-        // Call the parsers one last time, which will allow them
-        // to process the events that depend on stream closure
-        htp_time_t ts = { f->lastts_sec, 0 };
-        htp_connp_req_data(hstate->connp, &ts, NULL, 0);
+        htp_connp_close(hstate->connp, &ts);
         hstate->flags |= HTP_FLAG_STATE_CLOSED_TS;
         SCLogDebug("stream eof encountered, closing htp handle for ts");
     }
@@ -713,14 +709,6 @@ static int HTPHandleResponseData(Flow *f, void *htp_state,
         SCReturnInt(-1);
     }
 
-    if (hstate->connp->out_status == STREAM_STATE_ERROR) {
-        SCLogError(SC_ERR_ALPARSER, "Outbound parser is in error state, no"
-                " need to feed data to libhtp");
-        SCReturnInt(-1);
-    } else if (hstate->connp->out_status == STREAM_STATE_TUNNEL) {
-        SCReturnInt(0);
-    }
-
     /* Unset the body inspection (the callback should
      * reactivate it if necessary) */
     hstate->flags &=~ HTP_FLAG_NEW_BODY_SET;
@@ -728,7 +716,7 @@ static int HTPHandleResponseData(Flow *f, void *htp_state,
     htp_time_t ts = { f->lastts_sec, 0 };
     r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
     switch(r) {
-        case STREAM_STATE_ERROR:
+        case HTP_STREAM_ERROR:
             HTPHandleError(hstate);
 
             hstate->flags = HTP_FLAG_STATE_ERROR;
@@ -736,11 +724,13 @@ static int HTPHandleResponseData(Flow *f, void *htp_state,
             hstate->flags &= ~HTP_FLAG_NEW_BODY_SET;
             ret = -1;
             break;
-        case STREAM_STATE_DATA:
-        case STREAM_STATE_DATA_OTHER:
+        case HTP_STREAM_DATA:
+        case HTP_STREAM_DATA_OTHER:
             HTPHandleWarning(hstate);
             hstate->flags |= HTP_FLAG_STATE_DATA;
             break;
+        case HTP_STREAM_TUNNEL:
+            break;
         default:
             HTPHandleWarning(hstate);
             hstate->flags &= ~HTP_FLAG_STATE_DATA;
@@ -750,11 +740,7 @@ static int HTPHandleResponseData(Flow *f, void *htp_state,
     /* if we the TCP connection is closed, then close the HTTP connection */
     if ((pstate->flags & APP_LAYER_PARSER_EOF) &&
         !(hstate->flags & HTP_FLAG_STATE_CLOSED_TC)) {
-        hstate->connp->out_status = STREAM_STATE_CLOSED;
-        // Call the parsers one last time, which will allow them
-        // to process the events that depend on stream closure
-        htp_time_t ts = { f->lastts_sec, 0 };
-        htp_connp_res_data(hstate->connp, &ts, NULL, 0);
+        htp_connp_close(hstate->connp, &ts);
         hstate->flags |= HTP_FLAG_STATE_CLOSED_TC;
     }
 
@@ -762,83 +748,6 @@ static int HTPHandleResponseData(Flow *f, void *htp_state,
     SCReturnInt(ret);
 }
 
-#ifdef HAVE_HTP_URI_NORMALIZE_HOOK
-/**
- *  \brief Normalize the query part of the URI as if it's part of the URI.
- *
- *  Called twice if double decoding is enabled.
- *
- *  \param c HTP connection pointer
- *
- *  \retval HOOK_OK we won't fail
- *
- *  This functionality requires the uri normalize hook introduced in libhtp
- *  version 0.2.5.
- */
-static int HTPCallbackRequestUriNormalizeQuery(htp_connp_t *c)
-{
-    SCEnter();
-
-    if (c == NULL || c->in_tx == NULL || c->in_tx->parsed_uri == NULL)
-    {
-        SCReturnInt(HOOK_OK);
-    }
-
-    /* uri normalize the query string as well */
-    if (c->in_tx->parsed_uri->query != NULL) {
-#ifdef HAVE_HTP_DECODE_QUERY_INPLACE
-        htp_decode_query_inplace(c->cfg, c->in_tx,
-                c->in_tx->parsed_uri->query);
-#else
-        htp_decode_path_inplace(c->cfg, c->in_tx,
-                c->in_tx->parsed_uri->query);
-#endif /* HAVE_HTP_DECODE_QUERY_INPLACE */
-    }
-    SCReturnInt(HOOK_OK);
-}
-
-/**
- *  \brief Normalize the path part of the URI (again). Used by double decoding
- *         option.
- *
- *  \param c HTP connection pointer
- *
- *  \retval HOOK_OK we won't fail
- *
- *  This functionality requires the uri normalize hook introduced in libhtp
- *  version 0.2.5.
- */
-static int HTPCallbackRequestUriNormalizePath(htp_connp_t *c)
-{
-    SCEnter();
-
-    if (c == NULL || c->in_tx == NULL || c->in_tx->parsed_uri == NULL)
-    {
-        SCReturnInt(HOOK_OK);
-    }
-
-    /* uri normalize the path string  */
-    if (c->in_tx->parsed_uri->path != NULL) {
-        htp_decode_path_inplace(c->cfg, c->in_tx,
-                c->in_tx->parsed_uri->path);
-
-        /* Handle UTF-8 in path */
-        if (c->cfg->path_convert_utf8) {
-            /* Decode Unicode characters into a single-byte stream, using best-fit mapping */
-            htp_utf8_decode_path_inplace(c->cfg, c->in_tx, c->in_tx->parsed_uri->path);
-        } else {
-            /* Only validate path as a UTF-8 stream */
-            htp_utf8_validate_path(c->in_tx, c->in_tx->parsed_uri->path);
-        }
-
-        /* normalize after decoding */
-        htp_normalize_uri_path_inplace(c->in_tx->parsed_uri->path);
-    }
-
-    SCReturnInt(HOOK_OK);
-}
-#endif /* HAVE_HTP_URI_NORMALIZE_HOOK */
-
 /**
  *  \param name /Lowercase/ version of the variable name
  */
@@ -1023,11 +932,11 @@ static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
  *  set the HTP_BOUNDARY_SET in the transaction.
  */
 static int HtpRequestBodySetupMultipart(htp_tx_data_t *d, HtpTxUserData *htud) {
-    htp_header_t *cl = table_getc(d->tx->request_headers, "content-length");
+    htp_header_t *cl = htp_table_get_c(d->tx->request_headers, "content-length");
     if (cl != NULL)
-        htud->request_body.content_len = htp_parse_content_length(cl->value);
+        htud->request_body.content_len = SC_htp_parse_content_length(cl->value);
 
-    htp_header_t *h = (htp_header_t *)table_getc(d->tx->request_headers,
+    htp_header_t *h = (htp_header_t *)htp_table_get_c(d->tx->request_headers,
             "Content-Type");
     if (h != NULL && bstr_len(h->value) > 0) {
         uint8_t *boundary = NULL;
@@ -1206,7 +1115,7 @@ static void HtpRequestBodyReassemble(HtpTxUserData *htud,
         SCLogDebug("chunk %p", cur);
 
         /* skip body chunks entirely before what we parsed already */
-        if (cur->stream_offset + cur->len <= htud->request_body.body_parsed) {
+        if ((uint64_t )cur->stream_offset + cur->len <= htud->request_body.body_parsed) {
             SCLogDebug("skipping chunk");
             continue;
         }
@@ -1663,7 +1572,7 @@ int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
         size_t filename_len = 0;
 
         /* try Content-Disposition header first */
-        htp_header_t *h = (htp_header_t *)table_getc(tx->response_headers,
+        htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->response_headers,
                 "Content-Disposition");
         if (h != NULL && bstr_len(h->value) > 0) {
             /* parse content-disposition */
@@ -1718,14 +1627,17 @@ end:
 /**
  * \brief Function callback to append chunks for Requests
  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
- * \retval int HOOK_OK if all goes well
+ * \retval int HTP_OK if all goes well
  */
 int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 {
     SCEnter();
 
     if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_REQUEST_BODY))
-        SCReturnInt(HOOK_OK);
+        SCReturnInt(HTP_OK);
+
+    if (d->data == NULL || d->len == 0)
+        SCReturnInt(HTP_OK);
 
 #ifdef PRINT
     printf("HTPBODY START: \n");
@@ -1733,79 +1645,82 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
     printf("HTPBODY END: \n");
 #endif
 
-    HtpState *hstate = (HtpState *)d->tx->connp->user_data;
+    HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
     if (hstate == NULL) {
-        SCReturnInt(HOOK_ERROR);
+        SCReturnInt(HTP_ERROR);
     }
 
     SCLogDebug("New request body data available at %p -> %p -> %p, bodylen "
                "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
 
-    HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
-    if (htud == NULL) {
-        htud = SCMalloc(sizeof(HtpTxUserData));
-        if (unlikely(htud == NULL)) {
-            SCReturnInt(HOOK_OK);
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
+    if (tx_ud == NULL) {
+        tx_ud = SCMalloc(sizeof(HtpTxUserData));
+        if (unlikely(tx_ud == NULL)) {
+            SCReturnInt(HTP_OK);
         }
-        memset(htud, 0, sizeof(HtpTxUserData));
-        htud->operation = HTP_BODY_REQUEST;
+        memset(tx_ud, 0, sizeof(HtpTxUserData));
 
-        if (d->tx->request_method_number == M_POST) {
+        /* Set the user data for handling body chunks on this transaction */
+        htp_tx_set_user_data(d->tx, tx_ud);
+    }
+    if (!tx_ud->response_body_init) {
+        tx_ud->response_body_init = 1;
+        tx_ud->operation = HTP_BODY_REQUEST;
+
+        if (d->tx->request_method_number == HTP_M_POST) {
             SCLogDebug("POST");
-            int r = HtpRequestBodySetupMultipart(d, htud);
+            int r = HtpRequestBodySetupMultipart(d, tx_ud);
             if (r == 1) {
-                htud->request_body_type = HTP_BODY_REQUEST_MULTIPART;
+                tx_ud->request_body_type = HTP_BODY_REQUEST_MULTIPART;
             } else if (r == 0) {
-                htud->request_body_type = HTP_BODY_REQUEST_POST;
+                tx_ud->request_body_type = HTP_BODY_REQUEST_POST;
                 SCLogDebug("not multipart");
             }
-        } else if (d->tx->request_method_number == M_PUT) {
-            if (HtpRequestBodySetupPUT(d, htud) == 0) {
-                htud->request_body_type = HTP_BODY_REQUEST_PUT;
+        } else if (d->tx->request_method_number == HTP_M_PUT) {
+            if (HtpRequestBodySetupPUT(d, tx_ud) == 0) {
+                tx_ud->request_body_type = HTP_BODY_REQUEST_PUT;
             }
         }
-
-        /* Set the user data for handling body chunks on this transaction */
-        htp_tx_set_user_data(d->tx, htud);
     }
 
-    SCLogDebug("htud->request_body.content_len_so_far %"PRIu64, htud->request_body.content_len_so_far);
+    SCLogDebug("tx_ud->request_body.content_len_so_far %"PRIu64, tx_ud->request_body.content_len_so_far);
     SCLogDebug("hstate->cfg->request_body_limit %u", hstate->cfg->request_body_limit);
 
     /* within limits, add the body chunk to the state. */
-    if (hstate->cfg->request_body_limit == 0 || htud->request_body.content_len_so_far < hstate->cfg->request_body_limit)
+    if (hstate->cfg->request_body_limit == 0 || tx_ud->request_body.content_len_so_far < hstate->cfg->request_body_limit)
     {
         uint32_t len = (uint32_t)d->len;
 
         if (hstate->cfg->request_body_limit > 0 &&
-                (htud->request_body.content_len_so_far + len) > hstate->cfg->request_body_limit)
+                (tx_ud->request_body.content_len_so_far + len) > hstate->cfg->request_body_limit)
         {
-            len = hstate->cfg->request_body_limit - htud->request_body.content_len_so_far;
+            len = hstate->cfg->request_body_limit - tx_ud->request_body.content_len_so_far;
             BUG_ON(len > (uint32_t)d->len);
         }
         SCLogDebug("len %u", len);
 
-        int r = HtpBodyAppendChunk(htud, &htud->request_body, (uint8_t *)d->data, len);
+        int r = HtpBodyAppendChunk(tx_ud, &tx_ud->request_body, (uint8_t *)d->data, len);
         if (r < 0) {
-            htud->tsflags |= HTP_REQ_BODY_COMPLETE;
+            tx_ud->tsflags |= HTP_REQ_BODY_COMPLETE;
         } else if (hstate->cfg->request_body_limit > 0 &&
-            htud->request_body.content_len_so_far >= hstate->cfg->request_body_limit)
+            tx_ud->request_body.content_len_so_far >= hstate->cfg->request_body_limit)
         {
-            htud->tsflags |= HTP_REQ_BODY_COMPLETE;
-        } else if (htud->request_body.content_len_so_far == htud->request_body.content_len) {
-            htud->tsflags |= HTP_REQ_BODY_COMPLETE;
+            tx_ud->tsflags |= HTP_REQ_BODY_COMPLETE;
+        } else if (tx_ud->request_body.content_len_so_far == tx_ud->request_body.content_len) {
+            tx_ud->tsflags |= HTP_REQ_BODY_COMPLETE;
         }
 
         uint8_t *chunks_buffer = NULL;
         uint32_t chunks_buffer_len = 0;
 
-        if (htud->request_body_type == HTP_BODY_REQUEST_MULTIPART) {
+        if (tx_ud->request_body_type == HTP_BODY_REQUEST_MULTIPART) {
             /* multi-part body handling starts here */
-            if (!(htud->tsflags & HTP_BOUNDARY_SET)) {
+            if (!(tx_ud->tsflags & HTP_BOUNDARY_SET)) {
                 goto end;
             }
 
-            HtpRequestBodyReassemble(htud, &chunks_buffer, &chunks_buffer_len);
+            HtpRequestBodyReassemble(tx_ud, &chunks_buffer, &chunks_buffer_len);
             if (chunks_buffer == NULL) {
                 goto end;
             }
@@ -1815,103 +1730,110 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
             printf("REASSCHUNK END: \n");
 #endif
 
-            HtpRequestBodyHandleMultipart(hstate, htud, chunks_buffer, chunks_buffer_len);
+            HtpRequestBodyHandleMultipart(hstate, tx_ud, chunks_buffer, chunks_buffer_len);
 
             if (chunks_buffer != NULL) {
                 SCFree(chunks_buffer);
             }
-        } else if (htud->request_body_type == HTP_BODY_REQUEST_POST) {
-            HtpRequestBodyHandlePOST(hstate, htud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
-        } else if (htud->request_body_type == HTP_BODY_REQUEST_PUT) {
-            HtpRequestBodyHandlePUT(hstate, htud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
+        } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST) {
+            HtpRequestBodyHandlePOST(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
+        } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_PUT) {
+            HtpRequestBodyHandlePUT(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
         }
 
     }
 
 end:
     /* see if we can get rid of htp body chunks */
-    HtpBodyPrune(&htud->request_body);
+    HtpBodyPrune(&tx_ud->request_body);
 
     /* set the new chunk flag */
     hstate->flags |= HTP_FLAG_NEW_BODY_SET;
 
-    SCReturnInt(HOOK_OK);
+    SCReturnInt(HTP_OK);
 }
 
 /**
  * \brief Function callback to append chunks for Responses
  * \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
- * \retval int HOOK_OK if all goes well
+ * \retval int HTP_OK if all goes well
  */
 int HTPCallbackResponseBodyData(htp_tx_data_t *d)
 {
     SCEnter();
 
     if (!(SC_ATOMIC_GET(htp_config_flags) & HTP_REQUIRE_RESPONSE_BODY))
-        SCReturnInt(HOOK_OK);
+        SCReturnInt(HTP_OK);
 
-    HtpState *hstate = (HtpState *)d->tx->connp->user_data;
+    if (d->data == NULL || d->len == 0)
+        SCReturnInt(HTP_OK);
+
+    HtpState *hstate = htp_connp_get_user_data(d->tx->connp);
     if (hstate == NULL) {
-        SCReturnInt(HOOK_ERROR);
+        SCReturnInt(HTP_ERROR);
     }
 
     SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
                "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
 
-    HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
-    if (htud == NULL) {
-        htud = SCMalloc(sizeof(HtpTxUserData));
-        if (unlikely(htud == NULL)) {
-            SCReturnInt(HOOK_OK);
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
+    if (tx_ud == NULL) {
+        tx_ud = SCMalloc(sizeof(HtpTxUserData));
+        if (unlikely(tx_ud == NULL)) {
+            SCReturnInt(HTP_OK);
         }
-        memset(htud, 0, sizeof(HtpTxUserData));
-        htud->operation = HTP_BODY_RESPONSE;
+        memset(tx_ud, 0, sizeof(HtpTxUserData));
+
+        /* Set the user data for handling body chunks on this transaction */
+        htp_tx_set_user_data(d->tx, tx_ud);
+    }
+    if (!tx_ud->request_body_init) {
+        tx_ud->request_body_init = 1;
+        tx_ud->operation = HTP_BODY_RESPONSE;
 
-        htp_header_t *cl = table_getc(d->tx->response_headers, "content-length");
+        htp_header_t *cl = htp_table_get_c(d->tx->response_headers, "content-length");
         if (cl != NULL)
-            htud->response_body.content_len = htp_parse_content_length(cl->value);
+            tx_ud->response_body.content_len = SC_htp_parse_content_length(cl->value);
 
-        /* Set the user data for handling body chunks on this transaction */
-        htp_tx_set_user_data(d->tx, htud);
     }
 
-    SCLogDebug("htud->response_body.content_len_so_far %"PRIu64, htud->response_body.content_len_so_far);
+    SCLogDebug("tx_ud->response_body.content_len_so_far %"PRIu64, tx_ud->response_body.content_len_so_far);
     SCLogDebug("hstate->cfg->response_body_limit %u", hstate->cfg->response_body_limit);
 
     /* within limits, add the body chunk to the state. */
-    if (hstate->cfg->response_body_limit == 0 || htud->response_body.content_len_so_far < hstate->cfg->response_body_limit)
+    if (hstate->cfg->response_body_limit == 0 || tx_ud->response_body.content_len_so_far < hstate->cfg->response_body_limit)
     {
         uint32_t len = (uint32_t)d->len;
 
         if (hstate->cfg->response_body_limit > 0 &&
-                (htud->response_body.content_len_so_far + len) > hstate->cfg->response_body_limit)
+                (tx_ud->response_body.content_len_so_far + len) > hstate->cfg->response_body_limit)
         {
-            len = hstate->cfg->response_body_limit - htud->response_body.content_len_so_far;
+            len = hstate->cfg->response_body_limit - tx_ud->response_body.content_len_so_far;
             BUG_ON(len > (uint32_t)d->len);
         }
         SCLogDebug("len %u", len);
 
-        int r = HtpBodyAppendChunk(htud, &htud->response_body, (uint8_t *)d->data, len);
+        int r = HtpBodyAppendChunk(tx_ud, &tx_ud->response_body, (uint8_t *)d->data, len);
         if (r < 0) {
-            htud->tcflags |= HTP_RES_BODY_COMPLETE;
+            tx_ud->tcflags |= HTP_RES_BODY_COMPLETE;
         } else if (hstate->cfg->response_body_limit > 0 &&
-            htud->response_body.content_len_so_far >= hstate->cfg->response_body_limit)
+            tx_ud->response_body.content_len_so_far >= hstate->cfg->response_body_limit)
         {
-            htud->tcflags |= HTP_RES_BODY_COMPLETE;
-        } else if (htud->response_body.content_len_so_far == htud->response_body.content_len) {
-            htud->tcflags |= HTP_RES_BODY_COMPLETE;
+            tx_ud->tcflags |= HTP_RES_BODY_COMPLETE;
+        } else if (tx_ud->response_body.content_len_so_far == tx_ud->response_body.content_len) {
+            tx_ud->tcflags |= HTP_RES_BODY_COMPLETE;
         }
 
-        HtpResponseBodyHandle(hstate, htud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
+        HtpResponseBodyHandle(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
     }
 
     /* see if we can get rid of htp body chunks */
-    HtpBodyPrune(&htud->response_body);
+    HtpBodyPrune(&tx_ud->response_body);
 
     /* set the new chunk flag */
     hstate->flags |= HTP_FLAG_NEW_BODY_SET;
 
-    SCReturnInt(HOOK_OK);
+    SCReturnInt(HTP_OK);
 }
 
 /**
@@ -1953,12 +1875,12 @@ void HTPFreeConfig(void)
  *  \param  connp   pointer to the current connection parser which has the htp
  *                  state in it as user data
  */
-static int HTPCallbackRequest(htp_connp_t *connp) {
+static int HTPCallbackRequest(htp_tx_t *tx) {
     SCEnter();
 
-    HtpState *hstate = (HtpState *)connp->user_data;
+    HtpState *hstate = htp_connp_get_user_data(tx->connp);
     if (hstate == NULL) {
-        SCReturnInt(HOOK_ERROR);
+        SCReturnInt(HTP_ERROR);
     }
 
     SCLogDebug("transaction_cnt %"PRIu64", list_size %"PRIu64,
@@ -1966,8 +1888,8 @@ static int HTPCallbackRequest(htp_connp_t *connp) {
 
     SCLogDebug("HTTP request completed");
 
-    if (connp->in_tx != NULL) {
-        HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(connp->in_tx);
+    if (tx != NULL) {
+        HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
         if (htud != NULL) {
             if (htud->tsflags & HTP_FILENAME_SET) {
                 SCLogDebug("closing file that was being stored");
@@ -1980,7 +1902,7 @@ static int HTPCallbackRequest(htp_connp_t *connp) {
     /* request done, do raw reassembly now to inspect state and stream
      * at the same time. */
     AppLayerTriggerRawStreamReassembly(hstate->f);
-    SCReturnInt(HOOK_OK);
+    SCReturnInt(HTP_OK);
 }
 
 /**
@@ -1989,12 +1911,12 @@ static int HTPCallbackRequest(htp_connp_t *connp) {
  *  \param  connp   pointer to the current connection parser which has the htp
  *                  state in it as user data
  */
-static int HTPCallbackResponse(htp_connp_t *connp) {
+static int HTPCallbackResponse(htp_tx_t *tx) {
     SCEnter();
 
-    HtpState *hstate = (HtpState *)connp->user_data;
+    HtpState *hstate = htp_connp_get_user_data(tx->connp);
     if (hstate == NULL) {
-        SCReturnInt(HOOK_ERROR);
+        SCReturnInt(HTP_ERROR);
     }
 
     /* we have one whole transaction now */
@@ -2003,8 +1925,8 @@ static int HTPCallbackResponse(htp_connp_t *connp) {
     /* Unset the body inspection (if any) */
     hstate->flags &=~ HTP_FLAG_NEW_BODY_SET;
 
-    if (connp->out_tx != NULL) {
-        HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(connp->out_tx);
+    if (tx != NULL) {
+        HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
         if (htud != NULL) {
             if (htud->tcflags & HTP_FILENAME_SET) {
                 SCLogDebug("closing file that was being stored");
@@ -2017,10 +1939,110 @@ static int HTPCallbackResponse(htp_connp_t *connp) {
     /* response done, do raw reassembly now to inspect state and stream
      * at the same time. */
     AppLayerTriggerRawStreamReassembly(hstate->f);
-    SCReturnInt(HOOK_OK);
+    SCReturnInt(HTP_OK);
 }
 
-static void HTPConfigSetDefaults(HTPCfgRec *cfg_prec)
+static int HTPCallbackRequestLine(htp_tx_t *tx)
+{
+    HtpTxUserData *tx_ud;
+    bstr *request_uri_normalized;
+
+    request_uri_normalized = SCHTPGenerateNormalizedUri(tx, tx->parsed_uri);
+    if (request_uri_normalized == NULL)
+        return HTP_OK;
+
+    tx_ud = SCMalloc(sizeof(*tx_ud));
+    if (tx_ud == NULL)
+        return HTP_OK;
+    memset(tx_ud, 0, sizeof(*tx_ud));
+    tx_ud->request_uri_normalized = request_uri_normalized;
+    htp_tx_set_user_data(tx, tx_ud);
+
+    return HTP_OK;
+}
+
+static int HTPCallbackDoubleDecodeQuery(htp_tx_t *tx)
+{
+    if (tx->parsed_uri == NULL || tx->parsed_uri->query == NULL)
+        return HTP_OK;
+
+    uint64_t flags = 0;
+    htp_urldecode_inplace(tx->cfg, HTP_DECODER_URLENCODED, tx->parsed_uri->query, &flags);
+
+    return HTP_OK;
+}
+
+static int HTPCallbackDoubleDecodePath(htp_tx_t *tx)
+{
+    if (tx->parsed_uri == NULL || tx->parsed_uri->path == NULL)
+        return HTP_OK;
+
+    uint64_t flags = 0;
+    htp_urldecode_inplace(tx->cfg, HTP_DECODER_URL_PATH, tx->parsed_uri->path, &flags);
+
+    return HTP_OK;
+}
+
+static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
+{
+    if (tx_data->len == 0)
+        return HTP_OK;
+
+    HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
+    if (tx_ud == NULL) {
+        tx_ud = SCMalloc(sizeof(*tx_ud));
+        if (tx_ud == NULL)
+            return HTP_OK;
+        memset(tx_ud, 0, sizeof(*tx_ud));
+        htp_tx_set_user_data(tx_data->tx, tx_ud);
+    }
+    tx_ud->request_headers_raw = SCRealloc(tx_ud->request_headers_raw,
+                                           tx_ud->request_headers_raw_len + tx_data->len);
+    if (tx_ud->request_headers_raw == NULL) {
+        tx_ud->request_headers_raw_len = 0;
+        htp_tx_set_user_data(tx_data->tx, NULL);
+        SCFree(tx_ud);
+        return HTP_OK;
+    }
+    memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
+           tx_data->data, tx_data->len);
+    tx_ud->request_headers_raw_len += tx_data->len;
+
+    return HTP_OK;
+}
+
+static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
+{
+    if (tx_data->len == 0)
+        return HTP_OK;
+
+    HtpTxUserData *tx_ud = htp_tx_get_user_data(tx_data->tx);
+    if (tx_ud == NULL) {
+        tx_ud = SCMalloc(sizeof(*tx_ud));
+        if (tx_ud == NULL)
+            return HTP_OK;
+        memset(tx_ud, 0, sizeof(*tx_ud));
+        htp_tx_set_user_data(tx_data->tx, tx_ud);
+    }
+    tx_ud->response_headers_raw = SCRealloc(tx_ud->response_headers_raw,
+                                           tx_ud->response_headers_raw_len + tx_data->len);
+    if (tx_ud->response_headers_raw == NULL) {
+        tx_ud->response_headers_raw_len = 0;
+        htp_tx_set_user_data(tx_data->tx, NULL);
+        SCFree(tx_ud);
+        return HTP_OK;
+    }
+    memcpy(tx_ud->response_headers_raw + tx_ud->response_headers_raw_len,
+           tx_data->data, tx_data->len);
+    tx_ud->response_headers_raw_len += tx_data->len;
+
+    return HTP_OK;
+}
+
+/*
+ * We have a similar set function called HTPConfigSetDefaultsPhase1.
+ */
+static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
 {
     cfg_prec->request_body_limit = HTP_CONFIG_DEFAULT_REQUEST_BODY_LIMIT;
     cfg_prec->response_body_limit = HTP_CONFIG_DEFAULT_RESPONSE_BODY_LIMIT;
@@ -2028,15 +2050,34 @@ static void HTPConfigSetDefaults(HTPCfgRec *cfg_prec)
     cfg_prec->request_inspect_window = HTP_CONFIG_DEFAULT_REQUEST_INSPECT_WINDOW;
     cfg_prec->response_inspect_min_size = HTP_CONFIG_DEFAULT_RESPONSE_INSPECT_MIN_SIZE;
     cfg_prec->response_inspect_window = HTP_CONFIG_DEFAULT_RESPONSE_INSPECT_WINDOW;
-    htp_config_register_request(cfg_prec->cfg, HTPCallbackRequest);
-    htp_config_register_response(cfg_prec->cfg, HTPCallbackResponse);
-#ifdef HAVE_HTP_URI_NORMALIZE_HOOK
-    htp_config_register_request_uri_normalize(cfg_prec->cfg, HTPCallbackRequestUriNormalizeQuery);
-#endif
-    htp_config_set_generate_request_uri_normalized(cfg_prec->cfg, 1);
+
+    htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
+    htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
+    htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
+    htp_config_register_response_trailer_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
+
     htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
     htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
 
+    htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequest);
+    htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponse);
+
+    htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
+    htp_config_set_parse_request_auth(cfg_prec->cfg, 0);
+
+    return;
+}
+
+/*
+ * We have this splitup so that in case double decoding has been enabled
+ * for query and path, they would be called first on the callback queue,
+ * before the callback set by Phase2() is called.  We need this, since
+ * the callback in Phase2() generates the normalized uri which utilizes
+ * the query and path. */
+static void HTPConfigSetDefaultsPhase2(HTPCfgRec *cfg_prec)
+{
+    htp_config_register_request_line(cfg_prec->cfg, HTPCallbackRequestLine);
+
     return;
 }
 
@@ -2097,10 +2138,7 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
                 /* The IDS personality by default converts the path (and due to
                  * our query string callback also the query string) to lowercase.
                  * Signatures do not expect this, so override it. */
-                htp_config_set_path_case_insensitive(cfg_prec->cfg, 0);
-#ifdef HAVE_HTP_DECODE_QUERY_INPLACE
-                htp_config_set_query_case_insensitive(cfg_prec->cfg, 0);
-#endif
+                htp_config_set_convert_lowercase(cfg_prec->cfg, HTP_DECODER_URL_PATH, 0);
             } else {
                 SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Unknown personality "
                              "\"%s\", ignoring", p->val);
@@ -2121,6 +2159,7 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
                            "from conf file - %s.  Killing engine", p->val);
                 exit(EXIT_FAILURE);
             }
+
         } else if (strcasecmp("request-body-minimal-inspect-size", p->name) == 0) {
             if (ParseSizeStringU32(p->val, &cfg_prec->request_inspect_min_size) < 0) {
                 SCLogError(SC_ERR_SIZE_PARSE, "Error parsing request-body-minimal-inspect-size "
@@ -2135,6 +2174,18 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
                 exit(EXIT_FAILURE);
             }
 
+        } else if (strcasecmp("double-decode-path", p->name) == 0) {
+            if (ConfValIsTrue(p->val)) {
+                htp_config_register_request_line(cfg_prec->cfg,
+                                                 HTPCallbackDoubleDecodeQuery);
+            }
+
+        } else if (strcasecmp("double-decode-query", p->name) == 0) {
+            if (ConfValIsTrue(p->val)) {
+                htp_config_register_request_line(cfg_prec->cfg,
+                                                 HTPCallbackDoubleDecodePath);
+            }
+
         } else if (strcasecmp("response-body-minimal-inspect-size", p->name) == 0) {
             if (ParseSizeStringU32(p->val, &cfg_prec->response_inspect_min_size) < 0) {
                 SCLogError(SC_ERR_SIZE_PARSE, "Error parsing response-body-minimal-inspect-size "
@@ -2149,140 +2200,63 @@ static void HTPConfigParseParameters(HTPCfgRec *cfg_prec, ConfNode *s,
                 exit(EXIT_FAILURE);
             }
 
-        } else if (strcasecmp("double-decode-path", p->name) == 0) {
-            if (ConfValIsTrue(p->val)) {
-#ifdef HAVE_HTP_URI_NORMALIZE_HOOK
-                htp_config_register_request_uri_normalize(cfg_prec->cfg,
-                                                          HTPCallbackRequestUriNormalizePath);
-#else
-                SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "\"double-decode-path\" "
-                             "option requires at least libhtp version 0.2.5");
-#endif
-            } /* if */
-
-        } else if (strcasecmp("double-decode-query", p->name) == 0) {
-            if (ConfValIsTrue(p->val)) {
-#ifdef HAVE_HTP_URI_NORMALIZE_HOOK
-                htp_config_register_request_uri_normalize(cfg_prec->cfg,
-                                                          HTPCallbackRequestUriNormalizeQuery);
-#else
-                SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "\"double-decode-query\" "
-                             "option requires at least libhtp version 0.2.5");
-#endif
-            } /* if */
-
-        } else if (strcasecmp("path-backslash-separators", p->name) == 0) {
-            if (ConfValIsTrue(p->val))
-                htp_config_set_path_backslash_separators(cfg_prec->cfg, 1);
-            else
-                htp_config_set_path_backslash_separators(cfg_prec->cfg, 0);
-        } else if (strcasecmp("path-compress-separators", p->name) == 0) {
-            if (ConfValIsTrue(p->val))
-                htp_config_set_path_compress_separators(cfg_prec->cfg, 1);
-            else
-                htp_config_set_path_compress_separators(cfg_prec->cfg, 0);
-        } else if (strcasecmp("path-control-char-handling", p->name) == 0) {
-            if (strcasecmp(p->val, "none") == 0) {
-                htp_config_set_path_control_char_handling(cfg_prec->cfg, NONE);
-            } else if (strcasecmp(p->val, "status_400") == 0) {
-                htp_config_set_path_control_char_handling(cfg_prec->cfg,
-                                                          STATUS_400);
+        } else if (strcasecmp("path-convert-backslash-separators", p->name) == 0) {
+            htp_config_set_backslash_convert_slashes(cfg_prec->cfg,
+                                                     HTP_DECODER_URL_PATH,
+                                                     ConfValIsTrue(p->val));
+        } else if (strcasecmp("path-bestfit-replacement-char", p->name) == 0) {
+            if (strlen(p->val) == 1) {
+                htp_config_set_bestfit_replacement_byte(cfg_prec->cfg,
+                                                        HTP_DECODER_URL_PATH,
+                                                        p->val[0]);
             } else {
                 SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
-                           "for libhtp param path-control-char-handling");
+                           "for libhtp param path-bestfit-replacement-char");
             }
-        } else if (strcasecmp("path-convert-utf8", p->name) == 0) {
-            if (ConfValIsTrue(p->val))
-                htp_config_set_path_convert_utf8(cfg_prec->cfg, 1);
-            else
-                htp_config_set_path_convert_utf8(cfg_prec->cfg, 0);
-        } else if (strcasecmp("path-decode-separators", p->name) == 0) {
-            if (ConfValIsTrue(p->val))
-                htp_config_set_path_decode_separators(cfg_prec->cfg, 1);
-            else
-                htp_config_set_path_decode_separators(cfg_prec->cfg, 0);
-        } else if (strcasecmp("path-decode-u-encoding", p->name) == 0) {
-            if (ConfValIsTrue(p->val))
-                htp_config_set_path_decode_u_encoding(cfg_prec->cfg, 1);
-            else
-                htp_config_set_path_decode_u_encoding(cfg_prec->cfg, 0);
-        } else if (strcasecmp("path-invalid-encoding-handling", p->name) == 0) {
+        } else if (strcasecmp("path-convert-lowercase", p->name) == 0) {
+            htp_config_set_convert_lowercase(cfg_prec->cfg,
+                                             HTP_DECODER_URL_PATH,
+                                             ConfValIsTrue(p->val));
+        } else if (strcasecmp("path-nul-encoded-terminates", p->name) == 0) {
+            htp_config_set_nul_encoded_terminates(cfg_prec->cfg,
+                                                  HTP_DECODER_URL_PATH,
+                                                  ConfValIsTrue(p->val));
+        } else if (strcasecmp("path-nul-raw-terminates", p->name) == 0) {
+            htp_config_set_nul_raw_terminates(cfg_prec->cfg,
+                                              HTP_DECODER_URL_PATH,
+                                              ConfValIsTrue(p->val));
+        } else if (strcasecmp("path-separators-compress", p->name) == 0) {
+            htp_config_set_path_separators_compress(cfg_prec->cfg,
+                                                    HTP_DECODER_URL_PATH,
+                                                    ConfValIsTrue(p->val));
+        } else if (strcasecmp("path-separators-decode", p->name) == 0) {
+            htp_config_set_path_separators_decode(cfg_prec->cfg,
+                                                  HTP_DECODER_URL_PATH,
+                                                  ConfValIsTrue(p->val));
+        } else if (strcasecmp("path-u-encoding-decode", p->name) == 0) {
+            htp_config_set_u_encoding_decode(cfg_prec->cfg,
+                                             HTP_DECODER_URL_PATH,
+                                             ConfValIsTrue(p->val));
+        } else if (strcasecmp("path-url-encoding-invalid-handling", p->name) == 0) {
+            enum htp_url_encoding_handling_t handling;
             if (strcasecmp(p->val, "preserve_percent") == 0) {
-                htp_config_set_path_invalid_encoding_handling(cfg_prec->cfg,
-                                                              URL_DECODER_PRESERVE_PERCENT);
+                handling = HTP_URL_DECODE_PRESERVE_PERCENT;
             } else if (strcasecmp(p->val, "remove_percent") == 0) {
-                htp_config_set_path_invalid_encoding_handling(cfg_prec->cfg,
-                                                              URL_DECODER_REMOVE_PERCENT);
+                handling = HTP_URL_DECODE_REMOVE_PERCENT;
             } else if (strcasecmp(p->val, "decode_invalid") == 0) {
-                htp_config_set_path_invalid_encoding_handling(cfg_prec->cfg,
-                                                              URL_DECODER_DECODE_INVALID);
-            } else if (strcasecmp(p->val, "status_400") == 0) {
-                htp_config_set_path_invalid_encoding_handling(cfg_prec->cfg,
-                                                              STATUS_400);
-            } else {
-                SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
-                           "for libhtp param path-invalid-encoding-handling");
-            }
-        } else if (strcasecmp("path-invalid-utf8-handling", p->name) == 0) {
-            if (strcasecmp(p->val, "none") == 0) {
-                htp_config_set_path_invalid_utf8_handling(cfg_prec->cfg, NONE);
-            } else if (strcasecmp(p->val, "status_400") == 0) {
-                htp_config_set_path_invalid_utf8_handling(cfg_prec->cfg,
-                                                          STATUS_400);
-            } else {
-                SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
-                           "for libhtp param path-invalid-utf8-handling");
-            }
-        } else if (strcasecmp("path-nul-encoded-handling", p->name) == 0) {
-            if (strcasecmp(p->val, "terminate") == 0) {
-                htp_config_set_path_nul_encoded_handling(cfg_prec->cfg,
-                                                         TERMINATE);
-            } else if (strcasecmp(p->val, "status_400") == 0) {
-                htp_config_set_path_nul_encoded_handling(cfg_prec->cfg,
-                                                         STATUS_400);
-            } else if (strcasecmp(p->val, "status_404") == 0) {
-                htp_config_set_path_nul_encoded_handling(cfg_prec->cfg,
-                                                         STATUS_404);
-            } else {
-                SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
-                           "for libhtp param path-nul-encoded-handling");
-            }
-        } else if (strcasecmp("path-nul-raw-handling", p->name) == 0) {
-            if (strcasecmp(p->val, "terminate") == 0) {
-                htp_config_set_path_nul_raw_handling(cfg_prec->cfg,
-                                                     TERMINATE);
-            } else if (strcasecmp(p->val, "status_400") == 0) {
-                htp_config_set_path_nul_raw_handling(cfg_prec->cfg,
-                                                     STATUS_400);
-            } else if (strcasecmp(p->val, "status_404") == 0) {
-                htp_config_set_path_nul_raw_handling(cfg_prec->cfg,
-                                                     STATUS_404);
-            } else {
-                SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
-                           "for libhtp param path-nul-raw-handling");
-            }
-        } else if (strcasecmp("path-replacement-char", p->name) == 0) {
-            if (strlen(p->val) == 1) {
-                htp_config_set_path_replacement_char(cfg_prec->cfg,
-                                                     p->val[0]);
+                handling = HTP_URL_DECODE_PROCESS_INVALID;
             } else {
                 SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
-                           "for libhtp param set-path-replacement-char");
-            }
-        } else if (strcasecmp("path-unicode-mapping", p->name) == 0) {
-            if (strcasecmp(p->val, "bestfit") == 0) {
-                htp_config_set_path_unicode_mapping(cfg_prec->cfg,
-                                                    BESTFIT);
-            } else if (strcasecmp(p->val, "status_400") == 0) {
-                htp_config_set_path_unicode_mapping(cfg_prec->cfg,
-                                                    STATUS_400);
-            } else if (strcasecmp(p->val, "status_404") == 0) {
-                htp_config_set_path_unicode_mapping(cfg_prec->cfg,
-                                                    STATUS_404);
-            } else {
-                SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, "Invalid entry "
-                           "for libhtp param set-path-unicode-mapping");
+                           "for libhtp param path-url-encoding-invalid-handling");
+                return;
             }
+            htp_config_set_url_encoding_invalid_handling(cfg_prec->cfg,
+                                                         HTP_DECODER_URL_PATH,
+                                                         handling);
+        } else if (strcasecmp("path-utf8-convert-bestfit", p->name) == 0) {
+            htp_config_set_utf8_convert_bestfit(cfg_prec->cfg,
+                                                HTP_DECODER_URL_PATH,
+                                                ConfValIsTrue(p->val));
         } else {
             SCLogWarning(SC_ERR_UNKNOWN_VALUE, "LIBHTP Ignoring unknown "
                          "default config: %s", p->name);
@@ -2309,9 +2283,10 @@ void HTPConfigure(void)
         exit(EXIT_FAILURE);
     }
     SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
-    HTPConfigSetDefaults(&cfglist);
+    HTPConfigSetDefaultsPhase1(&cfglist);
     HTPConfigParseParameters(&cfglist, ConfGetNode("libhtp.default-config"),
                              cfgtree);
+    HTPConfigSetDefaultsPhase2(&cfglist);
 
     /* Read server config and create a parser for each IP in radix tree */
     ConfNode *server_config = ConfGetNode("libhtp.server-config");
@@ -2342,9 +2317,9 @@ void HTPConfigure(void)
             exit(EXIT_FAILURE);
         }
 
-
-        HTPConfigSetDefaults(htprec);
+        HTPConfigSetDefaultsPhase1(htprec);
         HTPConfigParseParameters(htprec, s, cfgtree);
+        HTPConfigSetDefaultsPhase2(htprec);
     }
 
     SCReturn;
@@ -2387,17 +2362,17 @@ static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
 
 static uint64_t HTPStateGetTxCnt(void *alstate)
 {
-    return (uint64_t)list_size(((HtpState *)alstate)->connp->conn->transactions);
+    return (uint64_t)htp_list_size(((htp_tx_t *)alstate)->conn->transactions);
 }
 
 static void *HTPStateGetTx(void *alstate, uint64_t tx_id)
 {
-    return list_get(((HtpState *)alstate)->connp->conn->transactions, tx_id);
+    return htp_list_get(((htp_tx_t *)alstate)->conn->transactions, tx_id);
 }
 
 static int HTPStateGetAlstateProgressCompletionStatus(uint8_t direction)
 {
-    return (direction == 0) ? TX_PROGRESS_WAIT : TX_PROGRESS_DONE;
+    return (direction == 0) ? HTP_REQUEST_COMPLETE : HTP_RESPONSE_COMPLETE;
 }
 
 static void HTPStateTruncate(void *state, uint8_t flags) {
@@ -2526,20 +2501,16 @@ int HTPParserTest01(void) {
     }
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-
-    htp_header_t *h = NULL;
-    table_iterator_reset(tx->request_headers);
-    table_iterator_next(tx->request_headers, (void **) & h);
-
-    if (strcmp(bstr_tocstr(h->value), "Victor/1.0")
-            || tx->request_method_number != M_POST ||
-            tx->request_protocol_number != HTTP_1_0)
+    htp_header_t *h =  htp_table_get_index(tx->request_headers, 0, NULL);
+    if (strcmp(bstr_util_strdup_to_c(h->value), "Victor/1.0")
+        || tx->request_method_number != HTP_M_POST ||
+        tx->request_protocol_number != HTP_PROTOCOL_1_0)
     {
         printf("expected header value: Victor/1.0 and got %s: and expected"
                 " method: POST and got %s, expected protocol number HTTP/1.0"
-                "  and got: %s \n", bstr_tocstr(h->value),
-                bstr_tocstr(tx->request_method),
-                bstr_tocstr(tx->request_protocol));
+                "  and got: %s \n", bstr_util_strdup_to_c(h->value),
+                bstr_util_strdup_to_c(tx->request_method),
+                bstr_util_strdup_to_c(tx->request_protocol));
         result = 0;
         goto end;
     }
@@ -2586,14 +2557,10 @@ int HTPParserTest02(void) {
     }
 
     htp_tx_t *tx = HTPStateGetTx(http_state, 0);
-
-    htp_header_t *h = NULL;
-    table_iterator_reset(tx->request_headers);
-    table_iterator_next(tx->request_headers, (void **) & h);
-
+    htp_header_t *h =  htp_table_get_index(tx->request_headers, 0, NULL);
     if ((tx->request_method) != NULL || h != NULL)
     {
-        printf("expected method NULL, got %s \n", bstr_tocstr(tx->request_method));
+        printf("expected method NULL, got %s \n", bstr_util_strdup_to_c(tx->request_method));
         result = 0;
         goto end;
     }
@@ -2651,16 +2618,13 @@ int HTPParserTest03(void) {
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
 
-    htp_header_t *h = NULL;
-    table_iterator_reset(tx->request_headers);
-    table_iterator_next(tx->request_headers, (void **) & h);
-
-    if (tx->request_method_number != M_UNKNOWN ||
-             h != NULL || tx->request_protocol_number != HTTP_1_0)
+    htp_header_t *h =  htp_table_get_index(tx->request_headers, 0, NULL);
+    if (tx->request_method_number != HTP_M_UNKNOWN ||
+        h != NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
     {
         printf("expected method M_UNKNOWN and got %s: , expected protocol "
-                "HTTP/1.0 and got %s \n", bstr_tocstr(tx->request_method),
-                bstr_tocstr(tx->request_protocol));
+                "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
+                bstr_util_strdup_to_c(tx->request_protocol));
         result = 0;
         goto end;
     }
@@ -2707,17 +2671,13 @@ int HTPParserTest04(void) {
     }
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-
-    htp_header_t *h = NULL;
-    table_iterator_reset(tx->request_headers);
-    table_iterator_next(tx->request_headers, (void **) & h);
-
-    if (tx->request_method_number != M_UNKNOWN ||
-            h != NULL || tx->request_protocol_number != PROTOCOL_UNKNOWN)
+    htp_header_t *h =  htp_table_get_index(tx->request_headers, 0, NULL);
+    if (tx->request_method_number != HTP_M_UNKNOWN ||
+        h != NULL || tx->request_protocol_number != HTP_PROTOCOL_0_9)
     {
         printf("expected method M_UNKNOWN and got %s: , expected protocol "
-                "NULL and got %s \n", bstr_tocstr(tx->request_method),
-                bstr_tocstr(tx->request_protocol));
+                "NULL and got %s \n", bstr_util_strdup_to_c(tx->request_method),
+                bstr_util_strdup_to_c(tx->request_protocol));
         result = 0;
         goto end;
     }
@@ -2814,17 +2774,13 @@ int HTPParserTest05(void) {
     }
 
     htp_tx_t *tx = HTPStateGetTx(http_state, 0);
-
-    htp_header_t *h = NULL;
-    table_iterator_reset(tx->request_headers);
-    table_iterator_next(tx->request_headers, (void **) & h);
-
-    if (tx->request_method_number != M_POST ||
-            h == NULL || tx->request_protocol_number != HTTP_1_0)
+    htp_header_t *h =  htp_table_get_index(tx->request_headers, 0, NULL);
+    if (tx->request_method_number != HTP_M_POST ||
+        h == NULL || tx->request_protocol_number != HTP_PROTOCOL_1_0)
     {
         printf("expected method M_POST and got %s: , expected protocol "
-                "HTTP/1.0 and got %s \n", bstr_tocstr(tx->request_method),
-                bstr_tocstr(tx->request_protocol));
+                "HTTP/1.0 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
+                bstr_util_strdup_to_c(tx->request_protocol));
         result = 0;
         goto end;
     }
@@ -2832,8 +2788,8 @@ int HTPParserTest05(void) {
     if (tx->response_status_number != 200) {
         printf("expected response 200 OK and got %"PRId32" %s: , expected protocol "
                 "HTTP/1.0 and got %s \n", tx->response_status_number,
-                bstr_tocstr(tx->response_message),
-                bstr_tocstr(tx->response_protocol));
+               bstr_util_strdup_to_c(tx->response_message),
+                bstr_util_strdup_to_c(tx->response_protocol));
         result = 0;
         goto end;
     }
@@ -2928,28 +2884,24 @@ int HTPParserTest06(void) {
     }
 
     htp_tx_t *tx = HTPStateGetTx(http_state, 0);
-
-    htp_header_t *h = NULL;
-    table_iterator_reset(tx->request_headers);
-    table_iterator_next(tx->request_headers, (void **) & h);
-
-    if (tx->request_method_number != M_GET ||
-            h == NULL || tx->request_protocol_number != HTTP_1_1)
+    htp_header_t *h =  htp_table_get_index(tx->request_headers, 0, NULL);
+    if (tx->request_method_number != HTP_M_GET ||
+        h == NULL || tx->request_protocol_number != HTP_PROTOCOL_1_1)
     {
         printf("expected method M_GET and got %s: , expected protocol "
-                "HTTP/1.1 and got %s \n", bstr_tocstr(tx->request_method),
-                bstr_tocstr(tx->request_protocol));
+                "HTTP/1.1 and got %s \n", bstr_util_strdup_to_c(tx->request_method),
+                bstr_util_strdup_to_c(tx->request_protocol));
         result = 0;
         goto end;
     }
 
     if (tx->response_status_number != 200 ||
-            h == NULL || tx->request_protocol_number != HTTP_1_1)
+        h == NULL || tx->request_protocol_number != HTP_PROTOCOL_1_1)
     {
         printf("expected response 200 OK and got %"PRId32" %s: , expected proto"
                 "col HTTP/1.1 and got %s \n", tx->response_status_number,
-                bstr_tocstr(tx->response_message),
-                bstr_tocstr(tx->response_protocol));
+                bstr_util_strdup_to_c(tx->response_message),
+                bstr_util_strdup_to_c(tx->response_protocol));
         result = 0;
         goto end;
     }
@@ -3010,19 +2962,22 @@ int HTPParserTest07(void) {
     size_t reflen = sizeof(ref) - 1;
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (reflen != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
-                (uintmax_t)reflen,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (memcmp(bstr_ptr(tx->request_uri_normalized), ref,
-                    bstr_size(tx->request_uri_normalized)) != 0)
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\" != \"");
             PrintRawUriFp(stdout, ref, reflen);
             printf("\": ");
@@ -3096,10 +3051,13 @@ libhtp:\n\
     }
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        //printf("uri %s\n", bstr_tocstr(tx->request_uri_normalized));
-        PrintRawDataFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized),
-                bstr_len(tx->request_uri_normalized));
+    if (tx == NULL)
+        goto end;
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
+        PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
+                       bstr_len(tx_ud->request_uri_normalized));
     }
 
     result = 1;
@@ -3172,10 +3130,13 @@ libhtp:\n\
     }
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        //printf("uri %s\n", bstr_tocstr(tx->request_uri_normalized));
-        PrintRawDataFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized),
-                bstr_len(tx->request_uri_normalized));
+    if (tx == NULL)
+        goto end;
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        //printf("uri %s\n", bstr_util_strdup_to_c(tx->request_uri_normalized));
+        PrintRawDataFp(stdout, bstr_ptr(tx_ud->request_uri_normalized),
+                       bstr_len(tx_ud->request_uri_normalized));
     }
 
     result = 1;
@@ -3238,15 +3199,12 @@ int HTPParserTest10(void) {
     }
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-    htp_header_t *h = NULL;
-    table_iterator_reset(tx->request_headers);
-    table_iterator_next(tx->request_headers, (void **) & h);
-
+    htp_header_t *h =  htp_table_get_index(tx->request_headers, 0, NULL);
     if (h == NULL) {
         goto end;
     }
 
-    char *name = bstr_tocstr(h->name);
+    char *name = bstr_util_strdup_to_c(h->name);
     if (name == NULL) {
         goto end;
     }
@@ -3258,7 +3216,7 @@ int HTPParserTest10(void) {
     }
     free(name);
 
-    char *value = bstr_tocstr(h->value);
+    char *value = bstr_util_strdup_to_c(h->value);
     if (value == NULL) {
         goto end;
     }
@@ -3325,20 +3283,23 @@ static int HTPParserTest11(void) {
     }
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (4 != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
+    if (tx != NULL && tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (4 != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be 2, is %"PRIuMAX,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (bstr_ptr(tx->request_uri_normalized)[0] != '/' ||
-            bstr_ptr(tx->request_uri_normalized)[1] != '%' ||
-            bstr_ptr(tx->request_uri_normalized)[2] != '0' ||
-            bstr_ptr(tx->request_uri_normalized)[3] != '0')
+        if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
+            bstr_ptr(tx_ud->request_uri_normalized)[1] != '%' ||
+            bstr_ptr(tx_ud->request_uri_normalized)[2] != '0' ||
+            bstr_ptr(tx_ud->request_uri_normalized)[3] != '0')
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\": ");
             goto end;
         }
@@ -3399,23 +3360,26 @@ static int HTPParserTest12(void) {
     }
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (7 != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (7 != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be 5, is %"PRIuMAX,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (bstr_ptr(tx->request_uri_normalized)[0] != '/' ||
-            bstr_ptr(tx->request_uri_normalized)[1] != '?' ||
-            bstr_ptr(tx->request_uri_normalized)[2] != 'a' ||
-            bstr_ptr(tx->request_uri_normalized)[3] != '=' ||
-            bstr_ptr(tx->request_uri_normalized)[4] != '%' ||
-            bstr_ptr(tx->request_uri_normalized)[5] != '0' ||
-            bstr_ptr(tx->request_uri_normalized)[6] != '0')
+        if (bstr_ptr(tx_ud->request_uri_normalized)[0] != '/' ||
+            bstr_ptr(tx_ud->request_uri_normalized)[1] != '?' ||
+            bstr_ptr(tx_ud->request_uri_normalized)[2] != 'a' ||
+            bstr_ptr(tx_ud->request_uri_normalized)[3] != '=' ||
+            bstr_ptr(tx_ud->request_uri_normalized)[4] != '%' ||
+            bstr_ptr(tx_ud->request_uri_normalized)[5] != '0' ||
+            bstr_ptr(tx_ud->request_uri_normalized)[6] != '0')
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\": ");
             goto end;
         }
@@ -3476,15 +3440,12 @@ int HTPParserTest13(void) {
     }
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-    htp_header_t *h = NULL;
-    table_iterator_reset(tx->request_headers);
-    table_iterator_next(tx->request_headers, (void **) & h);
-
+    htp_header_t *h =  htp_table_get_index(tx->request_headers, 0, NULL);
     if (h == NULL) {
         goto end;
     }
 
-    char *name = bstr_tocstr(h->name);
+    char *name = bstr_util_strdup_to_c(h->name);
     if (name == NULL) {
         goto end;
     }
@@ -3496,7 +3457,7 @@ int HTPParserTest13(void) {
     }
     free(name);
 
-    char *value = bstr_tocstr(h->value);
+    char *value = bstr_util_strdup_to_c(h->value);
     if (value == NULL) {
         goto end;
     }
@@ -3889,11 +3850,25 @@ libhtp:\n\
         goto end;
     }
 
-    /* Check that the HTP state config matches the correct one */
-    if (htp_state->connp->cfg != htp) {
+    if (HTPStateGetTxCnt(htp_state) != 2) {
+        printf("HTPStateGetTxCnt(htp_state) failure\n");
+        goto end;
+    }
+
+    htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
+    if (tx == NULL)
+        goto end;
+    if (tx->cfg != htp) {
         printf("wrong HTP config (%p instead of %p - default=%p): ",
-               htp_state->connp->cfg, htp, cfglist.cfg);
-        result = 0;
+               tx->cfg, htp, cfglist.cfg);
+        goto end;
+    }
+    tx = HTPStateGetTx(htp_state, 1);
+    if (tx == NULL)
+        goto end;
+    if (tx->cfg != htp) {
+        printf("wrong HTP config (%p instead of %p - default=%p): ",
+               tx->cfg, htp, cfglist.cfg);
         goto end;
     }
 
@@ -3910,6 +3885,8 @@ end:
     return result;
 }
 
+/* disabled when we upgraded to libhtp 0.5.x */
+#if 0
 int HTPParserConfigTest04(void)
 {
     int result = 0;
@@ -3980,6 +3957,7 @@ end:
 
     return result;
 }
+#endif
 
 /** \test Test %2f decoding in profile Apache_2_2
  *
@@ -4006,7 +3984,7 @@ static int HTPParserDecodingTest01(void)
 libhtp:\n\
 \n\
   default-config:\n\
-    personality: Apache_2_2\n\
+    personality: Apache_2\n\
 ";
 
     ConfCreateContextBackup();
@@ -4052,19 +4030,22 @@ libhtp:\n\
     size_t reflen = sizeof(ref1) - 1;
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (reflen != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
-                (uintmax_t)reflen,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (memcmp(bstr_ptr(tx->request_uri_normalized), ref1,
-                    bstr_size(tx->request_uri_normalized)) != 0)
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\" != \"");
             PrintRawUriFp(stdout, ref1, reflen);
             printf("\": ");
@@ -4072,23 +4053,26 @@ libhtp:\n\
         }
     }
 
-    uint8_t ref2[] = "/abc/def?ghi%2Fjkl";
+    uint8_t ref2[] = "/abc/def?ghi/jkl";
     reflen = sizeof(ref2) - 1;
 
     tx = HTPStateGetTx(htp_state, 1);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (reflen != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
-                (uintmax_t)reflen,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (memcmp(bstr_ptr(tx->request_uri_normalized), ref2,
-                    bstr_size(tx->request_uri_normalized)) != 0)
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\" != \"");
             PrintRawUriFp(stdout, ref2, reflen);
             printf("\": ");
@@ -4096,22 +4080,25 @@ libhtp:\n\
         }
     }
 
-    uint8_t ref3[] = "/abc/def?ghi%2Fjkl";
-    reflen = sizeof(ref2) - 1;
+    uint8_t ref3[] = "/abc/def?ghi%2fjkl";
+    reflen = sizeof(ref3) - 1;
     tx = HTPStateGetTx(htp_state, 2);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (reflen != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
-                (uintmax_t)reflen,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (memcmp(bstr_ptr(tx->request_uri_normalized), ref3,
-                    bstr_size(tx->request_uri_normalized)) != 0)
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\" != \"");
             PrintRawUriFp(stdout, ref3, reflen);
             printf("\": ");
@@ -4207,19 +4194,22 @@ libhtp:\n\
     size_t reflen = sizeof(ref1) - 1;
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (reflen != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    HtpTxUserData *tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
-                (uintmax_t)reflen,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (memcmp(bstr_ptr(tx->request_uri_normalized), ref1,
-                    bstr_size(tx->request_uri_normalized)) != 0)
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\" != \"");
             PrintRawUriFp(stdout, ref1, reflen);
             printf("\": ");
@@ -4231,19 +4221,22 @@ libhtp:\n\
     reflen = sizeof(ref2) - 1;
 
     tx = HTPStateGetTx(htp_state, 1);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (reflen != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
-                (uintmax_t)reflen,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (memcmp(bstr_ptr(tx->request_uri_normalized), ref2,
-                    bstr_size(tx->request_uri_normalized)) != 0)
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\" != \"");
             PrintRawUriFp(stdout, ref2, reflen);
             printf("\": ");
@@ -4251,22 +4244,25 @@ libhtp:\n\
         }
     }
 
-    uint8_t ref3[] = "/abc/def?ghi%2Fjkl";
+    uint8_t ref3[] = "/abc/def?ghi%2fjkl";
     reflen = sizeof(ref3) - 1;
     tx = HTPStateGetTx(htp_state, 2);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (reflen != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX" (3): ",
-                (uintmax_t)reflen,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (memcmp(bstr_ptr(tx->request_uri_normalized), ref3,
-                    bstr_size(tx->request_uri_normalized)) != 0)
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref3,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\" != \"");
             PrintRawUriFp(stdout, ref3, reflen);
             printf("\": ");
@@ -4360,19 +4356,22 @@ libhtp:\n\
     size_t reflen = sizeof(ref1) - 1;
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (reflen != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
-                (uintmax_t)reflen,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (memcmp(bstr_ptr(tx->request_uri_normalized), ref1,
-                    bstr_size(tx->request_uri_normalized)) != 0)
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\" != \"");
             PrintRawUriFp(stdout, ref1, reflen);
             printf("\": ");
@@ -4384,19 +4383,22 @@ libhtp:\n\
     reflen = sizeof(ref2) - 1;
 
     tx = HTPStateGetTx(htp_state, 1);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (reflen != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    tx_ud = (HtpTxUserData *)htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
-                (uintmax_t)reflen,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (memcmp(bstr_ptr(tx->request_uri_normalized), ref2,
-                    bstr_size(tx->request_uri_normalized)) != 0)
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref2,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\" != \"");
             PrintRawUriFp(stdout, ref2, reflen);
             printf("\": ");
@@ -4486,19 +4488,22 @@ libhtp:\n\
     size_t reflen = sizeof(ref1) - 1;
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (reflen != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
-                (uintmax_t)reflen,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (memcmp(bstr_ptr(tx->request_uri_normalized), ref1,
-                    bstr_size(tx->request_uri_normalized)) != 0)
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\" != \"");
             PrintRawUriFp(stdout, ref1, reflen);
             printf("\": ");
@@ -4588,19 +4593,22 @@ libhtp:\n\
     size_t reflen = sizeof(ref1) - 1;
 
     htp_tx_t *tx = HTPStateGetTx(htp_state, 0);
-    if (tx != NULL && tx->request_uri_normalized != NULL) {
-        if (reflen != bstr_size(tx->request_uri_normalized)) {
+    if (tx == NULL)
+        goto end;
+    HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data(tx);
+    if (tx_ud != NULL && tx_ud->request_uri_normalized != NULL) {
+        if (reflen != bstr_len(tx_ud->request_uri_normalized)) {
             printf("normalized uri len should be %"PRIuMAX", is %"PRIuMAX,
-                (uintmax_t)reflen,
-                (uintmax_t)bstr_size(tx->request_uri_normalized));
+                   (uintmax_t)reflen,
+                   bstr_len(tx_ud->request_uri_normalized));
             goto end;
         }
 
-        if (memcmp(bstr_ptr(tx->request_uri_normalized), ref1,
-                    bstr_size(tx->request_uri_normalized)) != 0)
+        if (memcmp(bstr_ptr(tx_ud->request_uri_normalized), ref1,
+                   bstr_len(tx_ud->request_uri_normalized)) != 0)
         {
             printf("normalized uri \"");
-            PrintRawUriFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized), bstr_size(tx->request_uri_normalized));
+            PrintRawUriFp(stdout, bstr_ptr(tx_ud->request_uri_normalized), bstr_len(tx_ud->request_uri_normalized));
             printf("\" != \"");
             PrintRawUriFp(stdout, ref1, reflen);
             printf("\": ");
@@ -4775,7 +4783,9 @@ void HTPParserRegisterTests(void) {
     UtRegisterTest("HTPParserConfigTest01", HTPParserConfigTest01, 1);
     UtRegisterTest("HTPParserConfigTest02", HTPParserConfigTest02, 1);
     UtRegisterTest("HTPParserConfigTest03", HTPParserConfigTest03, 1);
+#if 0 /* disabled when we upgraded to libhtp 0.5.x */
     UtRegisterTest("HTPParserConfigTest04", HTPParserConfigTest04, 1);
+#endif
 
     UtRegisterTest("HTPParserDecodingTest01", HTPParserDecodingTest01, 1);
     UtRegisterTest("HTPParserDecodingTest02", HTPParserDecodingTest02, 1);

diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h
index d9ea8352db63b84eddb1d8df08ea2241e365b7cf..9bfcce96f6b21366a56236fcd5fdb68de63f9dcd 100644 (file)
--- a/src/app-layer-htp.h
+++ b/src/app-layer-htp.h
@@ -190,9 +190,18 @@ typedef struct HtpBody_ {
   * the tx user data */
 typedef struct HtpTxUserData_ {
     /* Body of the request (if any) */
+    uint8_t request_body_init;
+    uint8_t response_body_init;
     HtpBody request_body;
     HtpBody response_body;
 
+    bstr *request_uri_normalized;
+
+    uint8_t *request_headers_raw;
+    uint8_t *response_headers_raw;
+    uint32_t request_headers_raw_len;
+    uint32_t response_headers_raw_len;
+
     /** Holds the boundary identificator string if any (used on
      *  multipart/form-data only)
      */
@@ -211,8 +220,10 @@ typedef struct HtpTxUserData_ {
 
 typedef struct HtpState_ {
 
-    htp_connp_t *connp;     /**< Connection parser structure for
-                                 each connection */
+    /* Connection parser structure for each connection */
+    htp_connp_t *connp;
+    /* Connection structure for each connection */
+    htp_conn_t *conn;
     Flow *f;                /**< Needed to retrieve the original flow when usin HTPLib callbacks */
     uint64_t transaction_cnt;
     uint64_t store_tx_id;

diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c
index 459e771b9796ef73343576abd756d6e7b336aa22..8b3f118b1a7e86192ba9d75e93d1f98f3297e1ff 100644 (file)
--- a/src/detect-engine-hcbd.c
+++ b/src/detect-engine-hcbd.c
@@ -157,12 +157,12 @@ static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
      * when they come */
     if (htud->request_body.content_len == 0) {
         if ((htud->request_body.content_len_so_far > 0) &&
-            tx->progress[0] != TX_PROGRESS_REQ_BODY) {
+            tx->request_progress != HTP_REQUEST_BODY) {
             /* final length of the body */
             htud->tsflags |= HTP_REQ_BODY_COMPLETE;
         }
     } else {
-        if (htud->request_body.content_len == tx->request_entity_len) {
+        if (htud->request_body.content_len == (uint64_t)tx->request_entity_len) {
              SCLogDebug("content_len reached");
              htud->tsflags |= HTP_RES_BODY_COMPLETE;
         }
@@ -284,7 +284,7 @@ int DetectEngineInspectHttpClientBody(ThreadVars *tv,
 
 
  end:
-    if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_REQ_BODY)
+    if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > HTP_REQUEST_BODY)
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     else
         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;

diff --git a/src/detect-engine-hcd.c b/src/detect-engine-hcd.c
index 2dc890670986d37251c6aedfe89b08fcb56f01e1..1666b233c210581887467960b915537f5f6fc671 100644 (file)
--- a/src/detect-engine-hcd.c
+++ b/src/detect-engine-hcd.c
@@ -66,15 +66,15 @@ int DetectEngineRunHttpCookieMpm(DetectEngineThreadCtx *det_ctx, Flow *f,
 
     htp_header_t *h = NULL;
     if (flags & STREAM_TOSERVER) {
-        h = (htp_header_t *)table_getc(tx->request_headers,
-                                       "Cookie");
+        h = (htp_header_t *)htp_table_get_c(tx->request_headers,
+                                            "Cookie");
         if (h == NULL) {
             SCLogDebug("HTTP cookie header not present in this request");
             goto end;
         }
     } else {
-        h = (htp_header_t *)table_getc(tx->response_headers,
-                                       "Set-Cookie");
+        h = (htp_header_t *)htp_table_get_c(tx->response_headers,
+                                            "Set-Cookie");
         if (h == NULL) {
             SCLogDebug("HTTP Set-Cookie header not present in this request");
             goto end;
@@ -111,15 +111,15 @@ int DetectEngineInspectHttpCookie(ThreadVars *tv,
     htp_tx_t *tx = (htp_tx_t *)txv;
     htp_header_t *h = NULL;
     if (flags & STREAM_TOSERVER) {
-        h = (htp_header_t *)table_getc(tx->request_headers,
-                                       "Cookie");
+        h = (htp_header_t *)htp_table_get_c(tx->request_headers,
+                                            "Cookie");
         if (h == NULL) {
             SCLogDebug("HTTP cookie header not present in this request");
             goto end;
         }
     } else {
-        h = (htp_header_t *)table_getc(tx->response_headers,
-                                       "Set-Cookie");
+        h = (htp_header_t *)htp_table_get_c(tx->response_headers,
+                                            "Set-Cookie");
         if (h == NULL) {
             SCLogDebug("HTTP Set-Cookie header not present in this request");
             goto end;
@@ -140,10 +140,10 @@ int DetectEngineInspectHttpCookie(ThreadVars *tv,
 
  end:
     if (flags & STREAM_TOSERVER) {
-        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_REQ_HEADERS)
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > HTP_REQUEST_HEADERS)
             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     } else {
-        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 1) > TX_PROGRESS_RES_HEADERS)
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 1) > HTP_RESPONSE_HEADERS)
             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     }
     return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;

diff --git a/src/detect-engine-hhd.c b/src/detect-engine-hhd.c
index 28c8fd57c2878bc4de413de2bc7ec512a9414ed0..d33c53ceb771b3c444d18e2e6978f4fc54d163ec 100644 (file)
--- a/src/detect-engine-hhd.c
+++ b/src/detect-engine-hhd.c
@@ -120,10 +120,14 @@ static uint8_t *DetectEngineHHDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
         index = (tx_id - det_ctx->hhd_start_tx_id);
     }
 
-    table_t *headers;
+    htp_table_t *headers;
     if (flags & STREAM_TOSERVER) {
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) <= HTP_REQUEST_HEADERS)
+            goto end;
         headers = tx->request_headers;
     } else {
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 1) <= HTP_RESPONSE_HEADERS)
+            goto end;
         headers = tx->response_headers;
     }
     if (headers == NULL)
@@ -132,9 +136,11 @@ static uint8_t *DetectEngineHHDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
     htp_header_t *h = NULL;
     headers_buffer = det_ctx->hhd_buffers[index];
     size_t headers_buffer_len = 0;
+    size_t i = 0;
 
-    table_iterator_reset(headers);
-    while (table_iterator_next(headers, (void **)&h) != NULL) {
+    size_t no_of_headers = htp_table_size(headers);
+    for (; i < no_of_headers; i++) {
+        h = htp_table_get_index(headers, i, NULL);
         size_t size1 = bstr_size(h->name);
         size_t size2 = bstr_size(h->value);
 
@@ -231,10 +237,10 @@ int DetectEngineInspectHttpHeader(ThreadVars *tv,
 
  end:
     if (flags & STREAM_TOSERVER) {
-        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_REQ_HEADERS)
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > HTP_REQUEST_HEADERS)
             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     } else {
-        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 1) > TX_PROGRESS_RES_HEADERS)
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 1) > HTP_RESPONSE_HEADERS)
             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     }
     return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;

diff --git a/src/detect-engine-hhhd.c b/src/detect-engine-hhhd.c
index 9ae7962d8ee123b5af02e9f65f629f2e4204b8ae..2a338ba487e34b013d11934a956b0717848c1e6a 100644 (file)
--- a/src/detect-engine-hhhd.c
+++ b/src/detect-engine-hhhd.c
@@ -63,12 +63,12 @@ int DetectEngineRunHttpHHMpm(DetectEngineThreadCtx *det_ctx, Flow *f,
 {
     uint32_t cnt = 0;
     htp_tx_t *tx = (htp_tx_t *)txv;
-    if (tx->parsed_uri == NULL || tx->parsed_uri->hostname == NULL)
+    if (tx->request_hostname == NULL)
         goto end;
-    uint8_t *hname = (uint8_t *)bstr_ptr(tx->parsed_uri->hostname);
+    uint8_t *hname = (uint8_t *)bstr_ptr(tx->request_hostname);
     if (hname == NULL)
         goto end;
-    uint32_t hname_len = bstr_len(tx->parsed_uri->hostname);
+    uint32_t hname_len = bstr_len(tx->request_hostname);
 
     cnt += HttpHHPatternSearch(det_ctx, hname, hname_len, flags);
 
@@ -97,12 +97,12 @@ int DetectEngineInspectHttpHH(ThreadVars *tv,
                               void *txv, uint64_t tx_id)
 {
     htp_tx_t *tx = (htp_tx_t *)txv;
-    if (tx->parsed_uri == NULL || tx->parsed_uri->hostname == NULL)
+    if (tx->parsed_uri == NULL || tx->request_hostname == NULL)
         goto end;
-    uint8_t *hname = (uint8_t *)bstr_ptr(tx->parsed_uri->hostname);
+    uint8_t *hname = (uint8_t *)bstr_ptr(tx->request_hostname);
     if (hname == NULL)
         goto end;
-    uint32_t hname_len = bstr_len(tx->parsed_uri->hostname);
+    uint32_t hname_len = bstr_len(tx->request_hostname);
 
     det_ctx->buffer_offset = 0;
     det_ctx->discontinue_matching = 0;
@@ -116,7 +116,7 @@ int DetectEngineInspectHttpHH(ThreadVars *tv,
         return DETECT_ENGINE_INSPECT_SIG_MATCH;
 
  end:
-    if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_REQ_HEADERS)
+    if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > HTP_REQUEST_HEADERS)
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     else
         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;

diff --git a/src/detect-engine-hmd.c b/src/detect-engine-hmd.c
index 09c76a42a22c37a6759e7a7d968d54ab29d75cc3..593e429a4f4d64bec2d41127d90810a8039a0667 100644 (file)
--- a/src/detect-engine-hmd.c
+++ b/src/detect-engine-hmd.c
@@ -94,7 +94,7 @@ int DetectEngineInspectHttpMethod(ThreadVars *tv,
 {
     htp_tx_t *tx = (htp_tx_t *)txv;
     if (tx->request_method == NULL) {
-        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_REQ_LINE)
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > HTP_REQUEST_LINE)
             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
         else
             return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;

diff --git a/src/detect-engine-hrhd.c b/src/detect-engine-hrhd.c
index fc8c8b796bbd63a167db77d5907350e12072fcdf..4266fd6f2f53dd0f4f2a4087b50e28a080ededc2 100644 (file)
--- a/src/detect-engine-hrhd.c
+++ b/src/detect-engine-hrhd.c
@@ -63,28 +63,31 @@ int DetectEngineRunHttpRawHeaderMpm(DetectEngineThreadCtx *det_ctx, Flow *f,
     SCEnter();
 
     uint32_t cnt = 0;
-    bstr *raw_headers;
     htp_tx_t *tx = (htp_tx_t *)txv;
+    HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+    if (tx_ud == NULL)
+        SCReturnInt(cnt);
+
     if (flags & STREAM_TOSERVER) {
-        raw_headers = htp_tx_get_request_headers_raw(tx);
-        if (raw_headers != NULL) {
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, txv, 0) <= HTP_REQUEST_HEADERS)
+            SCReturnInt(cnt);
+
+        if (tx_ud->request_headers_raw != NULL) {
             cnt = HttpRawHeaderPatternSearch(det_ctx,
-                                             (uint8_t *)bstr_ptr(raw_headers),
-                                             bstr_len(raw_headers), flags);
-        } else {
-            SCLogDebug("no raw headers");
+                                             tx_ud->request_headers_raw,
+                                             tx_ud->request_headers_raw_len,
+                                             flags);
         }
     } else {
-#ifdef HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW
-        raw_headers = htp_tx_get_response_headers_raw(tx);
-        if (raw_headers != NULL) {
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, txv, 1) <= HTP_RESPONSE_HEADERS)
+            SCReturnInt(cnt);
+
+        if (tx_ud->response_headers_raw != NULL) {
             cnt += HttpRawHeaderPatternSearch(det_ctx,
-                                              (uint8_t *)bstr_ptr(raw_headers),
-                                              bstr_len(raw_headers), flags);
-        } else {
-            SCLogDebug("no raw headers");
+                                              tx_ud->response_headers_raw,
+                                              tx_ud->response_headers_raw_len,
+                                              flags);
         }
-#endif /* HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW */
     }
 
     SCReturnInt(cnt);
@@ -110,17 +113,29 @@ int DetectEngineInspectHttpRawHeader(ThreadVars *tv,
                                      void *alstate,
                                      void *txv, uint64_t tx_id)
 {
-    htp_tx_t *tx = (htp_tx_t *)txv;
-    bstr *raw_headers = NULL;
+    HtpTxUserData *tx_ud = NULL;
+    uint8_t *headers_raw = NULL;
+    uint32_t headers_raw_len = 0;
+
     if (flags & STREAM_TOSERVER) {
-        raw_headers = htp_tx_get_request_headers_raw(tx);
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, txv, 0) <= HTP_REQUEST_HEADERS)
+            return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+    } else {
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, txv, 1) <= HTP_RESPONSE_HEADERS)
+            return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     }
-#ifdef HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW
-    else {
-        raw_headers = htp_tx_get_response_headers_raw(tx);
+
+    tx_ud = htp_tx_get_user_data(txv);
+    if (tx_ud == NULL)
+        goto end;
+    if (flags & STREAM_TOSERVER) {
+        headers_raw = tx_ud->request_headers_raw;
+        headers_raw_len = tx_ud->request_headers_raw_len;
+    } else {
+        headers_raw = tx_ud->response_headers_raw;
+        headers_raw_len = tx_ud->response_headers_raw_len;
     }
-#endif /* HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW */
-    if (raw_headers == NULL)
+    if (headers_raw == NULL)
         goto end;
 
     det_ctx->buffer_offset = 0;
@@ -128,8 +143,8 @@ int DetectEngineInspectHttpRawHeader(ThreadVars *tv,
     det_ctx->inspection_recursion_counter = 0;
     int r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HRHDMATCH],
                                           f,
-                                          (uint8_t *)bstr_ptr(raw_headers),
-                                          bstr_len(raw_headers),
+                                          headers_raw,
+                                          headers_raw_len,
                                           0,
                                           DETECT_ENGINE_CONTENT_INSPECTION_MODE_HRHD, NULL);
     if (r == 1)
@@ -137,10 +152,10 @@ int DetectEngineInspectHttpRawHeader(ThreadVars *tv,
 
  end:
     if (flags & STREAM_TOSERVER) {
-        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_REQ_HEADERS)
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, txv, 0) > HTP_REQUEST_HEADERS)
             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     } else {
-        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 1) > TX_PROGRESS_RES_HEADERS)
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, txv, 1) > HTP_RESPONSE_HEADERS)
             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     }
     return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
@@ -2725,7 +2740,6 @@ end:
 
 static int DetectEngineHttpRawHeaderTest28(void)
 {
-#ifdef HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW
     TcpSession ssn;
     Packet *p1 = NULL;
     Packet *p2 = NULL;
@@ -2841,9 +2855,6 @@ end:
     UTHFreePackets(&p1, 1);
     UTHFreePackets(&p2, 1);
     return result;
-#else
-    return 1;
-#endif
 }
 
 static int DetectEngineHttpRawHeaderTest29(void)

diff --git a/src/detect-engine-hrhhd.c b/src/detect-engine-hrhhd.c
index af5481a256cb528d228a893a7f84c1e142d9d0c4..a2563b6f24103da709d877b7b174923a2f279727 100644 (file)
--- a/src/detect-engine-hrhhd.c
+++ b/src/detect-engine-hrhhd.c
@@ -66,11 +66,11 @@ int DetectEngineRunHttpHRHMpm(DetectEngineThreadCtx *det_ctx, Flow *f,
     uint8_t *hname = NULL;
     uint32_t hname_len = 0;
 
-    if (tx->parsed_uri_incomplete == NULL || tx->parsed_uri_incomplete->hostname == NULL) {
+    if (tx->parsed_uri == NULL || tx->parsed_uri->hostname == NULL) {
         if (tx->request_headers == NULL)
             goto end;
         htp_header_t *h = NULL;
-        h = (htp_header_t *)table_getc(tx->request_headers, "Host");
+        h = (htp_header_t *)htp_table_get_c(tx->request_headers, "Host");
         if (h != NULL) {
             SCLogDebug("HTTP host header not present in this request");
             hname = (uint8_t *)bstr_ptr(h->value);
@@ -79,9 +79,9 @@ int DetectEngineRunHttpHRHMpm(DetectEngineThreadCtx *det_ctx, Flow *f,
             goto end;
         }
     } else {
-        hname = (uint8_t *)bstr_ptr(tx->parsed_uri_incomplete->hostname);
+        hname = (uint8_t *)bstr_ptr(tx->parsed_uri->hostname);
         if (hname != NULL)
-            hname_len = bstr_len(tx->parsed_uri_incomplete->hostname);
+            hname_len = bstr_len(tx->parsed_uri->hostname);
         else
             goto end;
     }
@@ -115,9 +115,9 @@ int DetectEngineInspectHttpHRH(ThreadVars *tv,
     uint8_t *hname;
     uint32_t hname_len;
     htp_tx_t *tx = (htp_tx_t *)txv;
-    if (tx->parsed_uri_incomplete == NULL || tx->parsed_uri_incomplete->hostname == NULL) {
+    if (tx->parsed_uri == NULL || tx->parsed_uri->hostname == NULL) {
         htp_header_t *h = NULL;
-        h = (htp_header_t *)table_getc(tx->request_headers, "Host");
+        h = (htp_header_t *)htp_table_get_c(tx->request_headers, "Host");
         if (h == NULL) {
             SCLogDebug("HTTP host header not present in this request");
             goto end;
@@ -125,10 +125,10 @@ int DetectEngineInspectHttpHRH(ThreadVars *tv,
         hname = (uint8_t *)bstr_ptr(h->value);
         hname_len = bstr_len(h->value);
     } else {
-        hname = (uint8_t *)bstr_ptr(tx->parsed_uri_incomplete->hostname);
+        hname = (uint8_t *)bstr_ptr(tx->parsed_uri->hostname);
         if (hname == NULL)
             goto end;
-        hname_len = bstr_len(tx->parsed_uri_incomplete->hostname);
+        hname_len = bstr_len(tx->parsed_uri->hostname);
     }
 
     det_ctx->buffer_offset = 0;
@@ -143,7 +143,7 @@ int DetectEngineInspectHttpHRH(ThreadVars *tv,
         return DETECT_ENGINE_INSPECT_SIG_MATCH;
 
  end:
-    if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_REQ_HEADERS)
+    if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > HTP_REQUEST_HEADERS)
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     else
         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;

diff --git a/src/detect-engine-hrud.c b/src/detect-engine-hrud.c
index 1a39ca72783a707c42e29ed4931b23a1a5895a56..caf5ab77ec68b9157e8f803c6f2b4bced29d48b6 100644 (file)
--- a/src/detect-engine-hrud.c
+++ b/src/detect-engine-hrud.c
@@ -100,7 +100,7 @@ int DetectEngineInspectHttpRawUri(ThreadVars *tv,
 {
     htp_tx_t *tx = (htp_tx_t *)txv;
     if (tx->request_uri == NULL) {
-        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_REQ_LINE)
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > HTP_REQUEST_LINE)
             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
         else
             return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;

diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c
index 7e69a5e57d1432a9c0c3603b1d0b5f55f8f74f06..e9b823388f53a0dda1627d8277daeeb4b2ac94e0 100644 (file)
--- a/src/detect-engine-hsbd.c
+++ b/src/detect-engine-hsbd.c
@@ -156,12 +156,12 @@ static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
      * when they come */
     if (htud->response_body.content_len == 0) {
         if ((htud->response_body.content_len_so_far > 0) &&
-            tx->progress[1] != TX_PROGRESS_RES_BODY) {
+            tx->response_progress != HTP_RESPONSE_BODY) {
             /* final length of the body */
             htud->tcflags |= HTP_RES_BODY_COMPLETE;
         }
     } else {
-        if (htud->response_body.content_len == tx->response_entity_len) {
+        if (htud->response_body.content_len == (uint64_t)tx->response_entity_len) {
              SCLogDebug("content_len reached");
              htud->tcflags |= HTP_RES_BODY_COMPLETE;
         }
@@ -283,7 +283,7 @@ int DetectEngineInspectHttpServerBody(ThreadVars *tv,
         return DETECT_ENGINE_INSPECT_SIG_MATCH;
 
  end:
-    if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_RES_BODY)
+    if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 1) > HTP_RESPONSE_BODY)
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     else
         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;

diff --git a/src/detect-engine-hscd.c b/src/detect-engine-hscd.c
index 8257ff8ced901781deac99504c3f0767ff25ea35..33cfb3160242c62d369c1ce93460c22a9aa02d6c 100644 (file)
--- a/src/detect-engine-hscd.c
+++ b/src/detect-engine-hscd.c
@@ -98,7 +98,7 @@ int DetectEngineInspectHttpStatCode(ThreadVars *tv,
 {
     htp_tx_t *tx = (htp_tx_t *)txv;
     if (tx->response_status == NULL) {
-        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_RES_LINE)
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > HTP_RESPONSE_LINE)
             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
         else
             return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;

diff --git a/src/detect-engine-hsmd.c b/src/detect-engine-hsmd.c
index 1540cbb2028d9e3b61967afc4b3c0e36997a6ca6..ae641be73243e00f471983397a15418e8fdf5098 100644 (file)
--- a/src/detect-engine-hsmd.c
+++ b/src/detect-engine-hsmd.c
@@ -98,7 +98,7 @@ int DetectEngineInspectHttpStatMsg(ThreadVars *tv,
 {
     htp_tx_t *tx = (htp_tx_t *)txv;
     if (tx->response_message == NULL) {
-        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_RES_LINE)
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > HTP_RESPONSE_LINE)
             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
         else
             return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;

diff --git a/src/detect-engine-hua.c b/src/detect-engine-hua.c
index ef5baf164fffd8d9df799b6c801a823d61b2def5..d2c6ee3c094ec4f551269f55a85112310812b9dd 100644 (file)
--- a/src/detect-engine-hua.c
+++ b/src/detect-engine-hua.c
@@ -65,8 +65,8 @@ int DetectEngineRunHttpUAMpm(DetectEngineThreadCtx *det_ctx, Flow *f,
     if (tx->request_headers == NULL)
         goto end;
 
-    htp_header_t *h = (htp_header_t *)table_getc(tx->request_headers,
-                                                 "User-Agent");
+    htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
+                                                      "User-Agent");
     if (h == NULL) {
         SCLogDebug("HTTP user agent header not present in this request");
         goto end;
@@ -100,8 +100,8 @@ int DetectEngineInspectHttpUA(ThreadVars *tv,
                               void *txv, uint64_t tx_id)
 {
     htp_tx_t *tx = (htp_tx_t *)txv;
-    htp_header_t *h = (htp_header_t *)table_getc(tx->request_headers,
-                                                 "User-Agent");
+    htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
+                                                      "User-Agent");
     if (h == NULL) {
         SCLogDebug("HTTP user agent header not present in this request");
         goto end;
@@ -120,7 +120,7 @@ int DetectEngineInspectHttpUA(ThreadVars *tv,
         return DETECT_ENGINE_INSPECT_SIG_MATCH;
 
  end:
-    if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_REQ_HEADERS)
+    if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > HTP_REQUEST_HEADERS)
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
     else
         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;

diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
index 78dfc280e9a840a1718ab3de7ca173fa989374a2..8fb7231363c1220d97440a67212ea2d800c5e6fb 100644 (file)
--- a/src/detect-engine-mpm.c
+++ b/src/detect-engine-mpm.c
@@ -2842,6 +2842,7 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
         if (s->mpm_sm != NULL) {
             int sm_list = SigMatchListSMBelongsTo(s, s->mpm_sm);
             BUG_ON(sm_list == -1);
+
             DetectContentData *cd = (DetectContentData *)s->mpm_sm->ctx;
             DetectFPAndItsId *dup = (DetectFPAndItsId *)ahb;
             for (; dup != struct_offset; dup++) {
@@ -2873,5 +2874,6 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
     de_ctx->max_fp_id = max_id;
 
     SCFree(ahb);
+
     return 0;
 }

diff --git a/src/detect-engine-state.c b/src/detect-engine-state.c
index f3c630b1dc8447edefecdf68a97ac4f10ee81bef..7941f0d55af4bbfbf6e4c845a1e259ba3a6ae4dc 100644 (file)
--- a/src/detect-engine-state.c
+++ b/src/detect-engine-state.c
@@ -268,12 +268,11 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx,
 
         if (alproto == ALPROTO_HTTP) {
             htp_state = (HtpState *)alstate;
-            if (htp_state->connp == NULL || htp_state->connp->conn == NULL) {
+            if (htp_state->conn == NULL) {
                 FLOWLOCK_UNLOCK(f);
                 goto end;
             }
         }
-
         tx_id = AppLayerTransactionGetInspectId(f, flags);
         SCLogDebug("tx_id %"PRIu64, tx_id);
         total_txs = AppLayerGetTxCnt(alproto, alstate);
@@ -534,7 +533,7 @@ void DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx,
 
                 if (alproto == ALPROTO_HTTP) {
                     htp_state = (HtpState *)alstate;
-                    if (htp_state->connp == NULL || htp_state->connp->conn == NULL) {
+                    if (htp_state->conn == NULL) {
                         FLOWLOCK_UNLOCK(f);
                         RULE_PROFILING_END(det_ctx, s, match);
                         goto end;

diff --git a/src/detect-engine-uri.c b/src/detect-engine-uri.c
index d9cb33aecb8a73d063aafce09fc7353ec6f3161d..1cbd440e07c5553534bbe8c68de7f211e7f58be6 100644 (file)
--- a/src/detect-engine-uri.c
+++ b/src/detect-engine-uri.c
@@ -68,10 +68,10 @@ int DetectEngineInspectPacketUris(ThreadVars *tv,
                                   void *alstate,
                                   void *txv, uint64_t tx_id)
 {
-    htp_tx_t *tx = (htp_tx_t *)txv;
+    HtpTxUserData *tx_ud = htp_tx_get_user_data(txv);
 
-    if (tx->request_uri_normalized == NULL) {
-        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0) > TX_PROGRESS_REQ_LINE)
+    if (tx_ud == NULL || tx_ud->request_uri_normalized == NULL) {
+        if (AppLayerGetAlstateProgress(ALPROTO_HTTP, txv, 0) > HTP_REQUEST_LINE)
             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
         else
             return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
@@ -82,16 +82,16 @@ int DetectEngineInspectPacketUris(ThreadVars *tv,
     det_ctx->inspection_recursion_counter = 0;
 
 #if 0
-    PrintRawDataFp(stdout, (uint8_t *)bstr_ptr(tx->request_uri_normalized),
-                   bstr_len(tx->request_uri_normalized));
+    PrintRawDataFp(stdout, (uint8_t *)bstr_ptr(tx_ud->request_uri_normalized),
+                   bstr_len(tx_ud->request_uri_normalized));
 #endif
 
     /* Inspect all the uricontents fetched on each
      * transaction at the app layer */
     int r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_UMATCH],
                                           f,
-                                          (uint8_t *)bstr_ptr(tx->request_uri_normalized),
-                                          bstr_len(tx->request_uri_normalized),
+                                          bstr_ptr(tx_ud->request_uri_normalized),
+                                          bstr_len(tx_ud->request_uri_normalized),
                                           0,
                                           DETECT_ENGINE_CONTENT_INSPECTION_MODE_URI, NULL);
     if (r == 1) {

diff --git a/src/detect-parse.c b/src/detect-parse.c
index 804a7a74d10438ea47657d686b05eb3dd814584f..7856aa4fe6ae75e63139ca43dbd4513e5d1bb30c 100644 (file)
--- a/src/detect-parse.c
+++ b/src/detect-parse.c
@@ -1178,14 +1178,6 @@ int SigValidate(DetectEngineCtx *de_ctx, Signature *s) {
                     "inspecting response headers.");
             SCReturnInt(0);
         }
-#ifndef HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW
-        if (s->flags & SIG_FLAG_TOCLIENT) {
-            SCLogError(SC_ERR_INVALID_SIGNATURE,"http_raw_header signature with "
-                    "to_client flow direction. See issues #389 and #397. Update "
-                    "libhtp to at least 0.2.7.");
-            SCReturnInt(0);
-        }
-#endif /* HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW */
     }
 
     if (s->sm_lists[DETECT_SM_LIST_HHHDMATCH] != NULL) {

diff --git a/src/detect-uricontent.c b/src/detect-uricontent.c
index fa9315680bb9e41cc2f55349f550dd22e932ef31..2de13509ac443921c88a25ea195379e2881b1f9e 100644 (file)
--- a/src/detect-uricontent.c
+++ b/src/detect-uricontent.c
@@ -250,12 +250,14 @@ uint32_t DetectUricontentInspectMpm(DetectEngineThreadCtx *det_ctx, Flow *f,
     SCEnter();
 
     htp_tx_t *tx = (htp_tx_t *)txv;
+    HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
     uint32_t cnt = 0;
-    if (tx->request_uri_normalized == NULL)
+
+    if (tx_ud == NULL || tx_ud->request_uri_normalized == NULL)
         goto end;
     cnt = DoDetectAppLayerUricontentMatch(det_ctx, (uint8_t *)
-                                          bstr_ptr(tx->request_uri_normalized),
-                                          bstr_len(tx->request_uri_normalized),
+                                          bstr_ptr(tx_ud->request_uri_normalized),
+                                          bstr_len(tx_ud->request_uri_normalized),
                                           flags);
 
 end:
@@ -304,20 +306,20 @@ static int HTTPUriTest01(void) {
 
     htp_tx_t *tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, 0);
 
-    if (tx->request_method_number != M_GET ||
-            tx->request_protocol_number != HTTP_1_1)
+    if (tx->request_method_number != HTP_M_GET ||
+        tx->request_protocol_number != HTP_PROTOCOL_1_1)
     {
         goto end;
     }
 
-    if ((tx->parsed_uri->hostname == NULL) ||
-            (bstr_cmpc(tx->parsed_uri->hostname, "www.example.com") != 0))
+    if ((tx->request_hostname == NULL) ||
+            (bstr_cmp_c(tx->request_hostname, "www.example.com") != 0))
     {
         goto end;
     }
 
     if ((tx->parsed_uri->path == NULL) ||
-            (bstr_cmpc(tx->parsed_uri->path, "/images.gif") != 0))
+            (bstr_cmp_c(tx->parsed_uri->path, "/images.gif") != 0))
     {
         goto end;
     }
@@ -364,20 +366,20 @@ static int HTTPUriTest02(void) {
 
     htp_tx_t *tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, 0);
 
-    if (tx->request_method_number != M_GET ||
-            tx->request_protocol_number != HTTP_1_1)
+    if (tx->request_method_number != HTP_M_GET ||
+        tx->request_protocol_number != HTP_PROTOCOL_1_1)
     {
         goto end;
     }
 
-    if ((tx->parsed_uri->hostname == NULL) ||
-            (bstr_cmpc(tx->parsed_uri->hostname, "www.example.com") != 0))
+    if ((tx->request_hostname == NULL) ||
+            (bstr_cmp_c(tx->request_hostname, "www.example.com") != 0))
     {
         goto end;
     }
 
     if ((tx->parsed_uri->path == NULL) ||
-            (bstr_cmpc(tx->parsed_uri->path, "/images.gif") != 0))
+            (bstr_cmp_c(tx->parsed_uri->path, "/images.gif") != 0))
     {
         goto end;
     }
@@ -426,20 +428,20 @@ static int HTTPUriTest03(void) {
 
     htp_tx_t *tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, 0);
 
-    if (tx->request_method_number != M_UNKNOWN ||
-            tx->request_protocol_number != HTTP_1_1)
+    if (tx->request_method_number != HTP_M_UNKNOWN ||
+        tx->request_protocol_number != HTP_PROTOCOL_1_1)
     {
         goto end;
     }
 
-   if ((tx->parsed_uri->hostname == NULL) ||
-            (bstr_cmpc(tx->parsed_uri->hostname, "www.example.com") != 0))
+    if ((tx->request_hostname == NULL) ||
+            (bstr_cmp_c(tx->request_hostname, "www.example.com") != 0))
     {
         goto end;
     }
 
     if ((tx->parsed_uri->path == NULL) ||
-            (bstr_cmpc(tx->parsed_uri->path, "/images.gif") != 0))
+            (bstr_cmp_c(tx->parsed_uri->path, "/images.gif") != 0))
     {
         goto end;
     }
@@ -489,20 +491,20 @@ static int HTTPUriTest04(void) {
 
     htp_tx_t *tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, 0);
 
-    if (tx->request_method_number != M_GET ||
-            tx->request_protocol_number != HTTP_1_1)
+    if (tx->request_method_number != HTP_M_GET ||
+        tx->request_protocol_number != HTP_PROTOCOL_1_1)
     {
         goto end;
     }
 
-    if ((tx->parsed_uri->hostname == NULL) ||
-            (bstr_cmpc(tx->parsed_uri->hostname, "www.example.com") != 0))
+    if ((tx->request_hostname == NULL) ||
+            (bstr_cmp_c(tx->request_hostname, "www.example.com") != 0))
     {
         goto end;
     }
 
     if ((tx->parsed_uri->path == NULL) ||
-           (bstr_cmpc(tx->parsed_uri->path, "/images.gif") != 0))
+           (bstr_cmp_c(tx->parsed_uri->path, "/images.gif") != 0))
     {
         goto end;
     }

diff --git a/src/detect.c b/src/detect.c
index a00a944bb9bf34bd248433172939d04a1bc2a2f1..c9a1b45d21a22e2d524896abf4ac85f1eb4efec7 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -974,7 +974,7 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx,
                 tx_progress = AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 0);
 
                 if (p->flowflags & FLOW_PKT_TOSERVER) {
-                    if (tx_progress > TX_PROGRESS_REQ_LINE) {
+                    if (tx_progress > HTP_REQUEST_LINE) {
                         if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_URI) {
                             PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_URI);
                             DetectUricontentInspectMpm(det_ctx, p->flow, alstate, flags, tx, idx);
@@ -992,7 +992,7 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx,
                         }
                     }
 
-                    if (tx_progress >= TX_PROGRESS_REQ_HEADERS) {
+                    if (tx_progress >= HTP_REQUEST_HEADERS) {
                         if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_HHHD) {
                             PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_HHHD);
                             DetectEngineRunHttpHHMpm(det_ctx, p->flow, alstate, flags, tx, idx);
@@ -1025,7 +1025,7 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx,
                         }
                     }
 
-                    if (tx_progress >= TX_PROGRESS_REQ_BODY) {
+                    if (tx_progress >= HTP_REQUEST_BODY) {
                         if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_HCBD) {
                             PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_HCBD);
                             DetectEngineRunHttpClientBodyMpm(de_ctx, det_ctx, p->flow, alstate, flags, tx, idx);
@@ -1035,7 +1035,7 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx,
                 } else { /* implied FLOW_PKT_TOCLIENT */
                     tx_progress = AppLayerGetAlstateProgress(ALPROTO_HTTP, tx, 1);
 
-                    if (tx_progress > TX_PROGRESS_RES_LINE) {
+                    if (tx_progress > HTP_RESPONSE_LINE) {
                         if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_HSMD) {
                             PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_HSMD);
                             DetectEngineRunHttpStatMsgMpm(det_ctx, p->flow, alstate, flags, tx, idx);
@@ -1048,7 +1048,7 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx,
                         }
                     }
 
-                    if (tx_progress >= TX_PROGRESS_RES_HEADERS) {
+                    if (tx_progress >= HTP_RESPONSE_HEADERS) {
                         if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_HHD) {
                             PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_HHD);
                             DetectEngineRunHttpHeaderMpm(det_ctx, p->flow, alstate, flags, tx, idx);
@@ -1066,7 +1066,7 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx,
                         }
                     }
 
-                    if (tx_progress >= TX_PROGRESS_RES_BODY) {
+                    if (tx_progress >= HTP_RESPONSE_BODY) {
                         if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_HSBD) {
                             PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_HSBD);
                             DetectEngineRunHttpServerBodyMpm(de_ctx, det_ctx, p->flow, alstate, flags, tx, idx);

diff --git a/src/log-file.c b/src/log-file.c
index 4aaff8c7b112222015a041dec6a8639f86dcb537..8d1c0b1dcbad79b38a67675766a47fa6f78c5166 100644 (file)
--- a/src/log-file.c
+++ b/src/log-file.c
@@ -105,9 +105,13 @@ static void LogFileMetaGetUri(FILE *fp, Packet *p, File *ff) {
     HtpState *htp_state = (HtpState *)p->flow->alstate;
     if (htp_state != NULL) {
         htp_tx_t *tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, ff->txid);
-        if (tx != NULL && tx->request_uri_normalized != NULL) {
-            PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(tx->request_uri_normalized),
-                    bstr_len(tx->request_uri_normalized));
+        if (tx != NULL) {
+            HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+            if (tx_ud->request_uri_normalized != NULL) {
+                PrintRawJsonFp(fp,
+                               bstr_ptr(tx_ud->request_uri_normalized),
+                               bstr_len(tx_ud->request_uri_normalized));
+            }
             return;
         }
     }
@@ -120,18 +124,13 @@ static void LogFileMetaGetHost(FILE *fp, Packet *p, File *ff) {
     if (htp_state != NULL) {
         htp_tx_t *tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, ff->txid);
         if (tx != NULL) {
-            table_t *headers;
-            headers = tx->request_headers;
             htp_header_t *h = NULL;
-
-            table_iterator_reset(headers);
-            while (table_iterator_next(headers, (void **)&h) != NULL) {
-                if (bstr_len(h->name) >= 4 &&
-                        SCMemcmpLowercase((uint8_t *)"host", (uint8_t *)bstr_ptr(h->name), bstr_len(h->name)) == 0) {
-                    PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(h->value),
-                        bstr_len(h->value));
-                    return;
-                }
+            h = (htp_header_t *)htp_table_get_c(tx->request_headers,
+                                                "Host");
+            if (h != NULL) {
+                PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(h->value),
+                               bstr_len(h->value));
+                return;
             }
         }
     }
@@ -144,18 +143,13 @@ static void LogFileMetaGetReferer(FILE *fp, Packet *p, File *ff) {
     if (htp_state != NULL) {
         htp_tx_t *tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, ff->txid);
         if (tx != NULL) {
-            table_t *headers;
-            headers = tx->request_headers;
             htp_header_t *h = NULL;
-
-            table_iterator_reset(headers);
-            while (table_iterator_next(headers, (void **)&h) != NULL) {
-                if (bstr_len(h->name) >= 7 &&
-                        SCMemcmpLowercase((uint8_t *)"referer", (uint8_t *)bstr_ptr(h->name), bstr_len(h->name)) == 0) {
-                    PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(h->value),
-                        bstr_len(h->value));
-                    return;
-                }
+            h = (htp_header_t *)htp_table_get_c(tx->request_headers,
+                                                "Referer");
+            if (h != NULL) {
+                PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(h->value),
+                               bstr_len(h->value));
+                return;
             }
         }
     }
@@ -168,18 +162,13 @@ static void LogFileMetaGetUserAgent(FILE *fp, Packet *p, File *ff) {
     if (htp_state != NULL) {
         htp_tx_t *tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, ff->txid);
         if (tx != NULL) {
-            table_t *headers;
-            headers = tx->request_headers;
             htp_header_t *h = NULL;
-
-            table_iterator_reset(headers);
-            while (table_iterator_next(headers, (void **)&h) != NULL) {
-                if (bstr_len(h->name) >= 10 &&
-                        SCMemcmpLowercase((uint8_t *)"user-agent", (uint8_t *)bstr_ptr(h->name), bstr_len(h->name)) == 0) {
-                    PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(h->value),
-                        bstr_len(h->value));
-                    return;
-                }
+            h = (htp_header_t *)htp_table_get_c(tx->request_headers,
+                                                "User-Agent");
+            if (h != NULL) {
+                PrintRawJsonFp(fp, (uint8_t *)bstr_ptr(h->value),
+                               bstr_len(h->value));
+                return;
             }
         }
     }

diff --git a/src/log-filestore.c b/src/log-filestore.c
index 03fd6772fcbb8c75f8d7eacc46e5ecf7f84bc79b..2d6e3be3cd8f8cba1e86b5e0e850d2d4747f3719 100644 (file)
--- a/src/log-filestore.c
+++ b/src/log-filestore.c
@@ -108,9 +108,12 @@ static void LogFilestoreMetaGetUri(FILE *fp, Packet *p, File *ff) {
     HtpState *htp_state = (HtpState *)p->flow->alstate;
     if (htp_state != NULL) {
         htp_tx_t *tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, ff->txid);
-        if (tx != NULL && tx->request_uri_normalized != NULL) {
-            PrintRawUriFp(fp, (uint8_t *)bstr_ptr(tx->request_uri_normalized),
-                    bstr_len(tx->request_uri_normalized));
+        if (tx != NULL) {
+            HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+            if (tx_ud->request_uri_normalized != NULL) {
+                PrintRawUriFp(fp, bstr_ptr(tx_ud->request_uri_normalized),
+                              bstr_len(tx_ud->request_uri_normalized));
+            }
             return;
         }
     }
@@ -123,18 +126,13 @@ static void LogFilestoreMetaGetHost(FILE *fp, Packet *p, File *ff) {
     if (htp_state != NULL) {
         htp_tx_t *tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, ff->txid);
         if (tx != NULL) {
-            table_t *headers;
-            headers = tx->request_headers;
             htp_header_t *h = NULL;
-
-            table_iterator_reset(headers);
-            while (table_iterator_next(headers, (void **)&h) != NULL) {
-                if (bstr_len(h->name) >= 4 &&
-                        SCMemcmpLowercase((uint8_t *)"host", (uint8_t *)bstr_ptr(h->name), bstr_len(h->name)) == 0) {
-                    PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value),
-                        bstr_len(h->value));
-                    return;
-                }
+            h = (htp_header_t *)htp_table_get_c(tx->request_headers,
+                                                "Host");
+            if (h != NULL) {
+                PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value),
+                              bstr_len(h->value));
+                return;
             }
         }
     }
@@ -147,18 +145,13 @@ static void LogFilestoreMetaGetReferer(FILE *fp, Packet *p, File *ff) {
     if (htp_state != NULL) {
         htp_tx_t *tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, ff->txid);
         if (tx != NULL) {
-            table_t *headers;
-            headers = tx->request_headers;
             htp_header_t *h = NULL;
-
-            table_iterator_reset(headers);
-            while (table_iterator_next(headers, (void **)&h) != NULL) {
-                if (bstr_len(h->name) >= 7 &&
-                        SCMemcmpLowercase((uint8_t *)"referer", (uint8_t *)bstr_ptr(h->name), bstr_len(h->name)) == 0) {
-                    PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value),
-                        bstr_len(h->value));
-                    return;
-                }
+            h = (htp_header_t *)htp_table_get_c(tx->request_headers,
+                                                "Referer");
+            if (h != NULL) {
+                PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value),
+                              bstr_len(h->value));
+                return;
             }
         }
     }
@@ -171,18 +164,13 @@ static void LogFilestoreMetaGetUserAgent(FILE *fp, Packet *p, File *ff) {
     if (htp_state != NULL) {
         htp_tx_t *tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, ff->txid);
         if (tx != NULL) {
-            table_t *headers;
-            headers = tx->request_headers;
             htp_header_t *h = NULL;
-
-            table_iterator_reset(headers);
-            while (table_iterator_next(headers, (void **)&h) != NULL) {
-                if (bstr_len(h->name) >= 10 &&
-                        SCMemcmpLowercase((uint8_t *)"user-agent", (uint8_t *)bstr_ptr(h->name), bstr_len(h->name)) == 0) {
-                    PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value),
-                        bstr_len(h->value));
-                    return;
-                }
+            h = (htp_header_t *)htp_table_get_c(tx->request_headers,
+                                                "User-Agent");
+            if (h != NULL) {
+                PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value),
+                              bstr_len(h->value));
+                return;
             }
         }
     }

diff --git a/src/log-httplog.c b/src/log-httplog.c
index abc581100299fbbaac0f39a882f38009a6c4e1c9..f02252e7b5a57afd1e9ee118fa320779f13f86ef 100644 (file)
--- a/src/log-httplog.c
+++ b/src/log-httplog.c
@@ -254,7 +254,7 @@ static void LogHttpLogCustom(LogHttpLogThread *aft, htp_tx_t *tx, const struct t
             case LOG_HTTP_CF_REQUEST_HEADER:
             /* REQUEST HEADER */
                 if (tx->request_headers != NULL) {
-                    h_request_hdr = table_getc(tx->request_headers, httplog_ctx->cf_nodes[i]->data);
+                    h_request_hdr = htp_table_get_c(tx->request_headers, httplog_ctx->cf_nodes[i]->data);
                 }
                 if (h_request_hdr != NULL) {
                     PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset,
@@ -275,7 +275,7 @@ static void LogHttpLogCustom(LogHttpLogThread *aft, htp_tx_t *tx, const struct t
                             tx->response_status_number > 300 &&
                             tx->response_status_number < 303)
                     {
-                        htp_header_t *h_location = table_getc(tx->response_headers, "location");
+                        htp_header_t *h_location = htp_table_get_c(tx->response_headers, "location");
                         if (h_location != NULL) {
                             MemBufferWriteString(aft->buffer, "(");
 
@@ -292,7 +292,7 @@ static void LogHttpLogCustom(LogHttpLogThread *aft, htp_tx_t *tx, const struct t
             case LOG_HTTP_CF_RESPONSE_HEADER:
             /* RESPONSE HEADER */
                 if (tx->response_headers != NULL) {
-                    h_response_hdr = table_getc(tx->response_headers,
+                    h_response_hdr = htp_table_get_c(tx->response_headers,
                                     httplog_ctx->cf_nodes[i]->data);
                 }
                 if (h_response_hdr != NULL) {
@@ -319,7 +319,7 @@ static void LogHttpLogExtended(LogHttpLogThread *aft, htp_tx_t *tx)
     /* referer */
     htp_header_t *h_referer = NULL;
     if (tx->request_headers != NULL) {
-        h_referer = table_getc(tx->request_headers, "referer");
+        h_referer = htp_table_get_c(tx->request_headers, "referer");
     }
     if (h_referer != NULL) {
         PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size,
@@ -355,7 +355,7 @@ static void LogHttpLogExtended(LogHttpLogThread *aft, htp_tx_t *tx)
                        bstr_len(tx->response_status));
         /* Redirect? */
         if ((tx->response_status_number > 300) && ((tx->response_status_number) < 303)) {
-            htp_header_t *h_location = table_getc(tx->response_headers, "location");
+            htp_header_t *h_location = htp_table_get_c(tx->response_headers, "location");
             if (h_location != NULL) {
                 MemBufferWriteString(aft->buffer, " => ");
 
@@ -410,9 +410,6 @@ static TmEcode LogHttpLogIPWrapper(ThreadVars *tv, Packet *p, void *data, Packet
     tx_progress_done_value_ts = AppLayerGetAlstateProgressCompletionStatus(ALPROTO_HTTP, 0);
     tx_progress_done_value_tc = AppLayerGetAlstateProgressCompletionStatus(ALPROTO_HTTP, 1);
 
-    if (htp_state->connp == NULL || htp_state->connp->conn == NULL)
-        goto end;
-
     CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
 
     char srcip[46], dstip[46];
@@ -501,7 +498,7 @@ static TmEcode LogHttpLogIPWrapper(ThreadVars *tv, Packet *p, void *data, Packet
             /* user agent */
             htp_header_t *h_user_agent = NULL;
             if (tx->request_headers != NULL) {
-                h_user_agent = table_getc(tx->request_headers, "user-agent");
+                h_user_agent = htp_table_get_c(tx->request_headers, "user-agent");
             }
             if (h_user_agent != NULL) {
                 PrintRawUriBuf((char *)aft->buffer->buffer, &aft->buffer->offset, aft->buffer->size,

diff --git a/src/suricata-common.h b/src/suricata-common.h
index 54203b2f9cff3a35795c6f2fcf99697f91e3d375..a7555ec80e2fcd291de71972530a499eb4fb6a47 100644 (file)
--- a/src/suricata-common.h
+++ b/src/suricata-common.h
@@ -313,9 +313,5 @@ typedef enum PacketProfileDetectId_ {
 size_t strlcat(char *, const char *src, size_t siz);
 size_t strlcpy(char *dst, const char *src, size_t siz);
 
-#define table_getc(x, y) table_get_c(x, y)
-#define bstr_cmpc(x, y) bstr_cmp_c(x, y)
-#define bstr_tocstr(x) bstr_util_strdup_to_c(x)
-
 #endif /* __SURICATA_COMMON_H__ */
 

diff --git a/src/suricata.c b/src/suricata.c
index ba77b3c451404e593243a798a3c94cc5f8d01894..9b43515492dc650c79303386a70de83688f507a8 100644 (file)
--- a/src/suricata.c
+++ b/src/suricata.c
@@ -625,9 +625,6 @@ void SCPrintBuildInfo(void) {
 #ifdef HAVE_HTP_URI_NORMALIZE_HOOK
     strlcat(features, "HAVE_HTP_URI_NORMALIZE_HOOK ", sizeof(features));
 #endif
-#ifdef HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW
-    strlcat(features, "HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW ", sizeof(features));
-#endif
 #ifdef PCRE_HAVE_JIT
     strlcat(features, "PCRE_JIT ", sizeof(features));
 #endif
@@ -700,7 +697,7 @@ void SCPrintBuildInfo(void) {
     printf("L1 cache line size (CLS)=%d\n", CLS);
 #endif
 
-    printf("compiled with libhtp %s, linked against %s\n", HTP_BASE_VERSION_TEXT, htp_get_version());
+    printf("compiled with libhtp %s, linked against %s\n", HTP_VERSION_STRING, HTP_VERSION_STRING);
 
 #include "build-info.h"
 }
@@ -1290,11 +1287,6 @@ int main(int argc, char **argv)
 #endif
     }
 
-#ifndef HAVE_HTP_TX_GET_RESPONSE_HEADERS_RAW
-    SCLogWarning(SC_WARN_OUTDATED_LIBHTP, "libhtp < 0.2.7 detected. Keyword "
-        "http_raw_header will not be able to inspect response headers.");
-#endif
-
     SetBpfString(optind, argv);
 
     if (!list_keywords && !list_app_layer_protocols)

diff --git a/suricata.yaml.in b/suricata.yaml.in
index 4e56f16e437c57d5bd7a0600233b611d566e260b..dc6be13b961bd757ce35f06122e069ef35a34069 100644 (file)
--- a/suricata.yaml.in
+++ b/suricata.yaml.in
@@ -962,8 +962,7 @@ pcre:
 #   IIS_6_0
 #   IIS_7_0
 #   IIS_7_5
-#   Apache
-#   Apache_2_2
+#   Apache_2
 ###########################################################################
 libhtp:
 
@@ -989,7 +988,7 @@ libhtp:
 
      - apache:
          address: [192.168.1.0/24, 127.0.0.0/8, "::1"]
-         personality: Apache_2_2
+         personality: Apache_2
          # Can be specified in kb, mb, gb.  Just a number indicates
          # it's in bytes.
          request-body-limit: 4096