From: Shravan Rangarajuvenkata (shrarang) <shrarang@cisco.com>
Date: Mon, 20 Apr 2020 22:29:04 +0000 (+0000)
Subject: Merge pull request #2156 in SNORT/snort3 from ~KAMURTHI/snort3:http2_multi_stream... 
X-Git-Tag: 3.0.1-2~6
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2c7c87b97be21548ef5a7751c0576c7ce7c612ef;p=thirdparty%2Fsnort3.git

Merge pull request #2156 in SNORT/snort3 from ~KAMURTHI/snort3:http2_multi_stream to master

Squashed commit of the following:

commit af68aa5f7982ddeaa6d628dd21f9df6fd05192d6
Author: Kanimozhi Murthi <kamurthi@cisco.com>
Date:   Mon Apr 20 00:14:50 2020 -0400

    appid: Changing sessionAPI to accomodate stream_index
---

diff --git a/src/network_inspectors/appid/CMakeLists.txt b/src/network_inspectors/appid/CMakeLists.txt
index d333b2262..e0246f137 100644
--- a/src/network_inspectors/appid/CMakeLists.txt
+++ b/src/network_inspectors/appid/CMakeLists.txt
@@ -1,5 +1,6 @@
 set (APPID_INCLUDES
     appid_api.h
+    appid_app_descriptor.h
     appid_dns_session.h
     appid_http_session.h
     appid_session_api.h
diff --git a/src/network_inspectors/appid/appid_app_descriptor.cc b/src/network_inspectors/appid/appid_app_descriptor.cc
index c749bae09..17fb65159 100644
--- a/src/network_inspectors/appid/appid_app_descriptor.cc
+++ b/src/network_inspectors/appid/appid_app_descriptor.cc
@@ -9,10 +9,27 @@
 #endif
 
 #include "appid_app_descriptor.h"
+#include "app_info_table.h"
+#include "appid_config.h"
+#include "appid_module.h"
+#include "appid_peg_counts.h"
+#include "appid_types.h"
 #include "lua_detector_api.h"
 
 using namespace snort;
 
+void ApplicationDescriptor::set_id(AppId app_id)
+{
+    if ( my_id != app_id )
+    {
+        my_id = app_id;
+        if ( app_id > APP_ID_NONE )
+            update_stats(app_id);
+        else if ( app_id == APP_ID_UNKNOWN )
+            appid_stats.appid_unknown++;
+    }
+}
+
 void ApplicationDescriptor::set_id(const Packet& p, AppIdSession& asd,
     AppidSessionDirection dir, AppId app_id, AppidChangeBits& change_bits)
 {
@@ -23,3 +40,48 @@ void ApplicationDescriptor::set_id(const Packet& p, AppIdSession& asd,
     }
 }
 
+void ServiceAppDescriptor::update_stats(AppId id)
+{
+    AppIdPegCounts::inc_service_count(id);
+}
+
+void ServiceAppDescriptor::set_port_service_id(AppId id)
+{
+    if ( id != port_service_id )
+    {
+        port_service_id = id;
+        if ( id > APP_ID_NONE )
+            AppIdPegCounts::inc_service_count(id);
+    }
+}
+
+void ServiceAppDescriptor::set_id(AppId app_id, OdpContext& odp_ctxt)
+{
+    if (get_id() != app_id)
+    {
+        ApplicationDescriptor::set_id(app_id);
+        deferred = odp_ctxt.get_app_info_mgr().get_app_info_flags(app_id, APPINFO_FLAG_DEFER);
+    }
+}
+
+void ClientAppDescriptor::update_user(AppId app_id, const char* username)
+{
+    my_username = username;
+
+    if ( my_user_id != app_id )
+    {
+        my_user_id = app_id;
+        if ( app_id > APP_ID_NONE )
+            AppIdPegCounts::inc_user_count(app_id);
+    }
+}
+
+void ClientAppDescriptor::update_stats(AppId id)
+{
+    AppIdPegCounts::inc_client_count(id);
+}
+
+void PayloadAppDescriptor::update_stats(AppId id)
+{
+    AppIdPegCounts::inc_payload_count(id);
+}
diff --git a/src/network_inspectors/appid/appid_app_descriptor.h b/src/network_inspectors/appid/appid_app_descriptor.h
index 7ae080366..05097e119 100644
--- a/src/network_inspectors/appid/appid_app_descriptor.h
+++ b/src/network_inspectors/appid/appid_app_descriptor.h
@@ -33,14 +33,12 @@
 #include "protocols/packet.h"
 #include "pub_sub/appid_events.h"
 
-#include "app_info_table.h"
-#include "appid_config.h"
-#include "appid_module.h"
-#include "appid_peg_counts.h"
 #include "appid_types.h"
+#include "application_ids.h"
 
 class AppIdDetector;
 class AppIdSession;
+class OdpContext;
 
 class ApplicationDescriptor
 {
@@ -68,17 +66,7 @@ public:
         return my_id;
     }
 
-    virtual void set_id(AppId app_id)
-    {
-        if ( my_id != app_id )
-        {
-            my_id = app_id;
-            if ( app_id > APP_ID_NONE )
-                update_stats(app_id);
-            else if ( app_id == APP_ID_UNKNOWN )
-                appid_stats.appid_unknown++;
-        }
-    }
+    virtual void set_id(AppId app_id);
 
     virtual void set_id(const snort::Packet& p, AppIdSession& asd, AppidSessionDirection dir, AppId app_id, AppidChangeBits& change_bits);
 
@@ -118,14 +106,7 @@ class ServiceAppDescriptor : public ApplicationDescriptor
 public:
     ServiceAppDescriptor() = default;
 
-    void set_id(AppId app_id, OdpContext& odp_ctxt)
-    {
-        if (get_id() != app_id)
-        {
-            ApplicationDescriptor::set_id(app_id);
-            deferred = odp_ctxt.get_app_info_mgr().get_app_info_flags(app_id, APPINFO_FLAG_DEFER);
-        }
-    }
+    void set_id(AppId app_id, OdpContext& odp_ctxt);
 
     void reset() override
     {
@@ -133,25 +114,14 @@ public:
         port_service_id = APP_ID_NONE;
     }
 
-    void update_stats(AppId id) override
-    {
-        AppIdPegCounts::inc_service_count(id);
-    }
+    void update_stats(AppId id) override;
 
     AppId get_port_service_id() const
     {
         return port_service_id;
     }
 
-    void set_port_service_id(AppId id)
-    {
-        if ( id != port_service_id )
-        {
-            port_service_id = id;
-            if ( id > APP_ID_NONE )
-                AppIdPegCounts::inc_service_count(id);
-        }
-    }
+    void set_port_service_id(AppId id);
 
     bool get_deferred()
     {
@@ -176,18 +146,7 @@ public:
         my_user_id = APP_ID_NONE;
     }
 
-    void update_user(AppId app_id, const char* username)
-    {
-        if ( my_username != username )
-            my_username = username;
-
-        if ( my_user_id != app_id )
-        {
-            my_user_id = app_id;
-            if ( app_id > APP_ID_NONE )
-                AppIdPegCounts::inc_user_count(app_id);
-        }
-    }
+    void update_user(AppId app_id, const char* username);
 
     AppId get_user_id() const
     {
@@ -199,10 +158,7 @@ public:
         return my_username.empty() ? nullptr : my_username.c_str();
     }
 
-    void update_stats(AppId id) override
-    {
-        AppIdPegCounts::inc_client_count(id);
-    }
+    void update_stats(AppId id) override;
 
 private:
     std::string my_username;
@@ -219,10 +175,7 @@ public:
         ApplicationDescriptor::reset();
     }
 
-    void update_stats(AppId id) override
-    {
-        AppIdPegCounts::inc_payload_count(id);
-    }
+    void update_stats(AppId id) override;
 };
 
 #endif
diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc
index 36378e902..f008f96b0 100644
--- a/src/network_inspectors/appid/appid_discovery.cc
+++ b/src/network_inspectors/appid/appid_discovery.cc
@@ -672,7 +672,9 @@ bool AppIdDiscovery::do_host_port_based_discovery(Packet* p, AppIdSession& asd,
     const SfIp* ip;
     AppIdHttpSession* hsession = asd.get_http_session();
 
-    const TunnelDest* tun_dest = hsession->get_tun_dest();
+    const TunnelDest* tun_dest = nullptr;
+    if (hsession)
+        tun_dest  = hsession->get_tun_dest();
     if (tun_dest)
     {
         ip = &(tun_dest->ip);
diff --git a/src/network_inspectors/appid/appid_http_event_handler.cc b/src/network_inspectors/appid/appid_http_event_handler.cc
index e2b757625..7d00314e0 100644
--- a/src/network_inspectors/appid/appid_http_event_handler.cc
+++ b/src/network_inspectors/appid/appid_http_event_handler.cc
@@ -60,7 +60,10 @@ void HttpEventHandler::handle(DataEvent& event, Flow* flow)
 
     direction = event_type == REQUEST_EVENT ? APP_ID_FROM_INITIATOR : APP_ID_FROM_RESPONDER;
 
-    AppIdHttpSession* hsession = asd->get_http_session();
+    AppIdHttpSession* hsession = asd->get_http_session(0);
+
+    if (!hsession)
+        hsession = asd->create_http_session();
 
     if (direction == APP_ID_FROM_INITIATOR)
     {
diff --git a/src/network_inspectors/appid/appid_http_session.h b/src/network_inspectors/appid/appid_http_session.h
index 5ede3da0e..bb118c3af 100644
--- a/src/network_inspectors/appid/appid_http_session.h
+++ b/src/network_inspectors/appid/appid_http_session.h
@@ -29,6 +29,7 @@
 #include "pub_sub/appid_events.h"
 #include "sfip/sf_ip.h"
 
+#include "appid_app_descriptor.h"
 #include "appid_types.h"
 #include "application_ids.h"
 
@@ -36,42 +37,6 @@ class AppIdSession;
 class ChpMatchDescriptor;
 class HttpPatternMatchers;
 
-// These values are used in Lua code as raw numbers. Do NOT reassign new values.
-// 0 - 8 (inclusive)       : used heavily in CHP code. DO NOT CHANGE.
-// 9 - NUM_METADATA_FIELDS : extra metadata buffers, beyond CHP.
-// NUM_METADATA_FIELDS     : must always follow the last metadata FID.
-// NUM_HTTP_FIELDS         : number of CHP fields, so always RSP_BODY_FID + 1
-enum HttpFieldIds : uint8_t
-{
-    // 0-8: CHP fields. DO NOT CHANGE
-
-    // Request-side headers
-    REQ_AGENT_FID,          // 0
-    REQ_HOST_FID,           // 1
-    REQ_REFERER_FID,        // 2
-    REQ_URI_FID,            // 3
-    REQ_COOKIE_FID,         // 4
-    REQ_BODY_FID,           // 5
-    // Response-side headers
-    RSP_CONTENT_TYPE_FID,   // 6
-    RSP_LOCATION_FID,       // 7
-    RSP_BODY_FID,           // 8
-
-    // extra (non-CHP) metadata fields.
-    MISC_VIA_FID,           // 9
-    MISC_RESP_CODE_FID,     // 10
-    MISC_SERVER_FID,        // 11
-    MISC_XWW_FID,           // 12
-    MISC_URL_FID,           // 13
-
-    // Total number of metadata fields, always first after actual FIDs.
-    NUM_METADATA_FIELDS,    // 14
-
-    // Number of CHP fields, always 1 past RSP_BODY_FIELD
-    NUM_HTTP_FIELDS = MISC_VIA_FID,
-    MAX_KEY_PATTERN = REQ_URI_FID,     // DO NOT CHANGE, used in CHP
-};
-
 #define RESPONSE_CODE_PACKET_THRESHHOLD 0
 
 // These values are used in Lua code as raw numbers. Do NOT reassign new values.
@@ -97,6 +62,10 @@ public:
 
     AppIdHttpSession(AppIdSession&);
     virtual ~AppIdHttpSession();
+    ClientAppDescriptor client;
+    PayloadAppDescriptor payload;
+    AppId referred_payload_app_id = APP_ID_NONE;
+    AppId misc_app_id = APP_ID_NONE;
 
     int process_http_packet(AppidSessionDirection direction, AppidChangeBits& change_bits,
         HttpPatternMatchers& http_matchers);
diff --git a/src/network_inspectors/appid/appid_inspector.cc b/src/network_inspectors/appid/appid_inspector.cc
index 1e8e614dd..5a9b58812 100644
--- a/src/network_inspectors/appid/appid_inspector.cc
+++ b/src/network_inspectors/appid/appid_inspector.cc
@@ -38,6 +38,7 @@
 #include "appid_debug.h"
 #include "appid_discovery.h"
 #include "appid_http_event_handler.h"
+#include "appid_peg_counts.h"
 #include "appid_session.h"
 #include "appid_stats.h"
 #include "client_plugins/client_discovery.h"
diff --git a/src/network_inspectors/appid/appid_session.cc b/src/network_inspectors/appid/appid_session.cc
index 29d7c7e8d..a60f9fcae 100644
--- a/src/network_inspectors/appid/appid_session.cc
+++ b/src/network_inspectors/appid/appid_session.cc
@@ -227,8 +227,8 @@ void AppIdSession::reinit_session_data(AppidChangeBits& change_bits)
         payload.reset();
         referred_payload_app_id = tp_payload_app_id = APP_ID_NONE;
         clear_session_flags(APPID_SESSION_CONTINUE);
-        if (hsession)
-            hsession->set_field(MISC_URL_FID, nullptr, change_bits);
+        if (!hsessions.empty())
+            hsessions[0]->set_field(MISC_URL_FID, nullptr, change_bits);
     }
 
     //service
@@ -471,13 +471,13 @@ void AppIdSession::examine_rtmp_metadata(AppidChangeBits& change_bits)
     AppId referred_payload_id = APP_ID_NONE;
     char* version = nullptr;
 
-    if (!hsession)
-        hsession = new AppIdHttpSession(*this);
+    if (hsessions.empty())
+        return;
 
-    if (const char* url = hsession->get_cfield(MISC_URL_FID))
+    if (const char* url = hsessions[0]->get_cfield(MISC_URL_FID))
     {
         HttpPatternMatchers& http_matchers = ctxt.get_odp_ctxt().get_http_matchers();
-        const char* referer = hsession->get_cfield(REQ_REFERER_FID);
+        const char* referer = hsessions[0]->get_cfield(REQ_REFERER_FID);
         if (((http_matchers.get_appid_from_url(nullptr, url, &version,
             referer, &client_id, &service_id, &payload_id,
             &referred_payload_id, true, ctxt.get_odp_ctxt())) ||
@@ -591,7 +591,8 @@ void AppIdSession::delete_session_data()
         rna_ss = subtype;
     }
 
-    delete hsession;
+    for (auto* hsession: hsessions)
+        delete hsession;
     free_tls_session_data();
     delete dsession;
 }
@@ -880,7 +881,7 @@ void AppIdSession::reset_session_data()
     delete_session_data();
     netbios_name = nullptr;
     netbios_domain = nullptr;
-    hsession = nullptr;
+    hsessions.clear();
 
     tp_payload_app_id = APP_ID_UNKNOWN;
     tp_app_id = APP_ID_UNKNOWN;
@@ -906,17 +907,24 @@ void AppIdSession::clear_http_flags()
 
 void AppIdSession::clear_http_data()
 {
-    if (!hsession)
+    if (hsessions.empty())
         return;
-    hsession->clear_all_fields();
+    hsessions[0]->clear_all_fields();
 }
 
-AppIdHttpSession* AppIdSession::get_http_session()
+AppIdHttpSession* AppIdSession::create_http_session()
 {
-    if (!hsession)
-        hsession = new AppIdHttpSession(*this);
+    AppIdHttpSession* hsession = new AppIdHttpSession(*this);
+    hsessions.push_back(hsession);
     return hsession;
 }
+AppIdHttpSession* AppIdSession::get_http_session(uint32_t stream_index)
+{
+    if (stream_index < hsessions.size())
+        return hsessions[stream_index];
+    else
+        return nullptr;
+}
 
 AppIdDnsSession* AppIdSession::get_dns_session()
 {
diff --git a/src/network_inspectors/appid/appid_session.h b/src/network_inspectors/appid/appid_session.h
index d83dd7fb0..e057d99b5 100644
--- a/src/network_inspectors/appid/appid_session.h
+++ b/src/network_inspectors/appid/appid_session.h
@@ -32,6 +32,8 @@
 #include "app_info_table.h"
 #include "appid_api.h"
 #include "appid_app_descriptor.h"
+#include "appid_config.h"
+#include "appid_http_session.h"
 #include "appid_types.h"
 #include "application_ids.h"
 #include "detector_plugins/http_url_patterns.h"
@@ -328,6 +330,10 @@ public:
     AppId get_application_ids_client();
     AppId get_application_ids_payload();
     AppId get_application_ids_misc();
+    uint32_t get_hsessions_size()
+    {
+        return hsessions.size();
+    }
 
     bool is_ssl_session_decrypted();
     void examine_ssl_metadata(snort::Packet*, AppidChangeBits& change_bits);
@@ -346,7 +352,8 @@ public:
     void clear_http_data();
     void reset_session_data();
 
-    AppIdHttpSession* get_http_session();
+    AppIdHttpSession* create_http_session();
+    AppIdHttpSession* get_http_session(uint32_t stream_index = 0);
     AppIdDnsSession* get_dns_session();
 
     bool is_tp_appid_done() const;
@@ -400,7 +407,7 @@ public:
     }
 
 private:
-    AppIdHttpSession* hsession = nullptr;
+    std::vector<AppIdHttpSession*> hsessions;
     AppIdDnsSession* dsession = nullptr;
 
     void reinit_session_data(AppidChangeBits& change_bits);
diff --git a/src/network_inspectors/appid/appid_session_api.cc b/src/network_inspectors/appid/appid_session_api.cc
index 340be1745..b2c86f5fc 100644
--- a/src/network_inspectors/appid/appid_session_api.cc
+++ b/src/network_inspectors/appid/appid_session_api.cc
@@ -60,36 +60,110 @@ AppId AppIdSessionApi::get_only_service_app_id()
     return asd->pick_only_service_app_id();
 }
 
-AppId AppIdSessionApi::get_misc_app_id()
+AppId AppIdSessionApi::get_misc_app_id(uint32_t stream_index)
 {
-    return asd->get_application_ids_misc();
+    if (asd->is_http2)
+    {
+        if (stream_index >= asd->get_hsessions_size())
+            return APP_ID_UNKNOWN;
+        return asd->get_http_session(stream_index)->misc_app_id;
+    }
+    else
+        return asd->get_application_ids_misc();
 }
 
-AppId AppIdSessionApi::get_client_app_id()
+AppId AppIdSessionApi::get_client_app_id(uint32_t stream_index)
 {
-    return asd->get_application_ids_client();
+    if (asd->is_http2)
+    {
+        if (stream_index >= asd->get_hsessions_size())
+            return APP_ID_UNKNOWN;
+        return asd->get_http_session(stream_index)->client.get_id();
+    }
+    else
+        return asd->get_application_ids_client();
 }
 
-AppId AppIdSessionApi::get_payload_app_id()
+AppId AppIdSessionApi::get_payload_app_id(uint32_t stream_index)
 {
-    return asd->get_application_ids_payload();
+    if (asd->is_http2)
+    {
+        if (stream_index >= asd->get_hsessions_size())
+            return APP_ID_UNKNOWN;
+        return asd->get_http_session(stream_index)->payload.get_id();
+    }
+    else
+        return asd->get_application_ids_payload();
 }
 
-AppId AppIdSessionApi::get_referred_app_id()
+AppId AppIdSessionApi::get_referred_app_id(uint32_t stream_index)
 {
-    return asd->pick_referred_payload_app_id();
+    if (asd->is_http2)
+    {
+        if (stream_index >= asd->get_hsessions_size())
+            return APP_ID_UNKNOWN;
+        return asd->get_http_session(stream_index)->referred_payload_app_id;
+    }
+    else
+        return asd->pick_referred_payload_app_id();
 }
 
 void AppIdSessionApi::get_app_id(AppId& service, AppId& client,
-    AppId& payload, AppId& misc, AppId& referred)
+    AppId& payload, AppId& misc, AppId& referred, uint32_t stream_index)
 {
-    asd->get_application_ids(service, client, payload, misc);
-    referred = asd->pick_referred_payload_app_id();
+    if (asd->is_http2)
+    {
+        if (stream_index >= asd->get_hsessions_size())
+            service = client = payload = misc = referred = APP_ID_UNKNOWN;
+        else
+        {
+            service = asd->get_application_ids_service();
+            client = asd->get_http_session(stream_index)->client.get_id();
+            payload = asd->get_http_session(stream_index)->payload.get_id();
+            misc = asd->get_http_session(stream_index)->misc_app_id;
+            referred = asd->get_http_session(stream_index)->referred_payload_app_id;
+        }
+    }
+    else
+    {
+        asd->get_application_ids(service, client, payload, misc);
+        referred = asd->pick_referred_payload_app_id();
+    }
 }
 
 void AppIdSessionApi::get_app_id(AppId* service, AppId* client,
-    AppId* payload, AppId* misc, AppId* referred)
+    AppId* payload, AppId* misc, AppId* referred, uint32_t stream_index)
 {
+    if (asd->is_http2)
+    {
+        if (stream_index >= asd->get_hsessions_size())
+        {
+            if(service)
+                *service = APP_ID_UNKNOWN;
+            if(client)
+                *client = APP_ID_UNKNOWN;
+            if(payload)
+                *payload = APP_ID_UNKNOWN;
+            if(misc)
+                *misc = APP_ID_UNKNOWN;
+            if(referred)
+                *referred = APP_ID_UNKNOWN;
+        }
+        else
+        {
+            AppIdHttpSession* hsession = asd->get_http_session(stream_index);
+            if (service)
+                *service = asd->get_application_ids_service();
+            if (client)
+                *client = hsession->client.get_id();
+            if (payload)
+                *payload = hsession->payload.get_id();
+            if (misc)
+                *misc = hsession->misc_app_id;
+            if (referred)
+                *referred = hsession->referred_payload_app_id;
+        }
+    }
     if (service)
         *service = asd->get_application_ids_service();
     if (client)
@@ -154,9 +228,16 @@ bool AppIdSessionApi::is_appid_available()
         asd->get_session_flags(APPID_SESSION_NO_TPI)) );
 }
 
-const char* AppIdSessionApi::get_client_version()
+const char* AppIdSessionApi::get_client_version(uint32_t stream_index)
 {
-    return asd->client.get_version();
+    if (asd->is_http2)
+    {
+        if (stream_index >= asd->get_hsessions_size())
+            return nullptr;
+        return asd->get_http_session(stream_index)->client.get_version();
+    }
+    else
+        return asd->client.get_version();
 }
 
 uint64_t AppIdSessionApi::get_appid_session_attribute(uint64_t flags)
@@ -247,9 +328,9 @@ AppIdDnsSession* AppIdSessionApi::get_dns_session()
     return asd->get_dns_session();
 }
 
-AppIdHttpSession* AppIdSessionApi::get_http_session()
+AppIdHttpSession* AppIdSessionApi::get_http_session(uint32_t stream_index)
 {
-    return asd->get_http_session();
+    return asd->get_http_session(stream_index);
 }
 
 bool AppIdSessionApi::is_http_inspection_done()
diff --git a/src/network_inspectors/appid/appid_session_api.h b/src/network_inspectors/appid/appid_session_api.h
index cc38c51fd..e9f978c49 100644
--- a/src/network_inspectors/appid/appid_session_api.h
+++ b/src/network_inspectors/appid/appid_session_api.h
@@ -151,17 +151,17 @@ public:
     AppId get_service_app_id();
     AppId get_port_service_app_id();
     AppId get_only_service_app_id();
-    AppId get_misc_app_id();
-    AppId get_client_app_id();
-    AppId get_payload_app_id();
-    AppId get_referred_app_id();
-    void get_app_id(AppId& service, AppId& client, AppId& payload, AppId& misc, AppId& referred);
-    void get_app_id(AppId* service, AppId* client, AppId* payload, AppId* misc, AppId* referred);
+    AppId get_misc_app_id(uint32_t stream_index = 0);
+    AppId get_client_app_id(uint32_t stream_index = 0);
+    AppId get_payload_app_id(uint32_t stream_index = 0);
+    AppId get_referred_app_id(uint32_t stream_index = 0);
+    void get_app_id(AppId& service, AppId& client, AppId& payload, AppId& misc, AppId& referred, uint32_t stream_index = 0);
+    void get_app_id(AppId* service, AppId* client, AppId* payload, AppId* misc, AppId* referred, uint32_t stream_index = 0);
     bool is_ssl_session_decrypted();
     bool is_appid_inspecting_session();
     bool is_appid_available();
     const char* get_user_name(AppId* service, bool* isLoginSuccessful);
-    const char* get_client_version();
+    const char* get_client_version(uint32_t stream_index = 0);
     uint64_t get_appid_session_attribute(uint64_t flag);
     APPID_FLOW_TYPE get_flow_type();
     void get_service_info(const char** vendor, const char** version,
@@ -170,7 +170,7 @@ public:
     SfIp* get_service_ip();
     SfIp* get_initiator_ip();
     AppIdDnsSession* get_dns_session();
-    AppIdHttpSession* get_http_session();
+    AppIdHttpSession* get_http_session(uint32_t stream_index = 0);
     char* get_tls_host();
     DHCPData* get_dhcp_fp_data();
     void free_dhcp_fp_data(DHCPData*);
diff --git a/src/network_inspectors/appid/appid_types.h b/src/network_inspectors/appid/appid_types.h
index 42c71a7f7..2e1590413 100644
--- a/src/network_inspectors/appid/appid_types.h
+++ b/src/network_inspectors/appid/appid_types.h
@@ -21,6 +21,42 @@
 
 #ifndef APPID_TYPES_H
 #define APPID_TYPES_H
+#include <cstdint>
+// These values are used in Lua code as raw numbers. Do NOT reassign new values.
+// 0 - 8 (inclusive)       : used heavily in CHP code. DO NOT CHANGE.
+// 9 - NUM_METADATA_FIELDS : extra metadata buffers, beyond CHP.
+// NUM_METADATA_FIELDS     : must always follow the last metadata FID.
+// NUM_HTTP_FIELDS         : number of CHP fields, so always RSP_BODY_FID + 1
+enum HttpFieldIds : uint8_t
+{
+    // 0-8: CHP fields. DO NOT CHANGE
+
+    // Request-side headers
+    REQ_AGENT_FID,          // 0
+    REQ_HOST_FID,           // 1
+    REQ_REFERER_FID,        // 2
+    REQ_URI_FID,            // 3
+    REQ_COOKIE_FID,         // 4
+    REQ_BODY_FID,           // 5
+    // Response-side headers
+    RSP_CONTENT_TYPE_FID,   // 6
+    RSP_LOCATION_FID,       // 7
+    RSP_BODY_FID,           // 8
+
+    // extra (non-CHP) metadata fields.
+    MISC_VIA_FID,           // 9
+    MISC_RESP_CODE_FID,     // 10
+    MISC_SERVER_FID,        // 11
+    MISC_XWW_FID,           // 12
+    MISC_URL_FID,           // 13
+
+    // Total number of metadata fields, always first after actual FIDs.
+    NUM_METADATA_FIELDS,    // 14
+
+    // Number of CHP fields, always 1 past RSP_BODY_FIELD
+    NUM_HTTP_FIELDS = MISC_VIA_FID,
+    MAX_KEY_PATTERN = REQ_URI_FID,     // DO NOT CHANGE, used in CHP
+};
 
 enum AppidSessionDirection
 {
@@ -28,5 +64,4 @@ enum AppidSessionDirection
     APP_ID_FROM_RESPONDER,
     APP_ID_APPID_SESSION_DIRECTION_MAX
 };
-
 #endif
diff --git a/src/network_inspectors/appid/client_plugins/test/client_plugin_mock.h b/src/network_inspectors/appid/client_plugins/test/client_plugin_mock.h
index e01a97e4a..34189df39 100644
--- a/src/network_inspectors/appid/client_plugins/test/client_plugin_mock.h
+++ b/src/network_inspectors/appid/client_plugins/test/client_plugin_mock.h
@@ -20,6 +20,10 @@
 #ifndef CLIENT_PLUGIN_MOCK_H
 #define CLIENT_PLUGIN_MOCK_H
 
+#include "appid_detector.h"
+#include "appid_module.h"
+#include "appid_peg_counts.h"
+#include "utils/stats.h"
 namespace snort
 {
 // Stubs for messages
@@ -89,6 +93,7 @@ void AppIdDiscovery::register_udp_pattern(AppIdDetector*, const uint8_t* const,
     int, unsigned){}
 int AppIdDiscovery::add_service_port(AppIdDetector*, const ServiceDetectorPort&){return 0;}
 void ApplicationDescriptor::set_id(const snort::Packet&, AppIdSession&, AppidSessionDirection, AppId, AppidChangeBits&){}
+void ApplicationDescriptor::set_id(AppId){}
 AppIdDiscovery::AppIdDiscovery() { }
 AppIdDiscovery::~AppIdDiscovery() { }
 void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { }
diff --git a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
index d1e524373..c9d4dda54 100644
--- a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
+++ b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
@@ -30,7 +30,7 @@
 #include "search_engines/search_tool.h"
 #include "utils/util.h"
 
-#include "appid_http_session.h"
+#include "appid_types.h"
 #include "appid_utils/sf_mlmp.h"
 #include "application_ids.h"
 
diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
index 5a03142ef..5842342e4 100644
--- a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
+++ b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
@@ -19,6 +19,10 @@
 
 #ifndef DETECTOR_PLUGINS_MOCK_H
 #define DETECTOR_PLUGINS_MOCK_H
+#include "appid_detector.h"
+#include "appid_module.h"
+#include "appid_peg_counts.h"
+#include "utils/stats.h"
 
 namespace snort
 {
@@ -201,7 +205,12 @@ bool AppIdReloadTuner::tune_resources(unsigned int)
 {
     return true;
 }
-
+void ApplicationDescriptor::set_id(AppId){}
+void ServiceAppDescriptor::set_id(AppId, OdpContext&){}
+void ServiceAppDescriptor::update_stats(AppId){}
+void ClientAppDescriptor::update_user(AppId, const char*){}
+void ClientAppDescriptor::update_stats(AppId) {}
+void PayloadAppDescriptor::update_stats(AppId) {}
 void ServiceDiscovery::initialize()
 { }
 
diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc
index 7eabfc532..9ea39e67d 100644
--- a/src/network_inspectors/appid/lua_detector_api.cc
+++ b/src/network_inspectors/appid/lua_detector_api.cc
@@ -37,6 +37,7 @@
 #include "app_info_table.h"
 #include "appid_debug.h"
 #include "appid_inspector.h"
+#include "appid_peg_counts.h"
 #include "client_plugins/client_discovery.h"
 #include "detector_plugins/detector_dns.h"
 #include "detector_plugins/detector_pattern.h"
diff --git a/src/network_inspectors/appid/service_plugins/service_rtmp.cc b/src/network_inspectors/appid/service_plugins/service_rtmp.cc
index d1a459752..0c677c0da 100644
--- a/src/network_inspectors/appid/service_plugins/service_rtmp.cc
+++ b/src/network_inspectors/appid/service_plugins/service_rtmp.cc
@@ -633,6 +633,9 @@ fail:
 
 success:
     AppIdHttpSession* hsession = args.asd.get_http_session();
+    if (!hsession)
+        hsession = args.asd.create_http_session();
+
     if ( ss->swfUrl )
     {
         if ( !hsession->get_field(MISC_URL_FID) )
diff --git a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h
index b9d428538..c0c7b8148 100644
--- a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h
+++ b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h
@@ -19,6 +19,10 @@
 
 #ifndef SERVICE_PLUGIN_MOCK_H
 #define SERVICE_PLUGIN_MOCK_H
+#include "appid_detector.h"
+#include "appid_module.h"
+#include "appid_peg_counts.h"
+#include "utils/stats.h"
 
 namespace snort
 {
@@ -88,6 +92,12 @@ void AppIdDetector::add_info(AppIdSession&, const char*, AppidChangeBits&){}
 void AppIdDetector::add_user(AppIdSession&, const char*, AppId, bool){}
 void AppIdDetector::add_payload(AppIdSession&, AppId){}
 void AppIdDetector::add_app(const snort::Packet&, AppIdSession&, AppidSessionDirection, AppId, AppId, const char*, AppidChangeBits&){}
+void ApplicationDescriptor::set_id(AppId){}
+void ServiceAppDescriptor::set_id(AppId, OdpContext&){}
+void ServiceAppDescriptor::update_stats(AppId){}
+void ClientAppDescriptor::update_user(AppId, const char*){}
+void ClientAppDescriptor::update_stats(AppId) {}
+void PayloadAppDescriptor::update_stats(AppId) {}
 void AppIdDiscovery::add_pattern_data(AppIdDetector*, snort::SearchTool*, int,
         const uint8_t* const, unsigned, unsigned){}
 void AppIdDiscovery::register_detector(const std::string&, AppIdDetector*,  IpProtocol){}
diff --git a/src/network_inspectors/appid/test/appid_detector_test.cc b/src/network_inspectors/appid/test/appid_detector_test.cc
index eb4fc6d26..fa864cd15 100644
--- a/src/network_inspectors/appid/test/appid_detector_test.cc
+++ b/src/network_inspectors/appid/test/appid_detector_test.cc
@@ -72,20 +72,6 @@ TEST_GROUP(appid_detector_tests)
     }
 };
 
-TEST(appid_detector_tests, add_info)
-{
-    const char* info_url = "https://tools.ietf.org/html/rfc793";
-    AppidChangeBits change_bits;
-    AppIdDetector* ad = new TestDetector;
-    MockAppIdHttpSession* hsession = (MockAppIdHttpSession*)mock_session->get_http_session();
-    ad->add_info(*mock_session, info_url, change_bits);
-    STRCMP_EQUAL(hsession->get_cfield(MISC_URL_FID), URL);
-    hsession->reset();
-    ad->add_info(*mock_session, info_url, change_bits);
-    STRCMP_EQUAL(mock_session->get_http_session()->get_cfield(MISC_URL_FID), info_url);
-    delete ad;
-}
-
 TEST(appid_detector_tests, add_user)
 {
     const char* username = "snorty";
diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc
index d832a7a32..42a3aa441 100644
--- a/src/network_inspectors/appid/test/appid_discovery_test.cc
+++ b/src/network_inspectors/appid/test/appid_discovery_test.cc
@@ -25,6 +25,7 @@
 
 #include "host_tracker/host_cache.h"
 #include "network_inspectors/appid/appid_discovery.cc"
+#include "network_inspectors/appid/appid_peg_counts.h"
 
 #include "search_engines/search_tool.h"
 #include "utils/sflsq.cc"
@@ -123,6 +124,17 @@ SipPatternMatchers::~SipPatternMatchers() { }
 SslPatternMatchers::~SslPatternMatchers() { }
 
 void ApplicationDescriptor::set_id(const Packet&, AppIdSession&, AppidSessionDirection, AppId, AppidChangeBits&) { }
+void ApplicationDescriptor::set_id(AppId app_id){my_id = app_id;}
+void ServiceAppDescriptor::set_id(AppId app_id, OdpContext& odp_ctxt)
+{
+    set_id(app_id);
+    deferred = odp_ctxt.get_app_info_mgr().get_app_info_flags(app_id, APPINFO_FLAG_DEFER);
+}
+void ServiceAppDescriptor::update_stats(AppId){}
+void ServiceAppDescriptor::set_port_service_id(AppId){}
+void ClientAppDescriptor::update_user(AppId, const char*){}
+void ClientAppDescriptor::update_stats(AppId) {}
+void PayloadAppDescriptor::update_stats(AppId) {}
 
 // Stubs for AppIdModule
 AppIdModule::AppIdModule(): Module("appid_mock", "appid_mock_help") {}
diff --git a/src/network_inspectors/appid/test/appid_http_event_test.cc b/src/network_inspectors/appid/test/appid_http_event_test.cc
index 95d3415f9..0981aea9c 100644
--- a/src/network_inspectors/appid/test/appid_http_event_test.cc
+++ b/src/network_inspectors/appid/test/appid_http_event_test.cc
@@ -199,6 +199,7 @@ TEST_GROUP(appid_http_event)
         MemoryLeakWarningPlugin::turnOffNewDeleteOverloads();
         flow = new Flow;
         mock_session = new AppIdSession(IpProtocol::TCP, nullptr, 1492, appid_inspector);
+        mock_session->create_http_session();
         flow->set_flow_data(mock_session);
         appidDebug = new AppIdDebug();
         appidDebug->activate(nullptr, nullptr, 0);
diff --git a/src/network_inspectors/appid/test/appid_http_session_test.cc b/src/network_inspectors/appid/test/appid_http_session_test.cc
index 2011dce88..cf1270cba 100644
--- a/src/network_inspectors/appid/test/appid_http_session_test.cc
+++ b/src/network_inspectors/appid/test/appid_http_session_test.cc
@@ -38,6 +38,7 @@
 #include "appid_mock_definitions.h"
 #include "appid_mock_inspector.h"
 #include "appid_mock_flow.h"
+#include "appid_peg_counts.h"
 
 #include "detector_plugins/http_url_patterns.h"
 
@@ -101,12 +102,10 @@ OdpContext* AppIdContext::odp_ctxt = &stub_odp_ctxt;
 // AppIdSession mock functions
 AppIdSession::AppIdSession(IpProtocol, const SfIp*, uint16_t, AppIdInspector& inspector)
     : FlowData(inspector_id, &inspector), ctxt(stub_ctxt)
-{
-}
+{}
 
 AppIdSession::~AppIdSession()
-{
-}
+{}
 
 void AppIdSession::set_client_appid_data(AppId, AppidChangeBits&, char*)
 {
diff --git a/src/network_inspectors/appid/test/appid_mock_definitions.h b/src/network_inspectors/appid/test/appid_mock_definitions.h
index b26eec09d..d41b20ed8 100644
--- a/src/network_inspectors/appid/test/appid_mock_definitions.h
+++ b/src/network_inspectors/appid/test/appid_mock_definitions.h
@@ -21,7 +21,11 @@
 #ifndef APPID_MOCK_DEFINITIONS_H
 #define APPID_MOCK_DEFINITIONS_H
 
+#include "appid_detector.h"
+#include "appid_module.h"
+#include "appid_peg_counts.h"
 #include "service_inspectors/http_inspect/http_msg_header.h"
+#include "utils/stats.h"
 
 class Inspector;
 class ThirdPartyAppIdContext;
@@ -54,6 +58,18 @@ SearchTool::SearchTool(char const*, bool) { }
 SearchTool::~SearchTool() { }
 }
 
+void ApplicationDescriptor::set_id(AppId app_id){ my_id = app_id;}
+void ServiceAppDescriptor::set_id(AppId app_id, OdpContext&){ set_id(app_id); }
+void ServiceAppDescriptor::update_stats(AppId){}
+void ServiceAppDescriptor::set_port_service_id(AppId app_id){ port_service_id = app_id;}
+void ClientAppDescriptor::update_user(AppId app_id, const char* username)
+{
+    my_username = username;
+    my_user_id = app_id;
+}
+void ClientAppDescriptor::update_stats(AppId) {}
+void PayloadAppDescriptor::update_stats(AppId) {}
+
 AppIdDiscovery::AppIdDiscovery() { }
 AppIdDiscovery::~AppIdDiscovery() { }
 void ClientDiscovery::initialize() { }
diff --git a/src/network_inspectors/appid/test/appid_mock_session.h b/src/network_inspectors/appid/test/appid_mock_session.h
index 96fa00a47..dcb79d297 100644
--- a/src/network_inspectors/appid/test/appid_mock_session.h
+++ b/src/network_inspectors/appid/test/appid_mock_session.h
@@ -112,7 +112,8 @@ AppIdSession::AppIdSession(IpProtocol proto, const SfIp*, uint16_t, AppIdInspect
 
 AppIdSession::~AppIdSession()
 {
-    delete hsession;
+    for (auto* hsession: hsessions)
+        delete hsession;
     delete tsession;
     delete dsession;
     if (netbios_name)
@@ -245,14 +246,26 @@ bool AppIdSession::is_ssl_session_decrypted()
 {
     return is_session_decrypted;
 }
-
-AppIdHttpSession* AppIdSession::get_http_session()
+AppIdHttpSession* AppIdSession::create_http_session()
 {
-    if ( !hsession )
-        hsession = new MockAppIdHttpSession(*this);
+    AppIdHttpSession* hsession = new MockAppIdHttpSession(*this);
+    hsession->client.set_id(APPID_UT_ID);
+    hsession->payload.set_id(APPID_UT_ID);
+    hsession->misc_app_id = APPID_UT_ID;
+    hsession->referred_payload_app_id = APPID_UT_ID;
+    hsessions.push_back(hsession);
     return hsession;
 }
 
+AppIdHttpSession* AppIdSession::get_http_session(uint32_t stream_index)
+{
+    if (stream_index < hsessions.size())
+    {
+        return hsessions[stream_index];
+    }
+    return nullptr;
+}
+
 AppIdDnsSession* AppIdSession::get_dns_session()
 {
     if ( !dsession )
diff --git a/src/network_inspectors/appid/test/appid_session_api_test.cc b/src/network_inspectors/appid/test/appid_session_api_test.cc
index 599e2f1d9..abddc1396 100644
--- a/src/network_inspectors/appid/test/appid_session_api_test.cc
+++ b/src/network_inspectors/appid/test/appid_session_api_test.cc
@@ -87,26 +87,50 @@ TEST(appid_session_api, get_only_service_app_id)
 
 TEST(appid_session_api, get_misc_app_id)
 {
+    mock_session->is_http2 = false;
     AppId id = appid_session_api->get_misc_app_id();
     CHECK_EQUAL(id, APPID_UT_ID);
+    mock_session->is_http2 = true;
+    id = appid_session_api->get_client_app_id(0);
+    CHECK_EQUAL(APPID_UT_ID, id);
+    id = appid_session_api->get_client_app_id(3);
+    CHECK_EQUAL(APP_ID_UNKNOWN, id);
 }
 
 TEST(appid_session_api, get_client_app_id)
 {
+    mock_session->is_http2 = false;
     AppId id = appid_session_api->get_client_app_id();
     CHECK_EQUAL(id, APPID_UT_ID);
+    mock_session->is_http2 = true;
+    id = appid_session_api->get_client_app_id(0);
+    CHECK_EQUAL(APPID_UT_ID, id);
+    id = appid_session_api->get_client_app_id(3);
+    CHECK_EQUAL(APP_ID_UNKNOWN, id);
 }
 
 TEST(appid_session_api, get_payload_app_id)
 {
+    mock_session->is_http2 = false;
     AppId id = appid_session_api->get_payload_app_id();
     CHECK_EQUAL(id, APPID_UT_ID);
+    mock_session->is_http2 = true;
+    id = appid_session_api->get_payload_app_id(0);
+    CHECK_EQUAL(APPID_UT_ID, id);
+    id = appid_session_api->get_payload_app_id(2);
+    CHECK_EQUAL(APP_ID_UNKNOWN, id);
 }
 
 TEST(appid_session_api, get_referred_app_id)
 {
+    mock_session->is_http2 = false;
     AppId id = appid_session_api->get_referred_app_id();
     CHECK_EQUAL(id, APPID_UT_ID);
+    mock_session->is_http2 = true;
+    id = appid_session_api->get_payload_app_id(0);
+    CHECK_EQUAL(APPID_UT_ID, id);
+    id = appid_session_api->get_payload_app_id(2);
+    CHECK_EQUAL(APP_ID_UNKNOWN, id);
 }
 
 TEST(appid_session_api, get_service_port)
@@ -243,8 +267,20 @@ TEST(appid_session_api, get_client_version)
     const char* val;
     val = appid_session_api->get_client_version();
     STRCMP_EQUAL(val, APPID_UT_CLIENT_VERSION);
+    val = appid_session_api->get_client_version(0);
+    STRCMP_EQUAL(APPID_UT_CLIENT_VERSION, val);
+    mock_session->is_http2 = true;
+    val = appid_session_api->get_client_version(2);
+    STRCMP_EQUAL(nullptr, val);
+}
+TEST(appid_session_api, get_http_session)
+{
+    AppIdHttpSession* val;
+    val = appid_session_api->get_http_session();
+    CHECK_TRUE(val != nullptr);
+    val = appid_session_api->get_http_session(2);
+    CHECK_TRUE(val == nullptr);
 }
-
 TEST(appid_session_api, get_appid_session_attribute)
 {
     uint64_t flags = 0x0000000000000001;
@@ -372,6 +408,7 @@ int main(int argc, char** argv)
 {
     mock_init_appid_pegs();
     mock_session = new AppIdSession(IpProtocol::TCP, nullptr, 1492, appid_inspector);
+    mock_session->create_http_session();
     int rc = CommandLineTestRunner::RunAllTests(argc, argv);
     mock_cleanup_appid_pegs();
     return rc;
diff --git a/src/network_inspectors/appid/test/service_state_test.cc b/src/network_inspectors/appid/test/service_state_test.cc
index 672f88bf0..158b00320 100644
--- a/src/network_inspectors/appid/test/service_state_test.cc
+++ b/src/network_inspectors/appid/test/service_state_test.cc
@@ -72,6 +72,13 @@ THREAD_LOCAL AppIdStats appid_stats;
 void AppIdDebug::activate(const Flow*, const AppIdSession*, bool) { active = true; }
 
 void ApplicationDescriptor::set_id(const Packet&, AppIdSession&, AppidSessionDirection, AppId, AppidChangeBits&) { }
+void ApplicationDescriptor::set_id(AppId){}
+void ServiceAppDescriptor::set_id(AppId, OdpContext&){}
+void ServiceAppDescriptor::update_stats(AppId){}
+void ServiceAppDescriptor::set_port_service_id(AppId){}
+void ClientAppDescriptor::update_user(AppId, const char*){}
+void ClientAppDescriptor::update_stats(AppId) {}
+void PayloadAppDescriptor::update_stats(AppId) {}
 AppIdConfig::~AppIdConfig() { }
 AppIdConfig stub_config;
 AppIdContext stub_ctxt(stub_config);
diff --git a/src/network_inspectors/appid/tp_appid_utils.cc b/src/network_inspectors/appid/tp_appid_utils.cc
index 923635b48..f26d66eaa 100644
--- a/src/network_inspectors/appid/tp_appid_utils.cc
+++ b/src/network_inspectors/appid/tp_appid_utils.cc
@@ -86,7 +86,9 @@ static inline int check_ssl_appid_for_reinspect(AppId app_id, OdpContext& odp_ct
 static inline void process_http_session(AppIdSession& asd,
     ThirdPartyAppIDAttributeData& attribute_data, AppidChangeBits& change_bits)
 {
-    AppIdHttpSession* hsession = asd.get_http_session();
+    AppIdHttpSession* hsession = asd.get_http_session(0);
+    if (!hsession)
+        hsession = asd.create_http_session();
     string* field=0;
     bool own=true;
 
@@ -398,6 +400,8 @@ static inline void process_rtmp(AppIdSession& asd,
     ThirdPartyAppIDAttributeData& attribute_data, int confidence, AppidChangeBits& change_bits)
 {
     AppIdHttpSession* hsession = asd.get_http_session();
+    if (!hsession)
+        hsession = asd.create_http_session();
     AppId service_id = 0;
     AppId client_id = 0;
     AppId payload_id = 0;
@@ -799,6 +803,8 @@ bool do_tp_discovery(ThirdPartyAppIdContext& tp_appid_ctxt, AppIdSession& asd, I
                     }
 
                     AppIdHttpSession* hsession = asd.get_http_session();
+                    if (!hsession)
+                        hsession = asd.create_http_session();
                     hsession->process_http_packet(direction, change_bits, asd.ctxt.get_odp_ctxt().get_http_matchers());
 
                     // If SSL over HTTP tunnel, make sure Snort knows that it's encrypted.